diff --git a/.github/workflows/e2e-dev.yml b/.github/workflows/e2e-dev.yml new file mode 100644 index 000000000..bd47a682e --- /dev/null +++ b/.github/workflows/e2e-dev.yml @@ -0,0 +1,163 @@ +# E2E suite against the dev cloud stack — SDK → dev API → dev Runner → VM. +# +# Builds Python SDK, Node SDK, and the CLI from source using +# BOXLITE_DEPS_STUB=1 (skips vendor submodule builds — those only +# matter inside the runner, not the SDK client), then runs the REST +# e2e pytest suite against the dev environment. +# +# C and Go SDK tests are excluded — those SDKs wrap libboxlite.a over +# REST purely to test FFI bindings, not a real user path. Python, +# Node, and CLI cover the actual user-facing REST surface. +# +# Does NOT build or deploy the API or runner — the dev stack is +# assumed to be already running the version you want to test against. +# +# Authentication: boxlite API key stored as GitHub repo secret +# `BOXLITE_DEV_API_KEY`. No AWS credentials needed. + +name: E2E dev + +on: + push: + branches: + - chore/e2e-required-merge-gate + workflow_dispatch: + inputs: + api_url: + description: 'Dev API base URL' + type: string + required: false + default: 'https://api.dev.boxlite.ai/api' + +permissions: + contents: read + +concurrency: + group: e2e-dev + cancel-in-progress: true + +env: + BOXLITE_DEPS_STUB: "1" + API_URL: ${{ inputs.api_url || 'https://api.dev.boxlite.ai/api' }} + +jobs: + e2e: + name: E2E suite (dev) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v5 + + # ── Health check ────────────────────────────────────────────── + - name: Probe API health + run: | + curl -fsS --retry 6 --retry-delay 5 --retry-all-errors --max-time 10 \ + -o /dev/null "${API_URL}/health" + echo "::notice::${API_URL}/health returned 2xx" + + # ── System deps ────────────────────────────────────────────── + - name: Install system dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends protobuf-compiler + + # ── Rust toolchain ─────────────────────────────────────────── + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + + # ── CLI (cargo build, warms the shared boxlite core) ───────── + - name: Build CLI + run: | + cargo build --release -p boxlite-cli + sudo install -m 0755 target/release/boxlite /usr/local/bin/boxlite + boxlite --version + + # ── Node SDK (napi-rs) ─────────────────────────────────────── + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Build Node SDK (napi + tsc) + run: | + cd sdks/node + npm install --ignore-scripts + npm run build:native + npm run build + ls -lh native/ dist/ + + # ── Python SDK (maturin + PyO3) ────────────────────────────── + - name: Build & install Python SDK + run: | + cd sdks/python + pip install --break-system-packages --quiet maturin + maturin build --release + pip install --break-system-packages --force-reinstall ../../target/wheels/boxlite-*.whl + pip install --break-system-packages --quiet pytest pytest-asyncio pytest-timeout + + # ── Configure pytest credentials ───────────────────────────── + - name: Configure pytest profile p1 + env: + BOXLITE_DEV_API_KEY: ${{ secrets.BOXLITE_DEV_API_KEY }} + run: | + set -euo pipefail + + if [ -z "$BOXLITE_DEV_API_KEY" ]; then + echo "::error::Secret BOXLITE_DEV_API_KEY is not set. Add it in repo Settings → Secrets." + exit 1 + fi + echo "::add-mask::$BOXLITE_DEV_API_KEY" + + PATH_PREFIX=$(curl -fsS -H "Authorization: Bearer $BOXLITE_DEV_API_KEY" \ + "${API_URL}/v1/me" | python3 -c 'import json,sys; print(json.load(sys.stdin)["path_prefix"])') + [ -n "$PATH_PREFIX" ] || { echo "::error::Empty path_prefix from /v1/me"; exit 1; } + echo "Resolved path_prefix=${PATH_PREFIX}" + + mkdir -p ~/.boxlite + chmod 700 ~/.boxlite + { + printf '[profiles.p1]\n' + printf 'url = "%s"\n' "$API_URL" + printf 'api_key = "%s"\n' "$BOXLITE_DEV_API_KEY" + printf 'path_prefix = "%s"\n' "$PATH_PREFIX" + } > ~/.boxlite/credentials.toml + chmod 600 ~/.boxlite/credentials.toml + + # ── Run pytest ─────────────────────────────────────────────── + # --ignore rationale: + # - C/Go entry+coverage: FFI-binding tests whose SDKs aren't built here. + # - node-entry: needs runner journal access (unreachable from GHA + # against remote dev); node-coverage already covers the Node REST + # surface without journal checks. + # - exec-timeout: the timeout fires and kills the process on time, but + # the guest kills only the direct child — an orphaned grandchild + # (e.g. `sh`'s `sleep`) keeps the stdout pipe open, so the attach + # stream never EOFs and ex.wait() hangs to the 45s bound. Guest-side + # root cause + fix tracked in #910. Un-ignore once that lands. The + # kill itself is covered by sdks/python/tests/test_exec_timeout_sigalrm.py. + # test_path_verification is NOT ignored — despite the name it never + # touches the journal; it proves the API→Runner chain via the + # X-BoxLite-Api-Version response header + an exec marker. + - name: Run E2E suite + env: + BOXLITE_E2E_SKIP_PATH_VERIFY: '1' + BOXLITE_E2E_PROFILE: p1 + run: | + timeout 35m python3 -m pytest scripts/test/e2e/cases/ \ + --ignore=scripts/test/e2e/cases/test_c_entry.py \ + --ignore=scripts/test/e2e/cases/test_c_coverage.py \ + --ignore=scripts/test/e2e/cases/test_go_entry.py \ + --ignore=scripts/test/e2e/cases/test_go_coverage.py \ + --ignore=scripts/test/e2e/cases/test_node_entry.py \ + --ignore=scripts/test/e2e/cases/test_exec_timeout.py \ + -v -s --tb=short --no-header -p no:cacheprovider \ + --timeout=180 --timeout-method=thread \ + --junit-xml=pytest-junit.xml + + - name: Upload pytest junit XML + if: always() + uses: actions/upload-artifact@v4 + with: + name: pytest-junit + path: pytest-junit.xml + if-no-files-found: ignore diff --git a/scripts/deploy/build-runner-binary.sh b/scripts/deploy/build-runner-binary.sh new file mode 100755 index 000000000..29be05696 --- /dev/null +++ b/scripts/deploy/build-runner-binary.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash +# Build the Linux amd64 boxlite-runner tarball without GitHub Actions. +# +# Run this from the checkout you want to deploy. To deploy latest main to dev: +# git checkout main +# git pull --ff-only origin main +# scripts/deploy/build-runner-binary.sh --output-dir dist +# The generated tarball is intentionally named +# `v{VERSION}-dev-{BUILD_SEQUENCE}-{GUEST_HASH}`. This script is for +# non-published builds, so the embedded runtime cache must not share the +# official release directory `v{VERSION}`. +# +# The runner is a cgo binary that links libboxlite.a. Build the embedded runtime +# inputs first (boxlite-shim + boxlite-guest), force boxlite's build.rs to rerun +# so those binaries are embedded, then build/stage the C SDK archive for Go. +# +# Usage: +# scripts/deploy/build-runner-binary.sh +# scripts/deploy/build-runner-binary.sh --output-dir /tmp +# BOXLITE_RUNNER_BUILD_NUMBER=123 scripts/deploy/build-runner-binary.sh +# scripts/deploy/build-runner-binary.sh --skip-setup + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +OUTPUT_DIR="$ROOT_DIR/dist" +RUN_SETUP=1 +BUILD_SEQUENCE="${BOXLITE_RUNNER_BUILD_NUMBER:-${GITHUB_RUN_NUMBER:-}}" + +usage() { + sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//' + cat <<'EOF' +Options: + --output-dir DIR Directory for the runner tarball. Default: ./dist + --build-number ID Ordered dev build sequence. Default: BOXLITE_RUNNER_BUILD_NUMBER, + GITHUB_RUN_NUMBER, or current UTC timestamp. + --skip-setup Skip `make setup:build`. + -h, --help Show this help. +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --output-dir) + [[ $# -ge 2 ]] || { echo "error: --output-dir requires a value" >&2; exit 1; } + OUTPUT_DIR="$2" + shift 2 + ;; + --build-number) + [[ $# -ge 2 ]] || { echo "error: --build-number requires a value" >&2; exit 1; } + BUILD_SEQUENCE="$2" + shift 2 + ;; + --skip-setup) + RUN_SETUP=0 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "error: unknown argument $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +cd "$ROOT_DIR" + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || { echo "error: required command not found: $1" >&2; exit 1; } +} + +require_cmd cargo +require_cmd go +require_cmd make +require_cmd sha256sum +require_cmd tar + +find_embedded_runtime_dir() { + local guest_sha256="$1" + local runtime_dir + local runtime_guest_sha256 + + while IFS= read -r runtime_dir; do + [ -f "$runtime_dir/boxlite-guest" ] || continue + runtime_guest_sha256="$(sha256sum "$runtime_dir/boxlite-guest" | awk '{print $1}')" + if [[ "$runtime_guest_sha256" == "$guest_sha256" ]]; then + printf '%s\n' "$runtime_dir" + return 0 + fi + done < <(find "$ROOT_DIR/target/release/build" -maxdepth 3 -path '*/boxlite-*/out/runtime' -type d 2>/dev/null | sort) + + echo "error: embedded runtime payload not found for guest hash ${guest_sha256:0:12}" >&2 + exit 1 +} + +if [[ "$(uname -s)" != "Linux" || "$(uname -m)" != "x86_64" ]]; then + echo "error: runner tarball build currently requires a Linux x86_64 builder" >&2 + exit 1 +fi + +VERSION="$(grep -m 1 '^version' "$ROOT_DIR/Cargo.toml" | sed -E 's/^version *= *"([^"]+)".*/\1/')" +if [[ -z "$VERSION" ]]; then + echo "error: could not read version from Cargo.toml" >&2 + exit 1 +fi + +if [[ -z "$BUILD_SEQUENCE" ]]; then + BUILD_SEQUENCE="$(date -u +%Y%m%d%H%M%S)" +fi +BUILD_SEQUENCE="$(printf '%s' "$BUILD_SEQUENCE" | tr -c 'A-Za-z0-9._-' '-')" +BUILD_SEQUENCE="${BUILD_SEQUENCE#v}" +[[ -n "$BUILD_SEQUENCE" ]] || { echo "error: build sequence cannot be empty" >&2; exit 1; } + +TMP_DIR="$(mktemp -d)" +GO_WORK_BACKUP="$TMP_DIR/go.work.backup" +GO_WORK_SUM_BACKUP="$TMP_DIR/go.work.sum.backup" +HAD_GO_WORK=0 +HAD_GO_WORK_SUM=0 + +restore_go_work() { + if [[ "$HAD_GO_WORK" -eq 1 ]]; then + cp "$GO_WORK_BACKUP" "$ROOT_DIR/apps/go.work" + else + rm -f "$ROOT_DIR/apps/go.work" + fi + + if [[ "$HAD_GO_WORK_SUM" -eq 1 ]]; then + cp "$GO_WORK_SUM_BACKUP" "$ROOT_DIR/apps/go.work.sum" + else + rm -f "$ROOT_DIR/apps/go.work.sum" + fi + + rm -rf "$TMP_DIR" +} +trap restore_go_work EXIT + +if [[ -f "$ROOT_DIR/apps/go.work" ]]; then + HAD_GO_WORK=1 + cp "$ROOT_DIR/apps/go.work" "$GO_WORK_BACKUP" +fi +if [[ -f "$ROOT_DIR/apps/go.work.sum" ]]; then + HAD_GO_WORK_SUM=1 + cp "$ROOT_DIR/apps/go.work.sum" "$GO_WORK_SUM_BACKUP" +fi + +GO_VERSION="$(awk '/^go / { print $2; exit }' "$ROOT_DIR/apps/runner/go.mod")" +if [[ -z "$GO_VERSION" ]]; then + echo "error: could not read Go version from apps/runner/go.mod" >&2 + exit 1 +fi + +cat > "$ROOT_DIR/apps/go.work" < Building boxlite-runner from package v$VERSION at $(git rev-parse --short HEAD)" +echo "==> Non-release artifact version prefix: v${VERSION}-dev-${BUILD_SEQUENCE}" + +echo "==> Cleaning runner build artifacts" +cargo clean -p boxlite -p boxlite-c -p boxlite-shim -p boxlite-guest +rm -f \ + "$ROOT_DIR/sdks/go/libboxlite.a" \ + "$ROOT_DIR/target/release/libboxlite.a" \ + "$ROOT_DIR/target/release/libboxlite.so" \ + "$ROOT_DIR/target/release/boxlite-shim" +rm -f "$ROOT_DIR/target/x86_64-unknown-linux-gnu/release/boxlite-shim" +rm -f "$ROOT_DIR/target/x86_64-unknown-linux-musl/release/boxlite-guest" +rm -rf "$ROOT_DIR/sdks/c/dist" + +if [[ "$RUN_SETUP" -eq 1 ]]; then + make setup:build +fi + +scripts/build/build-shim.sh --profile release +scripts/build/build-guest.sh --profile release +GUEST_BIN="$ROOT_DIR/target/x86_64-unknown-linux-musl/release/boxlite-guest" +[[ -x "$GUEST_BIN" ]] || { echo "error: guest binary not found after build: $GUEST_BIN" >&2; exit 1; } +GUEST_SHA256="$(sha256sum "$GUEST_BIN" | awk '{print $1}')" +RUNTIME_SUFFIX="${GUEST_SHA256:0:12}" +RUNTIME_CACHE_SUFFIX="dev-${BUILD_SEQUENCE}-${RUNTIME_SUFFIX}" +RUNNER_VERSION="${VERSION}-${RUNTIME_CACHE_SUFFIX}" + +echo "==> Building libboxlite with runtime cache key v${RUNNER_VERSION}" +make dist:c +echo "==> Fixing libboxlite Go runtime symbols" +bash "$ROOT_DIR/scripts/build/fix-go-symbols.sh" "$ROOT_DIR/target/release/libboxlite.a" +RUNTIME_DIR="$(find_embedded_runtime_dir "$GUEST_SHA256")" +tar czf "$TMP_DIR/boxlite-runtime.tar.gz" -C "$RUNTIME_DIR" . +echo "==> Wrote embedded runtime payload from $RUNTIME_DIR" +cp "$ROOT_DIR/target/release/libboxlite.a" "$ROOT_DIR/sdks/go/libboxlite.a" + +go -C apps/runner mod download +go -C apps/common-go mod download +go -C apps/api-client-go mod download + +RUNNER_BIN="$TMP_DIR/boxlite-runner" +CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -C apps \ + -ldflags "-X github.com/boxlite-ai/runner/internal.Version=${RUNNER_VERSION}" \ + -o "$RUNNER_BIN" ./runner/cmd/runner +printf '%s boxlite-guest\n' "$GUEST_SHA256" > "$TMP_DIR/boxlite-runner.guest.sha256" +echo "==> Wrote guest hash sidecar ${GUEST_SHA256:0:12}" +printf '%s\n' "$RUNTIME_CACHE_SUFFIX" > "$TMP_DIR/boxlite-runner.runtime-suffix" +echo "==> Runtime cache directory: v${RUNNER_VERSION}" + +mkdir -p "$OUTPUT_DIR" +TARBALL="$OUTPUT_DIR/boxlite-runner-v${RUNNER_VERSION}-linux-amd64.tar.gz" +tar czf "$TARBALL" -C "$TMP_DIR" \ + boxlite-runner \ + boxlite-runner.guest.sha256 \ + boxlite-runner.runtime-suffix \ + boxlite-runtime.tar.gz + +if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$TARBALL" > "$TARBALL.sha256" +else + shasum -a 256 "$TARBALL" > "$TARBALL.sha256" +fi + +echo "==> Wrote $TARBALL" +echo "==> Wrote $TARBALL.sha256" diff --git a/scripts/deploy/runner-update-binary.sh b/scripts/deploy/runner-update-binary.sh index d4eeccbdb..476fca2f0 100755 --- a/scripts/deploy/runner-update-binary.sh +++ b/scripts/deploy/runner-update-binary.sh @@ -1,9 +1,13 @@ #!/usr/bin/env bash # Upgrade the boxlite-runner binary on the live Runner EC2 in-place. # -# Replaces /usr/local/bin/boxlite-runner with a freshly downloaded release -# binary and restarts the systemd unit. The EC2 instance itself is not -# replaced; box state under /var/lib/boxlite is preserved. +# Installs the new binary into a versioned release directory, points +# /usr/local/bin/boxlite-runner at it, restarts the systemd unit, and verifies +# that detached box shims that were alive before the restart are still alive and +# can be re-attached by the new runner. The EC2 instance itself is not replaced; +# box state under /var/lib/boxlite is preserved. Official versions are fetched +# directly from GitHub Releases by the target runner; dev builds upload the +# locally built runner tarball to a temporary S3 location first. # # Pair with the `ignoreChanges: ["ami", "userDataBase64"]` setting on the # Runner resource in apps/infra/sst.config.ts — that prevents `sst deploy` @@ -12,86 +16,650 @@ # # Usage: # scripts/deploy/runner-update-binary.sh # version from Cargo.toml -# scripts/deploy/runner-update-binary.sh 0.9.5 # explicit version +# scripts/deploy/runner-update-binary.sh 0.9.7 # official GitHub release +# scripts/deploy/runner-update-binary.sh 0.9.7-dev-123-58d8f01bcd02 +# scripts/deploy/runner-update-binary.sh --output-dir /tmp/dist 0.9.7-dev-123-58d8f01bcd02 +# scripts/deploy/runner-update-binary.sh --tarball dist/boxlite-runner-v0.9.7-dev-123-58d8f01bcd02-linux-amd64.tar.gz # AWS_REGION=us-west-2 scripts/deploy/runner-update-binary.sh # STAGE=production scripts/deploy/runner-update-binary.sh +# RUNNER_INSTANCE_ID=i-0123456789abcdef0 scripts/deploy/runner-update-binary.sh +# PROD_RUNNER_INSTANCE_ID=i-0123456789abcdef0 STAGE=production scripts/deploy/runner-update-binary.sh set -euo pipefail -AWS_REGION="${AWS_REGION:-ap-southeast-1}" STAGE="${STAGE:-dev}" +DEV_AWS_REGION="${DEV_AWS_REGION:-ap-southeast-1}" +DEV_RUNNER_INSTANCE_NAME="${DEV_RUNNER_INSTANCE_NAME:-boxlite-dev-runner-default}" +DEV_RUNNER_INSTANCE_ID="${DEV_RUNNER_INSTANCE_ID:-}" +PROD_AWS_REGION="${PROD_AWS_REGION:-us-west-2}" +PROD_RUNNER_INSTANCE_NAME="${PROD_RUNNER_INSTANCE_NAME:-boxlite-prod-runner-default}" +PROD_RUNNER_INSTANCE_ID="${PROD_RUNNER_INSTANCE_ID:-}" +AWS_REGION="${AWS_REGION:-}" +RUNNER_INSTANCE_NAME="${RUNNER_INSTANCE_NAME:-}" +RUNNER_INSTANCE_ID="${RUNNER_INSTANCE_ID:-}" REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +ARTIFACT_DIR="${RUNNER_ARTIFACT_DIR:-$REPO_ROOT/dist}" +RUNNER_ARTIFACT_S3_URI="${RUNNER_ARTIFACT_S3_URI:-}" +RUNNER_DOWNLOAD_CONNECT_TIMEOUT_SECONDS="${RUNNER_DOWNLOAD_CONNECT_TIMEOUT_SECONDS:-15}" +RUNNER_DOWNLOAD_MAX_TIME_SECONDS="${RUNNER_DOWNLOAD_MAX_TIME_SECONDS:-600}" +RUNNER_CHECKSUM_DOWNLOAD_MAX_TIME_SECONDS="${RUNNER_CHECKSUM_DOWNLOAD_MAX_TIME_SECONDS:-120}" +SSM_WAIT_TIMEOUT_SECONDS="${SSM_WAIT_TIMEOUT_SECONDS:-1800}" +SSM_WAIT_POLL_SECONDS="${SSM_WAIT_POLL_SECONDS:-10}" +VERSION="" +LOCAL_TARBALL_OVERRIDE="" -if [[ $# -ge 1 ]]; then - VERSION="$1" -else - VERSION=$(grep -m 1 '^version' "$REPO_ROOT/Cargo.toml" | sed -E 's/^version *= *"([^"]+)".*/\1/') +usage() { + sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//' +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --output-dir|--artifact-dir) + [[ $# -ge 2 ]] || { echo "error: $1 requires a value" >&2; exit 1; } + ARTIFACT_DIR="$2" + shift 2 + ;; + --version) + [[ $# -ge 2 ]] || { echo "error: --version requires a value" >&2; exit 1; } + VERSION="$2" + shift 2 + ;; + --tarball) + [[ $# -ge 2 ]] || { echo "error: --tarball requires a value" >&2; exit 1; } + LOCAL_TARBALL_OVERRIDE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + -*) + echo "error: unknown option $1" >&2 + usage >&2 + exit 1 + ;; + *) + if [[ -n "$VERSION" ]]; then + echo "error: unexpected extra argument $1" >&2 + usage >&2 + exit 1 + fi + VERSION="$1" + shift + ;; + esac +done + +command -v aws >/dev/null 2>&1 || { echo "error: required command not found: aws" >&2; exit 1; } + +case "$STAGE" in + dev|production) ;; + prod) STAGE="production" ;; + *) + echo "error: STAGE must be dev or production (got: $STAGE)" >&2 + exit 1 + ;; +esac + +case "$STAGE" in + dev) + AWS_REGION="${AWS_REGION:-$DEV_AWS_REGION}" + RUNNER_INSTANCE_ID="${RUNNER_INSTANCE_ID:-$DEV_RUNNER_INSTANCE_ID}" + RUNNER_INSTANCE_NAME="${RUNNER_INSTANCE_NAME:-$DEV_RUNNER_INSTANCE_NAME}" + ;; + production) + AWS_REGION="${AWS_REGION:-$PROD_AWS_REGION}" + RUNNER_INSTANCE_ID="${RUNNER_INSTANCE_ID:-$PROD_RUNNER_INSTANCE_ID}" + RUNNER_INSTANCE_NAME="${RUNNER_INSTANCE_NAME:-$PROD_RUNNER_INSTANCE_NAME}" + ;; +esac + +if [[ -z "$VERSION" ]]; then + if [[ -n "$LOCAL_TARBALL_OVERRIDE" ]]; then + TARBALL_BASENAME="$(basename "$LOCAL_TARBALL_OVERRIDE")" + VERSION="$(printf '%s\n' "$TARBALL_BASENAME" | sed -E 's/^boxlite-runner-v(.+)-linux-amd64\.tar\.gz$/\1/')" + if [[ "$VERSION" == "$TARBALL_BASENAME" ]]; then + echo "error: could not infer version from tarball name: $TARBALL_BASENAME" >&2 + exit 1 + fi + else + VERSION=$(grep -m 1 '^version' "$REPO_ROOT/Cargo.toml" | sed -E 's/^version *= *"([^"]+)".*/\1/') + fi if [[ -z "$VERSION" ]]; then echo "error: could not read version from Cargo.toml at $REPO_ROOT/Cargo.toml" >&2 exit 1 fi fi -echo "==> Upgrading boxlite-runner to v$VERSION on stage=$STAGE region=$AWS_REGION" +ASSET_TARBALL="boxlite-runner-v${VERSION}-linux-amd64.tar.gz" +LOCAL_TARBALL="${LOCAL_TARBALL_OVERRIDE:-$ARTIFACT_DIR/$ASSET_TARBALL}" +LOCAL_SHA="$LOCAL_TARBALL.sha256" +RUNTIME_CACHE_VERSION="${VERSION%%-dev-*}" +IS_DEV_VERSION=0 +if [[ "$VERSION" == *-dev-* ]]; then + IS_DEV_VERSION=1 +fi + +if [[ "$IS_DEV_VERSION" -eq 1 ]]; then + if [[ ! -f "$LOCAL_TARBALL" ]]; then + echo "error: local dev runner tarball not found: $LOCAL_TARBALL" >&2 + echo " run scripts/deploy/build-runner-binary.sh first, or set --output-dir/RUNNER_ARTIFACT_DIR" >&2 + exit 1 + fi + if [[ ! -f "$LOCAL_SHA" ]]; then + echo "error: local dev runner checksum not found: $LOCAL_SHA" >&2 + echo " run scripts/deploy/build-runner-binary.sh first" >&2 + exit 1 + fi +fi + +if [[ "$IS_DEV_VERSION" -eq 1 ]]; then + echo "==> Upgrading boxlite-runner from local dev artifact v$VERSION on stage=$STAGE region=$AWS_REGION" +else + echo "==> Upgrading boxlite-runner from GitHub release v$VERSION on stage=$STAGE region=$AWS_REGION" +fi -INSTANCE_ID=$(aws ec2 describe-instances --region "$AWS_REGION" \ - --filters "Name=tag:Name,Values=boxlite-runner-default" "Name=instance-state-name,Values=running" \ - --query 'Reservations[].Instances[].InstanceId' --output text) +if [[ -n "$RUNNER_INSTANCE_ID" ]]; then + INSTANCE_ID="$RUNNER_INSTANCE_ID" +else + INSTANCE_FILTERS=( + "Name=tag:Name,Values=${RUNNER_INSTANCE_NAME}" + "Name=instance-state-name,Values=running" + ) + INSTANCE_IDS=() + while IFS= read -r instance_id; do + INSTANCE_IDS+=("$instance_id") + done < <(aws ec2 describe-instances --region "$AWS_REGION" \ + --filters "${INSTANCE_FILTERS[@]}" \ + --query 'Reservations[].Instances[].InstanceId' --output text | tr '\t' '\n' | sed '/^$/d') -if [[ -z "$INSTANCE_ID" || "$INSTANCE_ID" == "None" ]]; then - echo "error: no running boxlite-runner-default instance found in region $AWS_REGION" >&2 - exit 1 + if [[ "${#INSTANCE_IDS[@]}" -ne 1 ]]; then + echo "error: expected exactly one running runner instance, found ${#INSTANCE_IDS[@]}" >&2 + echo " region=$AWS_REGION stage=$STAGE name=$RUNNER_INSTANCE_NAME" >&2 + if [[ "${#INSTANCE_IDS[@]}" -gt 0 ]]; then + printf ' matches: %s\n' "${INSTANCE_IDS[*]}" >&2 + fi + echo " set RUNNER_INSTANCE_ID=i-... to target an instance explicitly" >&2 + exit 1 + fi + INSTANCE_ID="${INSTANCE_IDS[0]}" fi echo " instance: $INSTANCE_ID" -ASSET_BASE="https://github.com/boxlite-ai/boxlite/releases/download/v${VERSION}" -ASSET_TARBALL="boxlite-runner-v${VERSION}-linux-amd64.tar.gz" +if [[ "$IS_DEV_VERSION" -eq 1 ]]; then + if [[ -z "$RUNNER_ARTIFACT_S3_URI" ]]; then + ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + RUNNER_ARTIFACT_S3_URI="s3://boxlite-${STAGE}-runner-builds/tmp/runner-rollouts/${ACCOUNT_ID}/${VERSION}/$(date -u +%Y%m%dT%H%M%SZ)" + fi + REMOTE_TARBALL_URL="${RUNNER_ARTIFACT_S3_URI%/}/$ASSET_TARBALL" + REMOTE_SHA_URL="$REMOTE_TARBALL_URL.sha256" + echo "==> Uploading local artifact to $RUNNER_ARTIFACT_S3_URI" + aws s3 cp --region "$AWS_REGION" "$LOCAL_TARBALL" "$REMOTE_TARBALL_URL" + aws s3 cp --region "$AWS_REGION" "$LOCAL_SHA" "$REMOTE_SHA_URL" + DOWNLOAD_TARBALL_URL=$(aws s3 presign --region "$AWS_REGION" "$REMOTE_TARBALL_URL" --expires-in 3600) + DOWNLOAD_SHA_URL=$(aws s3 presign --region "$AWS_REGION" "$REMOTE_SHA_URL" --expires-in 3600) +else + RELEASE_REPOSITORY="${BOXLITE_RELEASE_REPOSITORY:-boxlite-ai/boxlite}" + RELEASE_TAG="${BOXLITE_RELEASE_TAG:-v${VERSION}}" + RELEASE_BASE_URL="https://github.com/${RELEASE_REPOSITORY}/releases/download/${RELEASE_TAG}" + DOWNLOAD_TARBALL_URL="$RELEASE_BASE_URL/$ASSET_TARBALL" + DOWNLOAD_SHA_URL="$DOWNLOAD_TARBALL_URL.sha256" + echo "==> Target runner will download $DOWNLOAD_TARBALL_URL" +fi # Remote upgrade script. Mirrors the boot user-data's integrity policy and adds a # rollback: download + checksum-verify BEFORE stopping the unit (so a failed or -# corrupt fetch never takes the runner down), back up the live binary, swap it in, -# and restore the backup if the new binary fails to come up. +# corrupt fetch never takes the runner down), install the new binary into a +# versioned release directory, switch the live symlink, and switch back if the +# new binary fails to come up. read -r -d '' SCRIPT <&2 + exit 1 + fi + if [ "\$ENABLE_TLS" = true ]; then + RUNNER_BASE_URL="https://127.0.0.1:\$API_PORT" + CURL_TLS=(-k) + else + RUNNER_BASE_URL="http://127.0.0.1:\$API_PORT" + CURL_TLS=() + fi +} + +proc_start_time() { + awk '{print \$22}' "/proc/\$1/stat" 2>/dev/null || true +} + +proc_state() { + awk '{print \$3}' "/proc/\$1/stat" 2>/dev/null || true +} + +pid_alive() { + local pid="\$1" + [ -n "\$pid" ] || return 1 + kill -0 "\$pid" 2>/dev/null || return 1 + [ "\$(proc_state "\$pid")" != Z ] +} + +snapshot_live_detached_shims() { + : > "\$HOT_SNAPSHOT" + local boxes_dir="\$BOXLITE_HOME_DIR/boxes" + [ -d "\$boxes_dir" ] || return 0 + while IFS= read -r pid_file; do + local box_id pid start actual_start + box_id="\$(basename "\$(dirname "\$pid_file")")" + pid="\$(sed -n '1p' "\$pid_file" 2>/dev/null | tr -dc '0-9')" + start="\$(sed -n '2p' "\$pid_file" 2>/dev/null | tr -dc '0-9')" + pid_alive "\$pid" || continue + if [ -n "\$start" ]; then + actual_start="\$(proc_start_time "\$pid")" + [ "\$actual_start" = "\$start" ] || continue + fi + printf '%s %s %s\n' "\$box_id" "\$pid" "\$start" >> "\$HOT_SNAPSHOT" + done < <(find "\$boxes_dir" -mindepth 2 -maxdepth 2 -name shim.pid -type f 2>/dev/null | sort) +} + +wait_runner_ready() { + local i + for i in \$(seq 1 30); do + if curl -fsS "\${CURL_TLS[@]}" "\$RUNNER_BASE_URL/" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + echo "FATAL: runner API did not become ready at \$RUNNER_BASE_URL" >&2 + return 1 +} + +verify_runner_health_samples() { + local i + for i in 1 2; do + curl -fsS "\${CURL_TLS[@]}" \ + -H "Authorization: Bearer \$BOXLITE_RUNNER_TOKEN" \ + "\$RUNNER_BASE_URL/info" >/dev/null || { + echo "FATAL: runner health sample \$i failed at \$RUNNER_BASE_URL/info" >&2 + return 1 + } + echo "runner health sample \$i: ok" + [ "\$i" -eq 2 ] || sleep 2 + done +} + +runtime_cache_dirs() { + local version_dir="\${RUNTIME_CACHE_DIR_NAME:-v${RUNTIME_CACHE_VERSION}}" + local -a data_roots=() + local svc_user svc_home env_value + + svc_user=\$(systemctl show "\$SERVICE" --property=User --value 2>/dev/null || true) + if [ -z "\$svc_user" ]; then + svc_user=root + fi + + svc_home=\$(getent passwd "\$svc_user" 2>/dev/null | cut -d: -f6 || true) + [ -n "\$svc_home" ] && data_roots+=("\$svc_home/.local/share") + + while IFS= read -r env_value; do + case "\$env_value" in + XDG_DATA_HOME=*) data_roots+=("\${env_value#XDG_DATA_HOME=}") ;; + HOME=*) data_roots+=("\${env_value#HOME=}/.local/share") ;; + esac + done < <(systemctl show "\$SERVICE" --property=Environment --value | tr ' ' '\n') + + data_roots+=("/root/.local/share") + for home in /home/*; do + [ -d "\$home" ] && data_roots+=("\$home/.local/share") + done + + local seen=" " + local root cache_dir + for root in "\${data_roots[@]}"; do + [ -n "\$root" ] || continue + case "\$seen" in *" \$root "*) continue ;; esac + seen="\$seen\$root " + cache_dir="\$root/boxlite/runtimes/\$version_dir" + [ -d "\$cache_dir" ] && printf '%s\n' "\$cache_dir" + done +} + +primary_runtime_cache_dir() { + local version_dir="\${RUNTIME_CACHE_DIR_NAME:-v${RUNTIME_CACHE_VERSION}}" + local svc_user svc_home + + svc_user=\$(systemctl show "\$SERVICE" --property=User --value 2>/dev/null || true) + if [ -z "\$svc_user" ]; then + svc_user=root + fi + + svc_home=\$(getent passwd "\$svc_user" 2>/dev/null | cut -d: -f6 || true) + if [ -z "\$svc_home" ]; then + svc_home=/root + fi + + printf '%s\n' "\$svc_home/.local/share/boxlite/runtimes/\$version_dir" +} + +install_embedded_runtime_payload() { + local payload="\$1" + local cache_dir tmp_dir guest_hash + + if [ -z "\${GUEST_EXPECTED:-}" ]; then + echo "embedded runtime install skipped: no expected guest hash sidecar" + return 0 + fi + if [ ! -f "\$payload" ]; then + echo "embedded runtime install skipped: no runtime payload in tarball" + return 0 + fi + + cache_dir="\$(primary_runtime_cache_dir)" + tmp_dir="\${cache_dir}.tmp.\$\$" + rm -rf "\$tmp_dir" + mkdir -p "\$tmp_dir" + tar -xzf "\$payload" -C "\$tmp_dir" + + if [ ! -f "\$tmp_dir/boxlite-guest" ]; then + echo "FATAL: embedded runtime payload has no boxlite-guest" >&2 + rm -rf "\$tmp_dir" + return 1 + fi + + guest_hash=\$(sha256sum "\$tmp_dir/boxlite-guest" | awk '{print \$1}') + if [ "\$guest_hash" != "\$GUEST_EXPECTED" ]; then + echo "FATAL: embedded runtime payload guest hash mismatch (expected=\${GUEST_EXPECTED:0:12} actual=\${guest_hash:0:12})" >&2 + rm -rf "\$tmp_dir" + return 1 + fi + + mkdir -p "\$(dirname "\$cache_dir")" + rm -rf "\$cache_dir" + mv "\$tmp_dir" "\$cache_dir" + echo "embedded runtime payload installed: \${guest_hash:0:12} (\$cache_dir)" +} + +write_runtime_dir_override() { + local cache_dir + cache_dir="\$(primary_runtime_cache_dir)" + if [ ! -d "\$cache_dir" ]; then + echo "FATAL: runtime cache directory missing before start: \$cache_dir" >&2 + return 1 + fi + + mkdir -p "/etc/systemd/system/\$SERVICE.service.d" + cat > "/etc/systemd/system/\$SERVICE.service.d/runtime-dir.conf" <&2 + return 1 + fi + echo "embedded runtime guest hash verified: \${guest_hash:0:12} (\$cache_dir)" + done < <(runtime_cache_dirs) + + if [ "\$checked" -eq 0 ]; then + echo "FATAL: embedded runtime cache not found for \$RUNTIME_CACHE_DIR_NAME; refusing rollout before create/start can hit an old guest binary" >&2 + return 1 + fi +} + +verify_hot_adopted_shims() { + local count=0 + if [ ! -s "\$HOT_SNAPSHOT" ]; then + echo "hot rollout: no live detached shims to adopt" + return 0 + fi + + while read -r box_id before_pid before_start; do + [ -n "\$box_id" ] || continue + count=\$((count + 1)) + local pid_file now_pid now_start + pid_file="\$BOXLITE_HOME_DIR/boxes/\$box_id/shim.pid" + now_pid="\$(sed -n '1p' "\$pid_file" 2>/dev/null | tr -dc '0-9')" + now_start="\$(sed -n '2p' "\$pid_file" 2>/dev/null | tr -dc '0-9')" + [ "\$now_pid" = "\$before_pid" ] || { + echo "FATAL: box \$box_id shim pid changed across rollout (before=\$before_pid after=\$now_pid)" >&2 + return 1 + } + pid_alive "\$now_pid" || { + echo "FATAL: box \$box_id shim pid \$now_pid is not alive after rollout" >&2 + return 1 + } + if [ -n "\$before_start" ]; then + [ "\$now_start" = "\$before_start" ] || { + echo "FATAL: box \$box_id shim start-time changed across rollout" >&2 + return 1 + } + [ "\$(proc_start_time "\$now_pid")" = "\$before_start" ] || { + echo "FATAL: box \$box_id shim pid \$now_pid failed start-time verification" >&2 + return 1 + } + fi + + # This forces the new runner process through getOrFetchBox -> runtime.Get -> + # vmm_attach for boxes that only existed in the previous runner's memory. + curl -fsS "\${CURL_TLS[@]}" \ + -H "Authorization: Bearer \$BOXLITE_RUNNER_TOKEN" \ + "\$RUNNER_BASE_URL/v1/boxes/\$box_id/metrics" >/dev/null || { + echo "FATAL: new runner failed to attach/probe detached box \$box_id" >&2 + return 1 + } + echo "hot rollout: adopted \$box_id pid=\$now_pid" + done < "\$HOT_SNAPSHOT" + + echo "hot rollout: adopted \$count detached box(es)" +} + +prepare_release_target() { + local source_binary="\$1" + local release_id="\$2" + local release_dir="\$RELEASES_DIR/\$release_id" + mkdir -p "\$release_dir" + install -m 0755 "\$source_binary" "\$release_dir/boxlite-runner" + sha256sum "\$release_dir/boxlite-runner" | awk '{print \$1}' > "\$release_dir/boxlite-runner.sha256" + printf '%s\n' "\$release_dir/boxlite-runner" +} + +verify_release_target() { + local target="\$1" + local label="\$2" + local expected actual sha_file + + if [ ! -x "\$target" ]; then + echo "FATAL: \$label runner target is not executable: \$target" >&2 + return 1 + fi + + sha_file="\$(dirname "\$target")/boxlite-runner.sha256" + actual=\$(sha256sum "\$target" | awk '{print \$1}') + if [ -f "\$sha_file" ]; then + expected=\$(awk '{print \$1}' "\$sha_file") + if [ "\$expected" != "\$actual" ]; then + echo "FATAL: \$label runner target hash mismatch: \$target (expected=\${expected:0:12} actual=\${actual:0:12})" >&2 + return 1 + fi + else + echo "\$actual" > "\$sha_file" + echo "WARNING: \$label runner target had no hash sidecar; recorded current hash \${actual:0:12}" >&2 + fi + + echo "\$label runner target verified: \${actual:0:12} (\$target)" +} + +current_runner_target() { + if [ -L "\$RUNNER_BIN" ]; then + readlink -f "\$RUNNER_BIN" 2>/dev/null || true + elif [ -x "\$RUNNER_BIN" ]; then + local legacy_dir="\$RELEASES_DIR/legacy-\$(date +%Y%m%d%H%M%S)" + mkdir -p "\$legacy_dir" + cp -a "\$RUNNER_BIN" "\$legacy_dir/boxlite-runner" + sha256sum "\$legacy_dir/boxlite-runner" | awk '{print \$1}' > "\$legacy_dir/boxlite-runner.sha256" + printf '%s\n' "\$legacy_dir/boxlite-runner" + fi +} + +activate_runner_target() { + local target="\$1" + ln -sfn "\$target" "\$RUNNER_BIN" +} + +prune_old_releases() { + [ "\$KEEP_RELEASES" -gt 0 ] 2>/dev/null || return 0 + [ -d "\$RELEASES_DIR" ] || return 0 + find "\$RELEASES_DIR" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' \ + | sort -rn \ + | awk -v keep="\$KEEP_RELEASES" 'NR > keep { sub(/^[^ ]+ /, ""); print }' \ + | while IFS= read -r old_dir; do + rm -rf "\$old_dir" + done +} + +restart_with_target() { + local target="\$1" + systemctl stop "\$SERVICE" || true + activate_runner_target "\$target" || return 1 + write_runtime_dir_override || return 1 + systemctl start "\$SERVICE" || return 1 + sleep 2 + systemctl is-active --quiet "\$SERVICE" || return 1 + wait_runner_ready || return 1 + verify_runner_health_samples || return 1 + verify_embedded_runtime_hash || return 1 + verify_hot_adopted_shims || return 1 +} + +restart_rollback_target() { + local target="\$1" + local saved_guest_expected="\${GUEST_EXPECTED:-}" + local saved_runtime_suffix="\${RUNTIME_SUFFIX:-}" + local saved_runtime_cache_dir_name="\${RUNTIME_CACHE_DIR_NAME:-}" + + GUEST_EXPECTED="" + RUNTIME_SUFFIX="" + RUNTIME_CACHE_DIR_NAME="v${RUNTIME_CACHE_VERSION}" + restart_with_target "\$target" + local rc=\$? + + GUEST_EXPECTED="\$saved_guest_expected" + RUNTIME_SUFFIX="\$saved_runtime_suffix" + RUNTIME_CACHE_DIR_NAME="\$saved_runtime_cache_dir_name" + return "\$rc" +} + +load_runner_env +snapshot_live_detached_shims +echo "hot rollout: captured \$(wc -l < "\$HOT_SNAPSHOT" | tr -d ' ') live detached shim(s)" WORK=\$(mktemp -d) -trap 'rm -rf "\$WORK"' EXIT -curl -fsSL "${ASSET_BASE}/${ASSET_TARBALL}" -o "\$WORK/runner.tar.gz" -if curl -fsSL "${ASSET_BASE}/${ASSET_TARBALL}.sha256" -o "\$WORK/runner.sha256"; then +trap 'rm -rf "\$WORK"; rm -f "\$HOT_SNAPSHOT"' EXIT +curl -fL --show-error --silent \ + --retry 5 --retry-delay 2 --retry-connrefused \ + --connect-timeout "${RUNNER_DOWNLOAD_CONNECT_TIMEOUT_SECONDS}" \ + --max-time "${RUNNER_DOWNLOAD_MAX_TIME_SECONDS}" \ + "${DOWNLOAD_TARBALL_URL}" -o "\$WORK/runner.tar.gz" +if curl -fL --show-error --silent \ + --retry 3 --retry-delay 2 --retry-connrefused \ + --connect-timeout "${RUNNER_DOWNLOAD_CONNECT_TIMEOUT_SECONDS}" \ + --max-time "${RUNNER_CHECKSUM_DOWNLOAD_MAX_TIME_SECONDS}" \ + "${DOWNLOAD_SHA_URL}" -o "\$WORK/runner.sha256"; then EXPECTED=\$(awk '{print \$1}' "\$WORK/runner.sha256") ACTUAL=\$(sha256sum "\$WORK/runner.tar.gz" | awk '{print \$1}') [ "\$EXPECTED" = "\$ACTUAL" ] || { echo "FATAL: checksum mismatch (want \$EXPECTED got \$ACTUAL)" >&2; exit 1; } echo "checksum verified (\$ACTUAL)" else - echo "WARNING: no .sha256 published for v${VERSION}; installing without integrity verification" >&2 + echo "WARNING: no checksum available for runner tarball; installing without integrity verification" >&2 fi tar -xzf "\$WORK/runner.tar.gz" -C "\$WORK" test -x "\$WORK/boxlite-runner" || { echo "FATAL: tarball has no boxlite-runner binary" >&2; exit 1; } +if [ -f "\$WORK/boxlite-runner.guest.sha256" ]; then + GUEST_EXPECTED=\$(awk '{print \$1}' "\$WORK/boxlite-runner.guest.sha256") + echo "runner expected guest hash: \${GUEST_EXPECTED:0:12}" +else + GUEST_EXPECTED="" + echo "WARNING: tarball has no guest hash sidecar; skipping pre-install guest hash verification" >&2 +fi +if [ -f "\$WORK/boxlite-runner.runtime-suffix" ]; then + RUNTIME_SUFFIX=\$(tr -dc 'A-Za-z0-9._-' < "\$WORK/boxlite-runner.runtime-suffix") + if [ -n "\$RUNTIME_SUFFIX" ]; then + RUNTIME_CACHE_DIR_NAME="v${RUNTIME_CACHE_VERSION}-\$RUNTIME_SUFFIX" + echo "runner runtime cache key: \$RUNTIME_CACHE_DIR_NAME" + else + RUNTIME_CACHE_DIR_NAME="v${RUNTIME_CACHE_VERSION}" + echo "WARNING: runtime suffix sidecar is empty; using \$RUNTIME_CACHE_DIR_NAME" >&2 + fi +else + RUNTIME_CACHE_DIR_NAME="v${RUNTIME_CACHE_VERSION}" + echo "runner runtime cache key: \$RUNTIME_CACHE_DIR_NAME" +fi +install_embedded_runtime_payload "\$WORK/boxlite-runtime.tar.gz" || exit 1 -# Back up the live binary (if any) so a failed swap or start can roll back. -HAD_PREVIOUS=false -if [ -x /usr/local/bin/boxlite-runner ]; then - cp -a /usr/local/bin/boxlite-runner /usr/local/bin/boxlite-runner.bak - HAD_PREVIOUS=true +CURRENT_TARGET=\$(current_runner_target) +NEW_TARGET=\$(prepare_release_target "\$WORK/boxlite-runner" "v${VERSION}") +verify_release_target "\$NEW_TARGET" "new" || exit 1 +echo "install target: \$NEW_TARGET" +if [ -n "\$CURRENT_TARGET" ]; then + verify_release_target "\$CURRENT_TARGET" "rollback" || exit 1 + echo "rollback target: \$CURRENT_TARGET" +else + echo "rollback target: none" fi -systemctl stop boxlite-runner || true + # Swap + start + health as one guarded condition: a failing step here (install error, # start failure, or an unhealthy unit) routes to the rollback branch instead of aborting # the script under set -e — commands in an if-condition are exempt from set -e. -if install -m 0755 "\$WORK/boxlite-runner" /usr/local/bin/boxlite-runner && systemctl start boxlite-runner && sleep 2 && systemctl is-active --quiet boxlite-runner; then - [ "\$HAD_PREVIOUS" = true ] && rm -f /usr/local/bin/boxlite-runner.bak +if restart_with_target "\$NEW_TARGET"; then + prune_old_releases echo "systemd unit: active" - echo "new version:" - /usr/local/bin/boxlite-runner --version else - echo "upgrade failed; rolling back" >&2 - if [ "\$HAD_PREVIOUS" = true ]; then - mv -f /usr/local/bin/boxlite-runner.bak /usr/local/bin/boxlite-runner - systemctl restart boxlite-runner || true + echo "upgrade failed or detached-box adoption failed; rolling back" >&2 + if [ -n "\$CURRENT_TARGET" ]; then + if restart_rollback_target "\$CURRENT_TARGET"; then + echo "rollback complete" + else + echo "rollback failed" >&2 + fi fi - journalctl -u boxlite-runner --no-pager -n 50 || true + journalctl -u "\$SERVICE" --no-pager -n 50 || true exit 1 fi EOF @@ -108,10 +676,28 @@ CMD_ID=$(aws ssm send-command --region "$AWS_REGION" \ --query 'Command.CommandId' --output text) echo " command: $CMD_ID" -echo "==> Waiting for SSM command to finish..." +echo "==> Waiting for SSM command to finish (timeout=${SSM_WAIT_TIMEOUT_SECONDS}s)..." + +STATUS="" +DEADLINE=$((SECONDS + SSM_WAIT_TIMEOUT_SECONDS)) +while true; do + STATUS=$(aws ssm get-command-invocation --region "$AWS_REGION" \ + --command-id "$CMD_ID" --instance-id "$INSTANCE_ID" \ + --query 'Status' --output text 2>/dev/null || true) + + case "$STATUS" in + Success|Failed|Cancelled|TimedOut|Cancelling) + break + ;; + esac + + if (( SECONDS >= DEADLINE )); then + echo "error: SSM command still ${STATUS:-unknown} after ${SSM_WAIT_TIMEOUT_SECONDS}s" >&2 + exit 1 + fi -aws ssm wait command-executed --region "$AWS_REGION" \ - --command-id "$CMD_ID" --instance-id "$INSTANCE_ID" + sleep "$SSM_WAIT_POLL_SECONDS" +done STATUS=$(aws ssm get-command-invocation --region "$AWS_REGION" \ --command-id "$CMD_ID" --instance-id "$INSTANCE_ID" \