diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 6ae897f96..3ec60e4ff 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -3,20 +3,23 @@ FROM node:24-slim AS base ENV CI=true -RUN apt-get update && apt-get install -y --no-install-recommends bash curl && \ +RUN apt-get update && apt-get install -y --no-install-recommends bash curl python3 make g++ && \ rm -rf /var/lib/apt/lists/* RUN npm install -g corepack && corepack enable +# Yarn global cache persisted across builds via a BuildKit cache mount (both installs below) +ENV YARN_ENABLE_GLOBAL_CACHE=1 + WORKDIR /boxlite/apps FROM base AS deps COPY apps/package.json apps/yarn.lock apps/.yarnrc.yml ./ -RUN yarn install --immutable +RUN --mount=type=cache,target=/root/.yarn/berry/cache yarn install --immutable FROM base AS runtime-deps ENV NODE_ENV=production COPY apps/package.json apps/yarn.lock apps/.yarnrc.yml ./ -RUN yarn workspaces focus --all --production +RUN --mount=type=cache,target=/root/.yarn/berry/cache yarn workspaces focus --all --production FROM deps AS build COPY apps/nx.json apps/tsconfig.base.json apps/NOTICE ./ diff --git a/apps/infra/scripts/sst-with-cloudflare.mjs b/apps/infra/scripts/sst-with-cloudflare.mjs index 64c7e0001..0544a5ec9 100644 --- a/apps/infra/scripts/sst-with-cloudflare.mjs +++ b/apps/infra/scripts/sst-with-cloudflare.mjs @@ -8,7 +8,9 @@ * `run()` exists — so its credentials can't be `sst.Secret` like the app * secrets. They live in AWS SSM Parameter Store (SecureString) instead, keyed * per stage, and this wrapper fetches + exports them just before invoking sst. - * App secrets are NOT handled here; sst resolves those from its own store. + * App secrets are normally resolved by SST from its own store. For self-hosted + * deploys, set BOXLITE_ENV_SOURCE=ssm to sync /boxlite//env/* into this + * app's .env and load the subset that SST models as sst.Secret. * * Wired into the dev/deploy/remove npm scripts, plus a passthrough: * npm run deploy -- --stage dev → node sst-with-cloudflare.mjs deploy --stage dev @@ -18,9 +20,9 @@ * same credentials fetch the Cloudflare token from SSM — nothing extra to share. * * Seed the parameters once per stage: - * aws ssm put-parameter --region ap-southeast-1 --type SecureString \ + * aws ssm put-parameter --region --type SecureString \ * --name /boxlite//cloudflare-api-token --value "" - * aws ssm put-parameter --region ap-southeast-1 --type SecureString \ + * aws ssm put-parameter --region --type SecureString \ * --name /boxlite//cloudflare-account-id --value "" * * A credential already in the environment is used as-is (works offline / before @@ -30,8 +32,16 @@ */ import { execFileSync, spawnSync } from 'node:child_process' +import { chmodSync, mkdirSync, mkdtempSync, renameSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' -const REGION = process.env.AWS_REGION || 'ap-southeast-1' +const DEFAULT_REGION = 'ap-southeast-1' +const STAGE_REGIONS = { + dev: 'ap-southeast-1', + prod: 'us-west-2', + production: 'us-west-2', +} // SSM param consulted only when the matching env var is unset. const CREDS = [ @@ -39,6 +49,18 @@ const CREDS = [ { env: 'CLOUDFLARE_DEFAULT_ACCOUNT_ID', param: 'cloudflare-account-id' }, ] +const CONFIG_COMMANDS = new Set(['deploy', 'dev', 'diff', 'remove']) +const DEFAULT_PRODUCT_ENV_PREFIX_TEMPLATE = '/boxlite/{stage}/env' +const DEFAULT_SST_SECRET_KEYS = [ + 'OIDC_CLIENT_ID', + 'OIDC_MANAGEMENT_API_CLIENT_ID', + 'OIDC_MANAGEMENT_API_CLIENT_SECRET', + 'POSTHOG_API_KEY', + 'SVIX_AUTH_TOKEN', + 'SSH_PRIVATE_KEY_B64', + 'SSH_HOST_KEY_B64', +] + const sstArgs = process.argv.slice(2) if (sstArgs.length === 0) { console.error('sst-with-cloudflare: expected an sst subcommand (e.g. "deploy --stage dev")') @@ -56,12 +78,20 @@ function resolveStage(args) { return process.env.SST_STAGE || 'dev' } +function runAws(args, options = {}) { + return execFileSync('aws', args, { + encoding: 'utf8', + env: { ...process.env, AWS_REGION: REGION }, + maxBuffer: 10 * 1024 * 1024, + ...options, + }) +} + function fetchFromSsm(name) { try { - const out = execFileSync( - 'aws', + const out = runAws( ['ssm', 'get-parameter', '--region', REGION, '--name', name, '--with-decryption', '--query', 'Parameter.Value', '--output', 'text'], - { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }, + { stdio: ['ignore', 'pipe', 'pipe'] }, ).trim() return out && out !== 'None' ? out : null } catch (err) { @@ -70,7 +100,115 @@ function fetchFromSsm(name) { } } +function dotenvLine(key, value) { + return `${key}=${JSON.stringify(value)}` +} + +function writeAtomically(file, body) { + mkdirSync(dirname(file), { recursive: true }) + const tmp = `${file}.tmp-${process.pid}` + writeFileSync(tmp, body, { mode: 0o600 }) + renameSync(tmp, file) + chmodSync(file, 0o600) +} + +function envPrefixForStage() { + const explicit = process.env.BOXLITE_ENV_SSM_PREFIX || process.env.OPS_PRODUCT_ENV_SSM_PREFIX + const template = + process.env.BOXLITE_ENV_SSM_PREFIX_TEMPLATE || + process.env.OPS_PRODUCT_ENV_SSM_PREFIX_TEMPLATE || + DEFAULT_PRODUCT_ENV_PREFIX_TEMPLATE + const source = explicit || template + return source.replaceAll('{stage}', stage) +} + +function sstSecretKeys() { + return new Set( + (process.env.BOXLITE_SST_SECRET_KEYS || process.env.OPS_PRODUCT_SST_SECRET_KEYS || DEFAULT_SST_SECRET_KEYS.join(',')) + .split(',') + .map((key) => key.trim()) + .filter(Boolean), + ) +} + +function syncProductEnvFromSsm() { + const source = process.env.BOXLITE_ENV_SOURCE || process.env.SST_ENV_SOURCE || 'local' + if (source !== 'ssm' || !CONFIG_COMMANDS.has(sstArgs[0])) return + + const prefix = envPrefixForStage() + const outFile = process.env.BOXLITE_ENV_FILE || process.env.OPS_PRODUCT_ENV_FILE || join(process.cwd(), '.env') + let params + try { + const stdout = runAws([ + 'ssm', + 'get-parameters-by-path', + '--region', + REGION, + '--path', + prefix, + '--recursive', + '--with-decryption', + '--query', + 'Parameters[].{Name:Name,Value:Value,Version:Version}', + '--output', + 'json', + ]) + params = JSON.parse(stdout || '[]') + } catch (err) { + const detail = err.stderr ? String(err.stderr).trim() : err.message + console.error(`sst-env-sync: failed to read ${prefix} from SSM (${REGION}): ${detail}`) + process.exit(1) + } + + const records = params + .map((param) => ({ + key: String(param.Name || '').split('/').filter(Boolean).pop(), + value: String(param.Value ?? ''), + version: Number(param.Version || 0), + })) + .filter((param) => param.key) + .sort((a, b) => a.key.localeCompare(b.key)) + + if (records.length === 0) { + console.error(`sst-env-sync: no parameters found under ${prefix} in ${REGION}; seed SSM before running SST with BOXLITE_ENV_SOURCE=ssm`) + process.exit(1) + } + + writeAtomically(outFile, `${records.map((param) => dotenvLine(param.key, param.value)).join('\n')}\n`) + + const secretKeys = sstSecretKeys() + const secretLines = records + .filter((param) => secretKeys.has(param.key) && param.value) + .map((param) => dotenvLine(param.key, param.value)) + + if (secretLines.length) { + const tempDir = mkdtempSync(join(tmpdir(), 'boxlite-sst-secrets-')) + const secretFile = join(tempDir, '.sst-secrets.env') + try { + writeFileSync(secretFile, `${secretLines.join('\n')}\n`, { mode: 0o600 }) + console.log(`sst-env-sync: wrote ${outFile} (${records.length} key(s) from ${prefix}); loading ${secretLines.length} SST secret(s)`) + execFileSync('sst', ['secret', 'load', secretFile, '--stage', stage], { stdio: 'inherit', env: process.env }) + } finally { + rmSync(tempDir, { recursive: true, force: true }) + } + } else { + console.log(`sst-env-sync: wrote ${outFile} (${records.length} key(s) from ${prefix}); no SST secrets matched`) + } + + console.log( + `::ops-audit::${JSON.stringify({ + phase: 'env-sync', + stage, + region: REGION, + prefix, + keyVersions: records.map((param) => ({ key: param.key, version: param.version })), + sstSecretKeys: records.filter((param) => secretKeys.has(param.key) && param.value).map((param) => param.key), + })}`, + ) +} + const stage = resolveStage(sstArgs) +const REGION = process.env.BOXLITE_AWS_REGION || process.env.AWS_REGION || STAGE_REGIONS[stage] || DEFAULT_REGION for (const { env, param } of CREDS) { if (process.env[env]) continue // already provided — don't touch @@ -86,6 +224,8 @@ for (const { env, param } of CREDS) { } } +syncProductEnvFromSsm() + // node_modules/.bin is on PATH because this runs via `npm run`, so `sst` resolves. const result = spawnSync('sst', sstArgs, { stdio: 'inherit', env: process.env }) if (result.error) { diff --git a/apps/infra/sst.config.ts b/apps/infra/sst.config.ts index 179a21a9b..60470e3de 100644 --- a/apps/infra/sst.config.ts +++ b/apps/infra/sst.config.ts @@ -6,7 +6,9 @@ /// // ───────────────────────────────────────────────────────────────────────────── -// BoxLite control plane on AWS (ap-southeast-1). +// BoxLite control plane on AWS. Region is stage-scoped: dev stays in +// ap-southeast-1, prod/production stays in us-west-2 unless explicitly +// overridden by AWS_REGION/BOXLITE_AWS_REGION. // // Top of file: constants + helpers + the runner user-data builder. // Inside `run()`, resources are created in deploy order: @@ -18,7 +20,16 @@ // 5. observability 10. runner (EC2 + nested KVM) // ───────────────────────────────────────────────────────────────────────────── -const REGION = 'ap-southeast-1' +const DEFAULT_REGION = 'ap-southeast-1' +const STAGE_REGIONS: Record = { + dev: 'ap-southeast-1', + prod: 'us-west-2', + production: 'us-west-2', +} +const regionForStage = (stage = 'dev') => process.env.BOXLITE_AWS_REGION || process.env.AWS_REGION || STAGE_REGIONS[stage] || DEFAULT_REGION +const isProductionStage = (stage = '') => stage === 'prod' || stage === 'production' +const ACCOUNT_ID = '064212132677' +const IAM_ROLE_BOUNDARY_ARN = `arn:aws:iam::${ACCOUNT_ID}:policy/boxlite-role-boundary` // Container ports each service listens on internally const PORTS = { @@ -92,12 +103,13 @@ const runnerEndpoint = (override: string, port: number, scheme: string) => // ── app config ─────────────────────────────────────────────────────────────── export default $config({ app(input) { + const region = regionForStage(input?.stage) return { name: 'boxlite', - removal: input?.stage === 'production' ? 'retain' : 'remove', + removal: isProductionStage(input?.stage) ? 'retain' : 'remove', home: 'aws', providers: { - aws: { region: REGION, ...(process.env.AWS_PROFILE ? { profile: process.env.AWS_PROFILE } : {}) }, + aws: { region, ...(process.env.AWS_PROFILE ? { profile: process.env.AWS_PROFILE } : {}) }, cloudflare: '6.15.0', random: '4.16.6', // command provider: multi-runner post-deploy registration @@ -111,6 +123,15 @@ export default $config({ // Load .env overrides (anything unset falls back to auto-generated values) const { config } = await import('dotenv') config() + const region = regionForStage($app.stage) + + // BoxLite AWS guardrail: every managed IAM role must keep the account's + // permissions boundary. If this is omitted, Pulumi treats the boundary as + // drift and tries DeleteRolePermissionsBoundary, which the account policy + // explicitly denies. + $transform(aws.iam.Role, (args) => { + args.permissionsBoundary ??= IAM_ROLE_BOUNDARY_ARN + }) // Strip trailing slash from service.url so path concat produces clean URLs // (api.url = "https://api.dev.boxlite.ai/" → apiBase = "https://api.dev.boxlite.ai"). @@ -213,7 +234,11 @@ export default $config({ // S3 versioning is on in every stage: cheap, and the only guard against an // object-level overwrite/delete (which `removal` never covers). Redis is a // transient cache, so it needs neither. - const isProd = $app.stage === 'production' + const isProd = isProductionStage($app.stage) + const serviceImageCache = envOr( + 'SERVICE_IMAGE_CACHE', + $app.stage === 'prod' || $app.stage === 'production' ? 'false' : 'true', + ) === 'true' // Unique-but-stable suffix for the DB final snapshot: a fixed name would collide // with the snapshot a prior teardown of the same stage already created (RDS requires // unique final-snapshot ids). RandomId is stable across deploys (no drift) and is @@ -258,7 +283,7 @@ export default $config({ // removes the single largest by-volume consumer of fck-nat egress. new aws.ec2.VpcEndpoint('S3Gateway', { vpcId: vpc.nodes.vpc.id, - serviceName: `com.amazonaws.${REGION}.s3`, + serviceName: `com.amazonaws.${region}.s3`, vpcEndpointType: 'Gateway', routeTableIds: vpc.nodes.privateRouteTables.apply((tables) => tables.map((t) => t.id)), }) @@ -323,7 +348,7 @@ export default $config({ const otelCollector = new sst.aws.Service('OtelCollector', { cluster, - image: { context: '../..', dockerfile: 'apps/otel-collector/Dockerfile', cache: false }, + image: { context: '../..', dockerfile: 'apps/otel-collector/Dockerfile', cache: serviceImageCache }, command: [ '--config', '/otelcol/collector-config.yaml', @@ -374,6 +399,7 @@ export default $config({ image: { context: '../..', dockerfile: 'apps/api/Dockerfile', + cache: serviceImageCache, }, loadBalancer: { domain: serviceDomain('api'), @@ -407,7 +433,7 @@ export default $config({ // (ADMIN_OBSERVABILITY_CLOUDWATCH_REGION). actions: ['logs:DescribeLogGroups'], resources: [ - $interpolate`arn:aws:logs:${REGION}:${aws.getCallerIdentityOutput().accountId}:log-group:*`, + $interpolate`arn:aws:logs:${region}:${aws.getCallerIdentityOutput().accountId}:log-group:*`, ], }, { @@ -419,8 +445,8 @@ export default $config({ { actions: ['logs:FilterLogEvents'], resources: [ - $interpolate`arn:aws:logs:${REGION}:${aws.getCallerIdentityOutput().accountId}:log-group:/sst/cluster/${cluster.nodes.cluster.name}/*`, - $interpolate`arn:aws:logs:${REGION}:${aws.getCallerIdentityOutput().accountId}:log-group:/sst/cluster/${cluster.nodes.cluster.name}/*:*`, + $interpolate`arn:aws:logs:${region}:${aws.getCallerIdentityOutput().accountId}:log-group:/sst/cluster/${cluster.nodes.cluster.name}/*`, + $interpolate`arn:aws:logs:${region}:${aws.getCallerIdentityOutput().accountId}:log-group:/sst/cluster/${cluster.nodes.cluster.name}/*:*`, ], }, { @@ -536,7 +562,7 @@ export default $config({ // supported only for S3-compatible deployments (MinIO). S3_ENDPOINT: $interpolate`https://s3.${aws.getRegionOutput().name}.amazonaws.com`, S3_STS_ENDPOINT: $interpolate`https://sts.${aws.getRegionOutput().name}.amazonaws.com`, - S3_REGION: REGION, + S3_REGION: region, S3_DEFAULT_BUCKET: storage.name, S3_ACCOUNT_ID: aws.getCallerIdentityOutput().accountId, S3_ROLE_NAME: s3AccessRoleName, @@ -583,7 +609,7 @@ export default $config({ 'BOX_OTEL_ENDPOINT_URL', envOr('OTEL_EXPORTER_OTLP_ENDPOINT', otelCollectorOtlpHttpUrl), ), - ADMIN_OBSERVABILITY_CLOUDWATCH_REGION: envOr('ADMIN_OBSERVABILITY_CLOUDWATCH_REGION', REGION), + ADMIN_OBSERVABILITY_CLOUDWATCH_REGION: envOr('ADMIN_OBSERVABILITY_CLOUDWATCH_REGION', region), ADMIN_OBSERVABILITY_CLOUDWATCH_LOG_GROUPS: envOr('ADMIN_OBSERVABILITY_CLOUDWATCH_LOG_GROUPS', ''), ADMIN_OBSERVABILITY_CLOUDWATCH_LOG_GROUP_PREFIX: envOr( 'ADMIN_OBSERVABILITY_CLOUDWATCH_LOG_GROUP_PREFIX', @@ -591,7 +617,7 @@ export default $config({ ), ADMIN_OBSERVABILITY_CLOUDWATCH_LIMIT_PER_GROUP: envOr('ADMIN_OBSERVABILITY_CLOUDWATCH_LIMIT_PER_GROUP', '25'), ADMIN_OBSERVABILITY_CLOUDWATCH_MAX_LOG_GROUPS: envOr('ADMIN_OBSERVABILITY_CLOUDWATCH_MAX_LOG_GROUPS', '20'), - ADMIN_OBSERVABILITY_S3_REGION: envOr('ADMIN_OBSERVABILITY_S3_REGION', REGION), + ADMIN_OBSERVABILITY_S3_REGION: envOr('ADMIN_OBSERVABILITY_S3_REGION', region), ADMIN_OBSERVABILITY_S3_BUCKETS: envOr('ADMIN_OBSERVABILITY_S3_BUCKETS', storage.name), ADMIN_OBSERVABILITY_S3_MAX_OBJECTS: envOr('ADMIN_OBSERVABILITY_S3_MAX_OBJECTS', '25'), ...(process.env.ADMIN_OBSERVABILITY_CLICKSTACK_URL && { @@ -671,7 +697,7 @@ export default $config({ const proxyDomain = `proxy.${stackDomain}` new sst.aws.Service('Proxy', { cluster, - image: { context: '../..', dockerfile: 'apps/proxy/Dockerfile', cache: false }, + image: { context: '../..', dockerfile: 'apps/proxy/Dockerfile', cache: serviceImageCache }, loadBalancer: { domain: { name: proxyDomain, @@ -707,7 +733,7 @@ export default $config({ // get a stable, memorable hostname instead of the auto-generated NLB DNS name. const sshGateway = new sst.aws.Service('SshGateway', { cluster, - image: { context: '../..', dockerfile: 'apps/ssh-gateway/Dockerfile', cache: false }, + image: { context: '../..', dockerfile: 'apps/ssh-gateway/Dockerfile', cache: serviceImageCache }, loadBalancer: { rules: [{ listen: `${PORTS.SSH_GATEWAY}/tcp`, forward: `${PORTS.SSH_GATEWAY}/tcp` }] }, environment: { // api-client-go composes paths like "/box/ssh-access/validate" directly. @@ -906,7 +932,7 @@ export default $config({ otelCollectorOtlpHttpUrl, ghcrSecret ? ghcrSecret.arn : '', ]).apply(([apiUrl, token, otelEndpoint, ghcrSecretArn]) => - buildRunnerUserData({ apiUrl, token, otelEndpoint, ghcrSecretArn: ghcrSecretArn || undefined, ghcrUsername }), + buildRunnerUserData({ apiUrl, token, otelEndpoint, ghcrSecretArn: ghcrSecretArn || undefined, ghcrUsername, region }), ) // Runners hold load-bearing box state (/var/lib/boxlite + in-memory libkrun VMs). @@ -977,7 +1003,7 @@ export default $config({ `boxlite-runner-${index}`, $resolve([api.url, apiKey.result, otelCollectorOtlpHttpUrl, ghcrSecret ? ghcrSecret.arn : '']).apply( ([apiUrl, token, otelEndpoint, ghcrSecretArn]) => - buildRunnerUserData({ apiUrl, token, otelEndpoint, ghcrSecretArn: ghcrSecretArn || undefined, ghcrUsername }), + buildRunnerUserData({ apiUrl, token, otelEndpoint, ghcrSecretArn: ghcrSecretArn || undefined, ghcrUsername, region }), ), ) return { name, apiKey, instance } @@ -1018,6 +1044,7 @@ async function buildRunnerUserData(input: { otelEndpoint: string ghcrSecretArn?: string ghcrUsername?: string + region: string }): Promise { const { readFileSync } = await import('fs') const { resolve } = await import('path') @@ -1133,7 +1160,7 @@ Environment=API_VERSION=2 Environment=API_PORT=${PORTS.RUNNER} Environment=RUNNER_DOMAIN=\$HOST_IP Environment=BOXLITE_HOME_DIR=/var/lib/boxlite -Environment=AWS_REGION=${REGION} +Environment=AWS_REGION=${input.region} Environment=OTEL_LOGGING_ENABLED=true Environment=OTEL_TRACING_ENABLED=true Environment=OTEL_EXPORTER_OTLP_ENDPOINT=${input.otelEndpoint}${input.ghcrSecretArn ? ` diff --git a/apps/otel-collector/Dockerfile b/apps/otel-collector/Dockerfile index 4a82ea386..378eef7db 100644 --- a/apps/otel-collector/Dockerfile +++ b/apps/otel-collector/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 FROM node:22-alpine AS build ENV CI=true @@ -8,13 +9,14 @@ COPY --from=golang:1.25.4-alpine /usr/local/go/ /usr/local/go/ ENV PATH="/usr/local/go/bin:${PATH}" -RUN apk add --no-cache git +RUN apk add --no-cache git build-base python3 WORKDIR /boxlite/apps -# Yarn caching layer +# Yarn caching layer — global cache persisted across builds via a BuildKit cache mount COPY apps/package.json apps/yarn.lock apps/.yarnrc.yml ./ -RUN yarn install --immutable +ENV YARN_ENABLE_GLOBAL_CACHE=1 +RUN --mount=type=cache,target=/root/.yarn/berry/cache yarn install --immutable # Nx config COPY apps/nx.json ./ @@ -29,7 +31,8 @@ RUN head -1 go.work > go.work.tmp && printf '\nuse (\n\t./otel-collector/exporte ENV NX_DAEMON=false ENV GOWORK=/boxlite/apps/go.work -RUN go -C otel-collector/exporter mod download && \ +RUN --mount=type=cache,target=/root/go/pkg/mod \ + go -C otel-collector/exporter mod download && \ go -C api-client-go mod download && \ go -C common-go mod download @@ -39,7 +42,8 @@ COPY apps/api-client-go/ api-client-go/ COPY apps/common-go/ common-go/ RUN mkdir -p dist/apps/otel-collector -RUN yarn nx build otel-collector --configuration=production --nxBail=true +RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/go/pkg/mod \ + yarn nx build otel-collector --configuration=production --nxBail=true FROM alpine:3.18 AS otel-collector diff --git a/apps/proxy/Dockerfile b/apps/proxy/Dockerfile index 5838140b5..fd64bcd51 100644 --- a/apps/proxy/Dockerfile +++ b/apps/proxy/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 FROM node:22-alpine AS build ENV CI=true @@ -8,13 +9,14 @@ COPY --from=golang:1.25.4-alpine /usr/local/go/ /usr/local/go/ ENV PATH="/usr/local/go/bin:${PATH}" -RUN apk add --no-cache git +RUN apk add --no-cache git build-base python3 WORKDIR /boxlite/apps -# Yarn caching layer +# Yarn caching layer — global cache persisted across builds via a BuildKit cache mount COPY apps/package.json apps/yarn.lock apps/.yarnrc.yml ./ -RUN yarn install --immutable +ENV YARN_ENABLE_GLOBAL_CACHE=1 +RUN --mount=type=cache,target=/root/.yarn/berry/cache yarn install --immutable # Nx config COPY apps/nx.json ./ @@ -29,7 +31,8 @@ RUN head -1 go.work > go.work.tmp && printf '\nuse (\n\t./proxy\n\t./common-go\n ENV NX_DAEMON=false ENV GOWORK=/boxlite/apps/go.work -RUN go -C proxy mod download && go -C common-go mod download && go -C api-client-go mod download +RUN --mount=type=cache,target=/root/go/pkg/mod \ + go -C proxy mod download && go -C common-go mod download && go -C api-client-go mod download # Go source COPY apps/proxy/ proxy/ @@ -37,7 +40,7 @@ COPY apps/common-go/ common-go/ COPY apps/api-client-go/ api-client-go/ ARG VERSION=0.0.1 -RUN --mount=type=cache,target=/root/.cache/go-build \ +RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/go/pkg/mod \ VERSION=$VERSION yarn nx build proxy --configuration=production --nxBail=true FROM alpine:3.18 AS proxy diff --git a/apps/runner/Dockerfile b/apps/runner/Dockerfile index 51ac83831..3f8fd1199 100644 --- a/apps/runner/Dockerfile +++ b/apps/runner/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 FROM node:22-alpine AS build ENV CI=true @@ -12,9 +13,10 @@ RUN apk add --no-cache git build-base WORKDIR /boxlite -# Yarn caching layer +# Yarn caching layer — global cache persisted across builds via a BuildKit cache mount COPY apps/package.json apps/yarn.lock apps/.yarnrc.yml ./ -RUN yarn install --immutable +ENV YARN_ENABLE_GLOBAL_CACHE=1 +RUN --mount=type=cache,target=/root/.yarn/berry/cache yarn install --immutable # Nx config COPY apps/nx.json ./ @@ -31,7 +33,8 @@ RUN head -1 apps/go.work > apps/go.work.tmp && printf '\nuse (\n\t./runner\n\t./ ENV NX_DAEMON=false -RUN go -C apps/runner mod download \ +RUN --mount=type=cache,target=/root/go/pkg/mod \ + go -C apps/runner mod download \ && go -C apps/common-go mod download \ && go -C apps/api-client-go mod download @@ -49,7 +52,7 @@ COPY apps/common-go/ apps/common-go/ COPY apps/api-client-go/ apps/api-client-go/ ARG VERSION=0.0.1 -RUN --mount=type=cache,target=/root/.cache/go-build \ +RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/go/pkg/mod \ CGO_ENABLED=1 VERSION=$VERSION yarn nx build runner --configuration=production --nxBail=true # Runtime — lightweight Linux with KVM support diff --git a/apps/ssh-gateway/Dockerfile b/apps/ssh-gateway/Dockerfile index 76b21d628..465277e0b 100644 --- a/apps/ssh-gateway/Dockerfile +++ b/apps/ssh-gateway/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 FROM node:22-alpine AS build ENV CI=true @@ -8,13 +9,14 @@ COPY --from=golang:1.25.4-alpine /usr/local/go/ /usr/local/go/ ENV PATH="/usr/local/go/bin:${PATH}" -RUN apk add --no-cache git +RUN apk add --no-cache git build-base python3 WORKDIR /boxlite/apps -# Yarn caching layer +# Yarn caching layer — global cache persisted across builds via a BuildKit cache mount COPY apps/package.json apps/yarn.lock apps/.yarnrc.yml ./ -RUN yarn install --immutable +ENV YARN_ENABLE_GLOBAL_CACHE=1 +RUN --mount=type=cache,target=/root/.yarn/berry/cache yarn install --immutable # Nx config COPY apps/nx.json ./ @@ -28,13 +30,15 @@ RUN head -1 go.work > go.work.tmp && printf '\nuse (\n\t./ssh-gateway\n\t./api-c ENV NX_DAEMON=false ENV GOWORK=/boxlite/apps/go.work -RUN go -C ssh-gateway mod download && go -C api-client-go mod download +RUN --mount=type=cache,target=/root/go/pkg/mod \ + go -C ssh-gateway mod download && go -C api-client-go mod download # Go source COPY apps/ssh-gateway/ ssh-gateway/ COPY apps/api-client-go/ api-client-go/ -RUN yarn nx build ssh-gateway --configuration=production --nxBail=true +RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/go/pkg/mod \ + yarn nx build ssh-gateway --configuration=production --nxBail=true FROM alpine:3.18 AS ssh-gateway diff --git a/scripts/deploy/build-runner-binary.mjs b/scripts/deploy/build-runner-binary.mjs new file mode 100755 index 000000000..eca94c2f9 --- /dev/null +++ b/scripts/deploy/build-runner-binary.mjs @@ -0,0 +1,264 @@ +#!/usr/bin/env node +// Build the Linux amd64 boxlite-runner tarball without GitHub Actions. +// +// JS twin of build-runner-binary.sh — same CLI contract, same inputs, same +// artifact layout. The bash script remains the authority; behavioural parity +// is enforced by scripts/deploy/tests/parity/run.sh. Pick the implementation +// at the orchestration layer (ops-console OPS_RUNNER_IMPL); do not let the +// two drift apart — port fixes to both or flip the default deliberately. +// +// Usage: +// scripts/deploy/build-runner-binary.mjs +// scripts/deploy/build-runner-binary.mjs --output-dir /tmp +// BOXLITE_RUNNER_BUILD_NUMBER=123 scripts/deploy/build-runner-binary.mjs +// scripts/deploy/build-runner-binary.mjs --skip-setup +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import process from 'node:process' +import { fileURLToPath } from 'node:url' + +const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..') + +const USAGE = `Build the Linux amd64 boxlite-runner tarball without GitHub Actions. + +Options: + --output-dir DIR Directory for the runner tarball. Default: ./dist + --build-number ID Ordered dev build sequence. Default: BOXLITE_RUNNER_BUILD_NUMBER, + GITHUB_RUN_NUMBER, or current UTC timestamp. + --skip-setup Skip \`make setup:build\`. + -h, --help Show this help. +` + +function die(msg) { + process.stderr.write(`error: ${msg}\n`) + process.exit(1) +} + +function run(cmd, args, opts = {}) { + // stdio inherit keeps build output streaming to the job log like bash does. + execFileSync(cmd, args, { cwd: ROOT_DIR, stdio: 'inherit', ...opts }) +} + +function runCapture(cmd, args, opts = {}) { + return execFileSync(cmd, args, { cwd: ROOT_DIR, encoding: 'utf8', ...opts }).trim() +} + +function sha256File(file) { + return createHash('sha256').update(fs.readFileSync(file)).digest('hex') +} + +// ── argv ───────────────────────────────────────────────────────────────────── +let outputDir = path.join(ROOT_DIR, 'dist') +let runSetup = true +let buildSequence = process.env.BOXLITE_RUNNER_BUILD_NUMBER || process.env.GITHUB_RUN_NUMBER || '' + +const argv = process.argv.slice(2) +for (let i = 0; i < argv.length; i++) { + const arg = argv[i] + switch (arg) { + case '--output-dir': + if (i + 1 >= argv.length) die('--output-dir requires a value') + outputDir = argv[++i] + break + case '--build-number': + if (i + 1 >= argv.length) die('--build-number requires a value') + buildSequence = argv[++i] + break + case '--skip-setup': + runSetup = false + break + case '-h': + case '--help': + process.stdout.write(USAGE) + process.exit(0) + break + default: + process.stderr.write(`error: unknown argument ${arg}\n${USAGE}`) + process.exit(1) + } +} + +process.chdir(ROOT_DIR) + +// Mirror bash `command -v`: a PATH lookup, not an execution — the parity +// harness compares issued commands, and probes must not appear in that log. +function commandOnPath(cmd) { + for (const dir of (process.env.PATH || '').split(path.delimiter)) { + if (!dir) continue + try { + fs.accessSync(path.join(dir, cmd), fs.constants.X_OK) + return true + } catch { + /* keep looking */ + } + } + return false +} +for (const cmd of ['cargo', 'go', 'make', 'sha256sum', 'tar']) { + if (!commandOnPath(cmd)) die(`required command not found: ${cmd}`) +} + +function findEmbeddedRuntimeDir(guestSha256) { + const buildRoot = path.join(ROOT_DIR, 'target', 'release', 'build') + const candidates = [] + let crates = [] + try { + crates = fs.readdirSync(buildRoot) + } catch { + crates = [] + } + for (const crate of crates.sort()) { + if (!crate.startsWith('boxlite-')) continue + const runtimeDir = path.join(buildRoot, crate, 'out', 'runtime') + let stat + try { + stat = fs.statSync(runtimeDir) + } catch { + continue + } + if (stat.isDirectory()) candidates.push(runtimeDir) + } + for (const runtimeDir of candidates.sort()) { + const guest = path.join(runtimeDir, 'boxlite-guest') + if (!fs.existsSync(guest)) continue + if (sha256File(guest) === guestSha256) return runtimeDir + } + process.stderr.write(`error: embedded runtime payload not found for guest hash ${guestSha256.slice(0, 12)}\n`) + process.exit(1) +} + +// Respect PATH shims / remote environments the same way bash does: ask uname, +// not the Node runtime, so both implementations see the same world. +const unameS = runCapture('uname', ['-s']) +const unameM = runCapture('uname', ['-m']) +if (unameS !== 'Linux' || unameM !== 'x86_64') { + die('runner tarball build currently requires a Linux x86_64 builder') +} + +const cargoToml = fs.readFileSync(path.join(ROOT_DIR, 'Cargo.toml'), 'utf8') +const versionMatch = cargoToml.match(/^version *= *"([^"]+)"/m) +const VERSION = versionMatch ? versionMatch[1] : '' +if (!VERSION) die('could not read version from Cargo.toml') + +if (!buildSequence) { + buildSequence = new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14) +} +buildSequence = buildSequence.replace(/[^A-Za-z0-9._-]/g, '-').replace(/^v/, '') +if (!buildSequence) die('build sequence cannot be empty') + +const TMP_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'tmp.')) +const goWorkPath = path.join(ROOT_DIR, 'apps', 'go.work') +const goWorkSumPath = path.join(ROOT_DIR, 'apps', 'go.work.sum') +const goWorkBackup = fs.existsSync(goWorkPath) ? fs.readFileSync(goWorkPath) : null +const goWorkSumBackup = fs.existsSync(goWorkSumPath) ? fs.readFileSync(goWorkSumPath) : null + +function restoreGoWork() { + if (goWorkBackup !== null) fs.writeFileSync(goWorkPath, goWorkBackup) + else fs.rmSync(goWorkPath, { force: true }) + if (goWorkSumBackup !== null) fs.writeFileSync(goWorkSumPath, goWorkSumBackup) + else fs.rmSync(goWorkSumPath, { force: true }) + fs.rmSync(TMP_DIR, { recursive: true, force: true }) +} + +process.on('exit', restoreGoWork) +for (const sig of ['SIGINT', 'SIGTERM']) { + process.on(sig, () => process.exit(1)) +} + +try { + const goMod = fs.readFileSync(path.join(ROOT_DIR, 'apps', 'runner', 'go.mod'), 'utf8') + const goVersionMatch = goMod.match(/^go (\S+)/m) + const GO_VERSION = goVersionMatch ? goVersionMatch[1] : '' + if (!GO_VERSION) die('could not read Go version from apps/runner/go.mod') + + fs.writeFileSync( + goWorkPath, + `go ${GO_VERSION}\n\nuse (\n\t./runner\n\t./common-go\n\t./api-client-go\n\t../sdks/go\n)\n`, + ) + + const headSha = runCapture('git', ['rev-parse', '--short', 'HEAD']) + console.log(`==> Building boxlite-runner from package v${VERSION} at ${headSha}`) + console.log(`==> Non-release artifact version prefix: v${VERSION}-dev-${buildSequence}`) + + console.log('==> Cleaning runner build artifacts') + run('cargo', ['clean', '-p', 'boxlite', '-p', 'boxlite-c', '-p', 'boxlite-shim', '-p', 'boxlite-guest']) + for (const rel of [ + 'sdks/go/libboxlite.a', + 'target/release/libboxlite.a', + 'target/release/libboxlite.so', + 'target/release/boxlite-shim', + 'target/x86_64-unknown-linux-gnu/release/boxlite-shim', + 'target/x86_64-unknown-linux-musl/release/boxlite-guest', + ]) { + fs.rmSync(path.join(ROOT_DIR, rel), { force: true }) + } + fs.rmSync(path.join(ROOT_DIR, 'sdks', 'c', 'dist'), { recursive: true, force: true }) + + if (runSetup) run('make', ['setup:build']) + + run('scripts/build/build-shim.sh', ['--profile', 'release']) + run('scripts/build/build-guest.sh', ['--profile', 'release']) + + const GUEST_BIN = path.join(ROOT_DIR, 'target', 'x86_64-unknown-linux-musl', 'release', 'boxlite-guest') + let guestStat + try { + guestStat = fs.statSync(GUEST_BIN) + } catch { + guestStat = null + } + if (!guestStat || !(guestStat.mode & 0o111)) die(`guest binary not found after build: ${GUEST_BIN}`) + const GUEST_SHA256 = sha256File(GUEST_BIN) + const RUNTIME_SUFFIX = GUEST_SHA256.slice(0, 12) + const RUNTIME_CACHE_SUFFIX = `dev-${buildSequence}-${RUNTIME_SUFFIX}` + const RUNNER_VERSION = `${VERSION}-${RUNTIME_CACHE_SUFFIX}` + + console.log(`==> Building libboxlite with runtime cache key v${RUNNER_VERSION}`) + run('make', ['dist:c']) + const RUNTIME_DIR = findEmbeddedRuntimeDir(GUEST_SHA256) + run('tar', ['czf', path.join(TMP_DIR, 'boxlite-runtime.tar.gz'), '-C', RUNTIME_DIR, '.']) + console.log(`==> Wrote embedded runtime payload from ${RUNTIME_DIR}`) + fs.copyFileSync(path.join(ROOT_DIR, 'target', 'release', 'libboxlite.a'), path.join(ROOT_DIR, 'sdks', 'go', 'libboxlite.a')) + + run('go', ['-C', 'apps/runner', 'mod', 'download']) + run('go', ['-C', 'apps/common-go', 'mod', 'download']) + run('go', ['-C', 'apps/api-client-go', 'mod', 'download']) + + const RUNNER_BIN = path.join(TMP_DIR, 'boxlite-runner') + run( + 'go', + [ + 'build', + '-C', 'apps', + '-ldflags', `-X github.com/boxlite-ai/runner/internal.Version=${RUNNER_VERSION}`, + '-o', RUNNER_BIN, + './runner/cmd/runner', + ], + { env: { ...process.env, CGO_ENABLED: '1', GOOS: 'linux', GOARCH: 'amd64' } }, + ) + fs.writeFileSync(path.join(TMP_DIR, 'boxlite-runner.guest.sha256'), `${GUEST_SHA256} boxlite-guest\n`) + console.log(`==> Wrote guest hash sidecar ${GUEST_SHA256.slice(0, 12)}`) + fs.writeFileSync(path.join(TMP_DIR, 'boxlite-runner.runtime-suffix'), `${RUNTIME_CACHE_SUFFIX}\n`) + console.log(`==> Runtime cache directory: v${RUNNER_VERSION}`) + + fs.mkdirSync(outputDir, { recursive: true }) + const TARBALL = path.join(outputDir, `boxlite-runner-v${RUNNER_VERSION}-linux-amd64.tar.gz`) + run('tar', [ + 'czf', TARBALL, + '-C', TMP_DIR, + 'boxlite-runner', + 'boxlite-runner.guest.sha256', + 'boxlite-runner.runtime-suffix', + 'boxlite-runtime.tar.gz', + ]) + + fs.writeFileSync(`${TARBALL}.sha256`, `${sha256File(TARBALL)} ${TARBALL}\n`) + + console.log(`==> Wrote ${TARBALL}`) + console.log(`==> Wrote ${TARBALL}.sha256`) +} catch (err) { + if (err && err.status) process.exit(err.status) + throw err +} diff --git a/scripts/deploy/build-runner-binary.sh b/scripts/deploy/build-runner-binary.sh new file mode 100755 index 000000000..627d3026e --- /dev/null +++ b/scripts/deploy/build-runner-binary.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash +# Build the Linux amd64 boxlite-runner tarball without GitHub Actions. +# +# Run this from the checkout you want to deploy. To deploy latest main to dev: +# git checkout main +# git pull --ff-only origin main +# scripts/deploy/build-runner-binary.sh --output-dir dist +# The generated tarball is intentionally named +# `v{VERSION}-dev-{BUILD_SEQUENCE}-{GUEST_HASH}`. This script is for +# non-published builds, so the embedded runtime cache must not share the +# official release directory `v{VERSION}`. +# +# The runner is a cgo binary that links libboxlite.a. Build the embedded runtime +# inputs first (boxlite-shim + boxlite-guest), force boxlite's build.rs to rerun +# so those binaries are embedded, then build/stage the C SDK archive for Go. +# +# Usage: +# scripts/deploy/build-runner-binary.sh +# scripts/deploy/build-runner-binary.sh --output-dir /tmp +# BOXLITE_RUNNER_BUILD_NUMBER=123 scripts/deploy/build-runner-binary.sh +# scripts/deploy/build-runner-binary.sh --skip-setup + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +OUTPUT_DIR="$ROOT_DIR/dist" +RUN_SETUP=1 +BUILD_SEQUENCE="${BOXLITE_RUNNER_BUILD_NUMBER:-${GITHUB_RUN_NUMBER:-}}" + +usage() { + sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//' + cat <<'EOF' +Options: + --output-dir DIR Directory for the runner tarball. Default: ./dist + --build-number ID Ordered dev build sequence. Default: BOXLITE_RUNNER_BUILD_NUMBER, + GITHUB_RUN_NUMBER, or current UTC timestamp. + --skip-setup Skip `make setup:build`. + -h, --help Show this help. +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --output-dir) + [[ $# -ge 2 ]] || { echo "error: --output-dir requires a value" >&2; exit 1; } + OUTPUT_DIR="$2" + shift 2 + ;; + --build-number) + [[ $# -ge 2 ]] || { echo "error: --build-number requires a value" >&2; exit 1; } + BUILD_SEQUENCE="$2" + shift 2 + ;; + --skip-setup) + RUN_SETUP=0 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "error: unknown argument $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +cd "$ROOT_DIR" + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || { echo "error: required command not found: $1" >&2; exit 1; } +} + +require_cmd cargo +require_cmd go +require_cmd make +require_cmd sha256sum +require_cmd tar + +find_embedded_runtime_dir() { + local guest_sha256="$1" + local runtime_dir + local runtime_guest_sha256 + + while IFS= read -r runtime_dir; do + [ -f "$runtime_dir/boxlite-guest" ] || continue + runtime_guest_sha256="$(sha256sum "$runtime_dir/boxlite-guest" | awk '{print $1}')" + if [[ "$runtime_guest_sha256" == "$guest_sha256" ]]; then + printf '%s\n' "$runtime_dir" + return 0 + fi + done < <(find "$ROOT_DIR/target/release/build" -maxdepth 3 -path '*/boxlite-*/out/runtime' -type d 2>/dev/null | sort) + + echo "error: embedded runtime payload not found for guest hash ${guest_sha256:0:12}" >&2 + exit 1 +} + +if [[ "$(uname -s)" != "Linux" || "$(uname -m)" != "x86_64" ]]; then + echo "error: runner tarball build currently requires a Linux x86_64 builder" >&2 + exit 1 +fi + +VERSION="$(grep -m 1 '^version' "$ROOT_DIR/Cargo.toml" | sed -E 's/^version *= *"([^"]+)".*/\1/')" +if [[ -z "$VERSION" ]]; then + echo "error: could not read version from Cargo.toml" >&2 + exit 1 +fi + +if [[ -z "$BUILD_SEQUENCE" ]]; then + BUILD_SEQUENCE="$(date -u +%Y%m%d%H%M%S)" +fi +BUILD_SEQUENCE="$(printf '%s' "$BUILD_SEQUENCE" | tr -c 'A-Za-z0-9._-' '-')" +BUILD_SEQUENCE="${BUILD_SEQUENCE#v}" +[[ -n "$BUILD_SEQUENCE" ]] || { echo "error: build sequence cannot be empty" >&2; exit 1; } + +TMP_DIR="$(mktemp -d)" +GO_WORK_BACKUP="$TMP_DIR/go.work.backup" +GO_WORK_SUM_BACKUP="$TMP_DIR/go.work.sum.backup" +HAD_GO_WORK=0 +HAD_GO_WORK_SUM=0 + +restore_go_work() { + if [[ "$HAD_GO_WORK" -eq 1 ]]; then + cp "$GO_WORK_BACKUP" "$ROOT_DIR/apps/go.work" + else + rm -f "$ROOT_DIR/apps/go.work" + fi + + if [[ "$HAD_GO_WORK_SUM" -eq 1 ]]; then + cp "$GO_WORK_SUM_BACKUP" "$ROOT_DIR/apps/go.work.sum" + else + rm -f "$ROOT_DIR/apps/go.work.sum" + fi + + rm -rf "$TMP_DIR" +} +trap restore_go_work EXIT + +if [[ -f "$ROOT_DIR/apps/go.work" ]]; then + HAD_GO_WORK=1 + cp "$ROOT_DIR/apps/go.work" "$GO_WORK_BACKUP" +fi +if [[ -f "$ROOT_DIR/apps/go.work.sum" ]]; then + HAD_GO_WORK_SUM=1 + cp "$ROOT_DIR/apps/go.work.sum" "$GO_WORK_SUM_BACKUP" +fi + +GO_VERSION="$(awk '/^go / { print $2; exit }' "$ROOT_DIR/apps/runner/go.mod")" +if [[ -z "$GO_VERSION" ]]; then + echo "error: could not read Go version from apps/runner/go.mod" >&2 + exit 1 +fi + +cat > "$ROOT_DIR/apps/go.work" < Building boxlite-runner from package v$VERSION at $(git rev-parse --short HEAD)" +echo "==> Non-release artifact version prefix: v${VERSION}-dev-${BUILD_SEQUENCE}" + +echo "==> Cleaning runner build artifacts" +cargo clean -p boxlite -p boxlite-c -p boxlite-shim -p boxlite-guest +rm -f \ + "$ROOT_DIR/sdks/go/libboxlite.a" \ + "$ROOT_DIR/target/release/libboxlite.a" \ + "$ROOT_DIR/target/release/libboxlite.so" \ + "$ROOT_DIR/target/release/boxlite-shim" +rm -f "$ROOT_DIR/target/x86_64-unknown-linux-gnu/release/boxlite-shim" +rm -f "$ROOT_DIR/target/x86_64-unknown-linux-musl/release/boxlite-guest" +rm -rf "$ROOT_DIR/sdks/c/dist" + +if [[ "$RUN_SETUP" -eq 1 ]]; then + make setup:build +fi + +scripts/build/build-shim.sh --profile release +scripts/build/build-guest.sh --profile release +GUEST_BIN="$ROOT_DIR/target/x86_64-unknown-linux-musl/release/boxlite-guest" +[[ -x "$GUEST_BIN" ]] || { echo "error: guest binary not found after build: $GUEST_BIN" >&2; exit 1; } +GUEST_SHA256="$(sha256sum "$GUEST_BIN" | awk '{print $1}')" +RUNTIME_SUFFIX="${GUEST_SHA256:0:12}" +RUNTIME_CACHE_SUFFIX="dev-${BUILD_SEQUENCE}-${RUNTIME_SUFFIX}" +RUNNER_VERSION="${VERSION}-${RUNTIME_CACHE_SUFFIX}" + +echo "==> Building libboxlite with runtime cache key v${RUNNER_VERSION}" +make dist:c +RUNTIME_DIR="$(find_embedded_runtime_dir "$GUEST_SHA256")" +tar czf "$TMP_DIR/boxlite-runtime.tar.gz" -C "$RUNTIME_DIR" . +echo "==> Wrote embedded runtime payload from $RUNTIME_DIR" +cp "$ROOT_DIR/target/release/libboxlite.a" "$ROOT_DIR/sdks/go/libboxlite.a" + +go -C apps/runner mod download +go -C apps/common-go mod download +go -C apps/api-client-go mod download + +RUNNER_BIN="$TMP_DIR/boxlite-runner" +CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -C apps \ + -ldflags "-X github.com/boxlite-ai/runner/internal.Version=${RUNNER_VERSION}" \ + -o "$RUNNER_BIN" ./runner/cmd/runner +printf '%s boxlite-guest\n' "$GUEST_SHA256" > "$TMP_DIR/boxlite-runner.guest.sha256" +echo "==> Wrote guest hash sidecar ${GUEST_SHA256:0:12}" +printf '%s\n' "$RUNTIME_CACHE_SUFFIX" > "$TMP_DIR/boxlite-runner.runtime-suffix" +echo "==> Runtime cache directory: v${RUNNER_VERSION}" + +mkdir -p "$OUTPUT_DIR" +TARBALL="$OUTPUT_DIR/boxlite-runner-v${RUNNER_VERSION}-linux-amd64.tar.gz" +tar czf "$TARBALL" -C "$TMP_DIR" \ + boxlite-runner \ + boxlite-runner.guest.sha256 \ + boxlite-runner.runtime-suffix \ + boxlite-runtime.tar.gz + +if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$TARBALL" > "$TARBALL.sha256" +else + shasum -a 256 "$TARBALL" > "$TARBALL.sha256" +fi + +echo "==> Wrote $TARBALL" +echo "==> Wrote $TARBALL.sha256" diff --git a/scripts/deploy/runner-update-binary.mjs b/scripts/deploy/runner-update-binary.mjs new file mode 100755 index 000000000..ecf66c385 --- /dev/null +++ b/scripts/deploy/runner-update-binary.mjs @@ -0,0 +1,343 @@ +#!/usr/bin/env node +// Upgrade the boxlite-runner binary on the live Runner EC2 in-place. +// +// JS twin of runner-update-binary.sh — same CLI contract, same env inputs, +// same SSM flow. The bash script remains the authority; behavioural parity is +// enforced by scripts/deploy/tests/parity/run.sh. +// +// The remote upgrade payload (the script SSM runs on the runner) is NOT +// duplicated here: it is parsed out of the sibling runner-update-binary.sh +// heredoc at runtime and expanded with the same bash here-document semantics. +// One payload, two orchestrators — the halves cannot drift. +// +// Usage: +// scripts/deploy/runner-update-binary.mjs # version from Cargo.toml +// scripts/deploy/runner-update-binary.mjs 0.9.7 # official GitHub release +// scripts/deploy/runner-update-binary.mjs 0.9.7-dev-123-58d8f01bcd02 +// scripts/deploy/runner-update-binary.mjs --output-dir /tmp/dist 0.9.7-dev-123-58d8f01bcd02 +// scripts/deploy/runner-update-binary.mjs --tarball dist/boxlite-runner-v...-linux-amd64.tar.gz +// AWS_REGION=us-west-2 scripts/deploy/runner-update-binary.mjs +// STAGE=production scripts/deploy/runner-update-binary.mjs +// RUNNER_INSTANCE_ID=i-0123456789abcdef0 scripts/deploy/runner-update-binary.mjs +import { execFileSync } from 'node:child_process' +import fs from 'node:fs' +import path from 'node:path' +import process from 'node:process' +import { fileURLToPath } from 'node:url' + +const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)) +const REPO_ROOT = path.resolve(SCRIPT_DIR, '..', '..') + +function die(msg) { + process.stderr.write(`error: ${msg}\n`) + process.exit(1) +} + +function env(name, fallback = '') { + const v = process.env[name] + return v === undefined || v === '' ? fallback : v +} + +function aws(args) { + return execFileSync('aws', args, { encoding: 'utf8' }).trim() +} + +function awsRun(args) { + execFileSync('aws', args, { stdio: ['ignore', 'inherit', 'inherit'] }) +} + +function commandOnPath(cmd) { + for (const dir of (process.env.PATH || '').split(path.delimiter)) { + if (!dir) continue + try { + fs.accessSync(path.join(dir, cmd), fs.constants.X_OK) + return true + } catch { + /* keep looking */ + } + } + return false +} + +// ── bash here-document semantics ───────────────────────────────────────────── +// Replicates what `read -r -d '' SCRIPT < joins lines, \$ \` \\ unescape, unescaped ${NAME} expands, and +// read's IFS handling trims leading/trailing whitespace. +function expandHeredoc(text, vars) { + let out = '' + for (let i = 0; i < text.length; i++) { + const c = text[i] + if (c === '\\') { + const n = text[i + 1] + if (n === '\n') { + i++ // line continuation: both chars vanish + } else if (n === '$' || n === '`' || n === '\\') { + out += n + i++ + } else { + out += c + } + continue + } + if (c === '$' && text[i + 1] === '{') { + const close = text.indexOf('}', i + 2) + const name = close > 0 ? text.slice(i + 2, close) : '' + if (!/^\w+$/.test(name)) die(`heredoc expansion of unsupported form: \${${name}}`) + if (!(name in vars)) die(`heredoc references unknown variable: ${name}`) + out += vars[name] + i = close + continue + } + if (c === '$' && /\w/.test(text[i + 1] || '')) { + die(`heredoc uses unbraced expansion at offset ${i}; port it explicitly`) + } + out += c + } + return out.replace(/^[ \t\n]+/, '').replace(/[ \t\n]+$/, '') +} + +function remotePayloadTemplate() { + const authority = path.join(SCRIPT_DIR, 'runner-update-binary.sh') + let src + try { + src = fs.readFileSync(authority, 'utf8') + } catch { + die(`bash authority not found next to this script: ${authority}\n the JS twin sources its remote payload from the .sh — ship both files together`) + } + const lines = src.split('\n') + const start = lines.findIndex((l) => l.includes("read -r -d '' SCRIPT < i > start && l === 'EOF') + if (start < 0 || end < 0) die(`could not locate the SSM payload heredoc in ${authority}`) + return lines.slice(start + 1, end).join('\n') + '\n' +} + +// ── env defaults (mirror the .sh header block) ─────────────────────────────── +let STAGE = env('STAGE', 'dev') +const DEV_AWS_REGION = env('DEV_AWS_REGION', 'ap-southeast-1') +const DEV_RUNNER_INSTANCE_NAME = env('DEV_RUNNER_INSTANCE_NAME', 'boxlite-dev-runner-default') +const DEV_RUNNER_INSTANCE_ID = env('DEV_RUNNER_INSTANCE_ID') +const PROD_AWS_REGION = env('PROD_AWS_REGION', 'us-west-2') +const PROD_RUNNER_INSTANCE_NAME = env('PROD_RUNNER_INSTANCE_NAME', 'boxlite-prod-runner-default') +const PROD_RUNNER_INSTANCE_ID = env('PROD_RUNNER_INSTANCE_ID') +let AWS_REGION = env('AWS_REGION') +let RUNNER_INSTANCE_NAME = env('RUNNER_INSTANCE_NAME') +let RUNNER_INSTANCE_ID = env('RUNNER_INSTANCE_ID') +let ARTIFACT_DIR = env('RUNNER_ARTIFACT_DIR', path.join(REPO_ROOT, 'dist')) +let RUNNER_ARTIFACT_S3_URI = env('RUNNER_ARTIFACT_S3_URI') +const RUNNER_DOWNLOAD_CONNECT_TIMEOUT_SECONDS = env('RUNNER_DOWNLOAD_CONNECT_TIMEOUT_SECONDS', '15') +const RUNNER_DOWNLOAD_MAX_TIME_SECONDS = env('RUNNER_DOWNLOAD_MAX_TIME_SECONDS', '600') +const RUNNER_CHECKSUM_DOWNLOAD_MAX_TIME_SECONDS = env('RUNNER_CHECKSUM_DOWNLOAD_MAX_TIME_SECONDS', '120') +const SSM_WAIT_TIMEOUT_SECONDS = Number(env('SSM_WAIT_TIMEOUT_SECONDS', '1800')) +const SSM_WAIT_POLL_SECONDS = env('SSM_WAIT_POLL_SECONDS', '10') +let VERSION = '' +let LOCAL_TARBALL_OVERRIDE = '' + +const USAGE = `Upgrade the boxlite-runner binary on the live Runner EC2 in-place. +See runner-update-binary.sh --help for the full option list (same contract). +` + +// ── argv (mirror the .sh option loop) ──────────────────────────────────────── +const argv = process.argv.slice(2) +for (let i = 0; i < argv.length; i++) { + const arg = argv[i] + switch (arg) { + case '--output-dir': + case '--artifact-dir': + if (i + 1 >= argv.length) die(`${arg} requires a value`) + ARTIFACT_DIR = argv[++i] + break + case '--version': + if (i + 1 >= argv.length) die('--version requires a value') + VERSION = argv[++i] + break + case '--tarball': + if (i + 1 >= argv.length) die('--tarball requires a value') + LOCAL_TARBALL_OVERRIDE = argv[++i] + break + case '-h': + case '--help': + process.stdout.write(USAGE) + process.exit(0) + break + default: + if (arg.startsWith('-')) { + process.stderr.write(`error: unknown option ${arg}\n${USAGE}`) + process.exit(1) + } + if (VERSION) { + process.stderr.write(`error: unexpected extra argument ${arg}\n${USAGE}`) + process.exit(1) + } + VERSION = arg + } +} + +if (!commandOnPath('aws')) die('required command not found: aws') + +if (STAGE === 'prod') STAGE = 'production' +if (STAGE !== 'dev' && STAGE !== 'production') die(`STAGE must be dev or production (got: ${STAGE})`) + +if (STAGE === 'dev') { + AWS_REGION = AWS_REGION || DEV_AWS_REGION + RUNNER_INSTANCE_ID = RUNNER_INSTANCE_ID || DEV_RUNNER_INSTANCE_ID + RUNNER_INSTANCE_NAME = RUNNER_INSTANCE_NAME || DEV_RUNNER_INSTANCE_NAME +} else { + AWS_REGION = AWS_REGION || PROD_AWS_REGION + RUNNER_INSTANCE_ID = RUNNER_INSTANCE_ID || PROD_RUNNER_INSTANCE_ID + RUNNER_INSTANCE_NAME = RUNNER_INSTANCE_NAME || PROD_RUNNER_INSTANCE_NAME +} + +if (!VERSION) { + if (LOCAL_TARBALL_OVERRIDE) { + const base = path.basename(LOCAL_TARBALL_OVERRIDE) + const m = base.match(/^boxlite-runner-v(.+)-linux-amd64\.tar\.gz$/) + if (!m) die(`could not infer version from tarball name: ${base}`) + VERSION = m[1] + } else { + const cargo = fs.readFileSync(path.join(REPO_ROOT, 'Cargo.toml'), 'utf8') + const m = cargo.match(/^version *= *"([^"]+)"/m) + VERSION = m ? m[1] : '' + } + if (!VERSION) die(`could not read version from Cargo.toml at ${path.join(REPO_ROOT, 'Cargo.toml')}`) +} + +const ASSET_TARBALL = `boxlite-runner-v${VERSION}-linux-amd64.tar.gz` +const LOCAL_TARBALL = LOCAL_TARBALL_OVERRIDE || path.join(ARTIFACT_DIR, ASSET_TARBALL) +const LOCAL_SHA = `${LOCAL_TARBALL}.sha256` +const devIdx = VERSION.indexOf('-dev-') +const RUNTIME_CACHE_VERSION = devIdx >= 0 ? VERSION.slice(0, devIdx) : VERSION +const IS_DEV_VERSION = VERSION.includes('-dev-') + +if (IS_DEV_VERSION) { + if (!fs.existsSync(LOCAL_TARBALL)) { + process.stderr.write(`error: local dev runner tarball not found: ${LOCAL_TARBALL}\n`) + process.stderr.write(' run scripts/deploy/build-runner-binary.sh first, or set --output-dir/RUNNER_ARTIFACT_DIR\n') + process.exit(1) + } + if (!fs.existsSync(LOCAL_SHA)) { + process.stderr.write(`error: local dev runner checksum not found: ${LOCAL_SHA}\n`) + process.stderr.write(' run scripts/deploy/build-runner-binary.sh first\n') + process.exit(1) + } +} + +if (IS_DEV_VERSION) { + console.log(`==> Upgrading boxlite-runner from local dev artifact v${VERSION} on stage=${STAGE} region=${AWS_REGION}`) +} else { + console.log(`==> Upgrading boxlite-runner from GitHub release v${VERSION} on stage=${STAGE} region=${AWS_REGION}`) +} + +let INSTANCE_ID = RUNNER_INSTANCE_ID +if (!INSTANCE_ID) { + const out = aws([ + 'ec2', 'describe-instances', + '--region', AWS_REGION, + '--filters', + `Name=tag:Name,Values=${RUNNER_INSTANCE_NAME}`, + 'Name=instance-state-name,Values=running', + '--query', 'Reservations[].Instances[].InstanceId', + '--output', 'text', + ]) + const ids = out.split(/\s+/).filter(Boolean) + if (ids.length !== 1) { + process.stderr.write(`error: expected exactly one running runner instance, found ${ids.length}\n`) + process.stderr.write(` region=${AWS_REGION} stage=${STAGE} name=${RUNNER_INSTANCE_NAME}\n`) + if (ids.length > 0) process.stderr.write(` matches: ${ids.join(' ')}\n`) + process.stderr.write(' set RUNNER_INSTANCE_ID=i-... to target an instance explicitly\n') + process.exit(1) + } + INSTANCE_ID = ids[0] +} +console.log(` instance: ${INSTANCE_ID}`) + +let DOWNLOAD_TARBALL_URL +let DOWNLOAD_SHA_URL +if (IS_DEV_VERSION) { + if (!RUNNER_ARTIFACT_S3_URI) { + const accountId = aws(['sts', 'get-caller-identity', '--query', 'Account', '--output', 'text']) + const ts = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z') + RUNNER_ARTIFACT_S3_URI = `s3://boxlite-${STAGE}-runner-builds/tmp/runner-rollouts/${accountId}/${VERSION}/${ts}` + } + const remoteTarballUrl = `${RUNNER_ARTIFACT_S3_URI.replace(/\/$/, '')}/${ASSET_TARBALL}` + const remoteShaUrl = `${remoteTarballUrl}.sha256` + console.log(`==> Uploading local artifact to ${RUNNER_ARTIFACT_S3_URI}`) + awsRun(['s3', 'cp', '--region', AWS_REGION, LOCAL_TARBALL, remoteTarballUrl]) + awsRun(['s3', 'cp', '--region', AWS_REGION, LOCAL_SHA, remoteShaUrl]) + DOWNLOAD_TARBALL_URL = aws(['s3', 'presign', '--region', AWS_REGION, remoteTarballUrl, '--expires-in', '3600']) + DOWNLOAD_SHA_URL = aws(['s3', 'presign', '--region', AWS_REGION, remoteShaUrl, '--expires-in', '3600']) +} else { + const releaseRepository = env('BOXLITE_RELEASE_REPOSITORY', 'boxlite-ai/boxlite') + const releaseTag = env('BOXLITE_RELEASE_TAG', `v${VERSION}`) + const releaseBaseUrl = `https://github.com/${releaseRepository}/releases/download/${releaseTag}` + DOWNLOAD_TARBALL_URL = `${releaseBaseUrl}/${ASSET_TARBALL}` + DOWNLOAD_SHA_URL = `${DOWNLOAD_TARBALL_URL}.sha256` + console.log(`==> Target runner will download ${DOWNLOAD_TARBALL_URL}`) +} + +const SCRIPT = expandHeredoc(remotePayloadTemplate(), { + RUNTIME_CACHE_VERSION, + VERSION, + RUNNER_DOWNLOAD_CONNECT_TIMEOUT_SECONDS, + RUNNER_DOWNLOAD_MAX_TIME_SECONDS, + RUNNER_CHECKSUM_DOWNLOAD_MAX_TIME_SECONDS, + DOWNLOAD_TARBALL_URL, + DOWNLOAD_SHA_URL, +}) + +// Hand the script to SSM base64-encoded, exactly like the .sh (see its note on +// why: a single token with no shell metacharacters). +const SCRIPT_B64 = Buffer.from(SCRIPT, 'utf8').toString('base64') +const CMD_ID = aws([ + 'ssm', 'send-command', + '--region', AWS_REGION, + '--document-name', 'AWS-RunShellScript', + '--instance-ids', INSTANCE_ID, + '--comment', `boxlite-runner upgrade to v${VERSION}`, + '--parameters', `commands=["echo ${SCRIPT_B64} | base64 -d | bash"]`, + '--query', 'Command.CommandId', + '--output', 'text', +]) + +console.log(` command: ${CMD_ID}`) +console.log(`==> Waiting for SSM command to finish (timeout=${SSM_WAIT_TIMEOUT_SECONDS}s)...`) + +function invocation(query) { + return aws([ + 'ssm', 'get-command-invocation', + '--region', AWS_REGION, + '--command-id', CMD_ID, + '--instance-id', INSTANCE_ID, + '--query', query, + '--output', 'text', + ]) +} + +let STATUS = '' +const deadline = Date.now() + SSM_WAIT_TIMEOUT_SECONDS * 1000 +for (;;) { + try { + STATUS = invocation('Status') + } catch { + STATUS = '' + } + if (['Success', 'Failed', 'Cancelled', 'TimedOut', 'Cancelling'].includes(STATUS)) break + if (Date.now() >= deadline) { + process.stderr.write(`error: SSM command still ${STATUS || 'unknown'} after ${SSM_WAIT_TIMEOUT_SECONDS}s\n`) + process.exit(1) + } + execFileSync('sleep', [SSM_WAIT_POLL_SECONDS]) +} + +STATUS = invocation('Status') + +console.log() +console.log(`==> SSM status: ${STATUS}`) +console.log() +console.log(invocation('StandardOutputContent')) + +if (STATUS !== 'Success') { + console.log() + console.log('==> stderr:') + console.log(invocation('StandardErrorContent')) + process.exit(1) +} diff --git a/scripts/deploy/runner-update-binary.sh b/scripts/deploy/runner-update-binary.sh index d4eeccbdb..476fca2f0 100755 --- a/scripts/deploy/runner-update-binary.sh +++ b/scripts/deploy/runner-update-binary.sh @@ -1,9 +1,13 @@ #!/usr/bin/env bash # Upgrade the boxlite-runner binary on the live Runner EC2 in-place. # -# Replaces /usr/local/bin/boxlite-runner with a freshly downloaded release -# binary and restarts the systemd unit. The EC2 instance itself is not -# replaced; box state under /var/lib/boxlite is preserved. +# Installs the new binary into a versioned release directory, points +# /usr/local/bin/boxlite-runner at it, restarts the systemd unit, and verifies +# that detached box shims that were alive before the restart are still alive and +# can be re-attached by the new runner. The EC2 instance itself is not replaced; +# box state under /var/lib/boxlite is preserved. Official versions are fetched +# directly from GitHub Releases by the target runner; dev builds upload the +# locally built runner tarball to a temporary S3 location first. # # Pair with the `ignoreChanges: ["ami", "userDataBase64"]` setting on the # Runner resource in apps/infra/sst.config.ts — that prevents `sst deploy` @@ -12,86 +16,650 @@ # # Usage: # scripts/deploy/runner-update-binary.sh # version from Cargo.toml -# scripts/deploy/runner-update-binary.sh 0.9.5 # explicit version +# scripts/deploy/runner-update-binary.sh 0.9.7 # official GitHub release +# scripts/deploy/runner-update-binary.sh 0.9.7-dev-123-58d8f01bcd02 +# scripts/deploy/runner-update-binary.sh --output-dir /tmp/dist 0.9.7-dev-123-58d8f01bcd02 +# scripts/deploy/runner-update-binary.sh --tarball dist/boxlite-runner-v0.9.7-dev-123-58d8f01bcd02-linux-amd64.tar.gz # AWS_REGION=us-west-2 scripts/deploy/runner-update-binary.sh # STAGE=production scripts/deploy/runner-update-binary.sh +# RUNNER_INSTANCE_ID=i-0123456789abcdef0 scripts/deploy/runner-update-binary.sh +# PROD_RUNNER_INSTANCE_ID=i-0123456789abcdef0 STAGE=production scripts/deploy/runner-update-binary.sh set -euo pipefail -AWS_REGION="${AWS_REGION:-ap-southeast-1}" STAGE="${STAGE:-dev}" +DEV_AWS_REGION="${DEV_AWS_REGION:-ap-southeast-1}" +DEV_RUNNER_INSTANCE_NAME="${DEV_RUNNER_INSTANCE_NAME:-boxlite-dev-runner-default}" +DEV_RUNNER_INSTANCE_ID="${DEV_RUNNER_INSTANCE_ID:-}" +PROD_AWS_REGION="${PROD_AWS_REGION:-us-west-2}" +PROD_RUNNER_INSTANCE_NAME="${PROD_RUNNER_INSTANCE_NAME:-boxlite-prod-runner-default}" +PROD_RUNNER_INSTANCE_ID="${PROD_RUNNER_INSTANCE_ID:-}" +AWS_REGION="${AWS_REGION:-}" +RUNNER_INSTANCE_NAME="${RUNNER_INSTANCE_NAME:-}" +RUNNER_INSTANCE_ID="${RUNNER_INSTANCE_ID:-}" REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +ARTIFACT_DIR="${RUNNER_ARTIFACT_DIR:-$REPO_ROOT/dist}" +RUNNER_ARTIFACT_S3_URI="${RUNNER_ARTIFACT_S3_URI:-}" +RUNNER_DOWNLOAD_CONNECT_TIMEOUT_SECONDS="${RUNNER_DOWNLOAD_CONNECT_TIMEOUT_SECONDS:-15}" +RUNNER_DOWNLOAD_MAX_TIME_SECONDS="${RUNNER_DOWNLOAD_MAX_TIME_SECONDS:-600}" +RUNNER_CHECKSUM_DOWNLOAD_MAX_TIME_SECONDS="${RUNNER_CHECKSUM_DOWNLOAD_MAX_TIME_SECONDS:-120}" +SSM_WAIT_TIMEOUT_SECONDS="${SSM_WAIT_TIMEOUT_SECONDS:-1800}" +SSM_WAIT_POLL_SECONDS="${SSM_WAIT_POLL_SECONDS:-10}" +VERSION="" +LOCAL_TARBALL_OVERRIDE="" -if [[ $# -ge 1 ]]; then - VERSION="$1" -else - VERSION=$(grep -m 1 '^version' "$REPO_ROOT/Cargo.toml" | sed -E 's/^version *= *"([^"]+)".*/\1/') +usage() { + sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//' +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --output-dir|--artifact-dir) + [[ $# -ge 2 ]] || { echo "error: $1 requires a value" >&2; exit 1; } + ARTIFACT_DIR="$2" + shift 2 + ;; + --version) + [[ $# -ge 2 ]] || { echo "error: --version requires a value" >&2; exit 1; } + VERSION="$2" + shift 2 + ;; + --tarball) + [[ $# -ge 2 ]] || { echo "error: --tarball requires a value" >&2; exit 1; } + LOCAL_TARBALL_OVERRIDE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + -*) + echo "error: unknown option $1" >&2 + usage >&2 + exit 1 + ;; + *) + if [[ -n "$VERSION" ]]; then + echo "error: unexpected extra argument $1" >&2 + usage >&2 + exit 1 + fi + VERSION="$1" + shift + ;; + esac +done + +command -v aws >/dev/null 2>&1 || { echo "error: required command not found: aws" >&2; exit 1; } + +case "$STAGE" in + dev|production) ;; + prod) STAGE="production" ;; + *) + echo "error: STAGE must be dev or production (got: $STAGE)" >&2 + exit 1 + ;; +esac + +case "$STAGE" in + dev) + AWS_REGION="${AWS_REGION:-$DEV_AWS_REGION}" + RUNNER_INSTANCE_ID="${RUNNER_INSTANCE_ID:-$DEV_RUNNER_INSTANCE_ID}" + RUNNER_INSTANCE_NAME="${RUNNER_INSTANCE_NAME:-$DEV_RUNNER_INSTANCE_NAME}" + ;; + production) + AWS_REGION="${AWS_REGION:-$PROD_AWS_REGION}" + RUNNER_INSTANCE_ID="${RUNNER_INSTANCE_ID:-$PROD_RUNNER_INSTANCE_ID}" + RUNNER_INSTANCE_NAME="${RUNNER_INSTANCE_NAME:-$PROD_RUNNER_INSTANCE_NAME}" + ;; +esac + +if [[ -z "$VERSION" ]]; then + if [[ -n "$LOCAL_TARBALL_OVERRIDE" ]]; then + TARBALL_BASENAME="$(basename "$LOCAL_TARBALL_OVERRIDE")" + VERSION="$(printf '%s\n' "$TARBALL_BASENAME" | sed -E 's/^boxlite-runner-v(.+)-linux-amd64\.tar\.gz$/\1/')" + if [[ "$VERSION" == "$TARBALL_BASENAME" ]]; then + echo "error: could not infer version from tarball name: $TARBALL_BASENAME" >&2 + exit 1 + fi + else + VERSION=$(grep -m 1 '^version' "$REPO_ROOT/Cargo.toml" | sed -E 's/^version *= *"([^"]+)".*/\1/') + fi if [[ -z "$VERSION" ]]; then echo "error: could not read version from Cargo.toml at $REPO_ROOT/Cargo.toml" >&2 exit 1 fi fi -echo "==> Upgrading boxlite-runner to v$VERSION on stage=$STAGE region=$AWS_REGION" +ASSET_TARBALL="boxlite-runner-v${VERSION}-linux-amd64.tar.gz" +LOCAL_TARBALL="${LOCAL_TARBALL_OVERRIDE:-$ARTIFACT_DIR/$ASSET_TARBALL}" +LOCAL_SHA="$LOCAL_TARBALL.sha256" +RUNTIME_CACHE_VERSION="${VERSION%%-dev-*}" +IS_DEV_VERSION=0 +if [[ "$VERSION" == *-dev-* ]]; then + IS_DEV_VERSION=1 +fi + +if [[ "$IS_DEV_VERSION" -eq 1 ]]; then + if [[ ! -f "$LOCAL_TARBALL" ]]; then + echo "error: local dev runner tarball not found: $LOCAL_TARBALL" >&2 + echo " run scripts/deploy/build-runner-binary.sh first, or set --output-dir/RUNNER_ARTIFACT_DIR" >&2 + exit 1 + fi + if [[ ! -f "$LOCAL_SHA" ]]; then + echo "error: local dev runner checksum not found: $LOCAL_SHA" >&2 + echo " run scripts/deploy/build-runner-binary.sh first" >&2 + exit 1 + fi +fi + +if [[ "$IS_DEV_VERSION" -eq 1 ]]; then + echo "==> Upgrading boxlite-runner from local dev artifact v$VERSION on stage=$STAGE region=$AWS_REGION" +else + echo "==> Upgrading boxlite-runner from GitHub release v$VERSION on stage=$STAGE region=$AWS_REGION" +fi -INSTANCE_ID=$(aws ec2 describe-instances --region "$AWS_REGION" \ - --filters "Name=tag:Name,Values=boxlite-runner-default" "Name=instance-state-name,Values=running" \ - --query 'Reservations[].Instances[].InstanceId' --output text) +if [[ -n "$RUNNER_INSTANCE_ID" ]]; then + INSTANCE_ID="$RUNNER_INSTANCE_ID" +else + INSTANCE_FILTERS=( + "Name=tag:Name,Values=${RUNNER_INSTANCE_NAME}" + "Name=instance-state-name,Values=running" + ) + INSTANCE_IDS=() + while IFS= read -r instance_id; do + INSTANCE_IDS+=("$instance_id") + done < <(aws ec2 describe-instances --region "$AWS_REGION" \ + --filters "${INSTANCE_FILTERS[@]}" \ + --query 'Reservations[].Instances[].InstanceId' --output text | tr '\t' '\n' | sed '/^$/d') -if [[ -z "$INSTANCE_ID" || "$INSTANCE_ID" == "None" ]]; then - echo "error: no running boxlite-runner-default instance found in region $AWS_REGION" >&2 - exit 1 + if [[ "${#INSTANCE_IDS[@]}" -ne 1 ]]; then + echo "error: expected exactly one running runner instance, found ${#INSTANCE_IDS[@]}" >&2 + echo " region=$AWS_REGION stage=$STAGE name=$RUNNER_INSTANCE_NAME" >&2 + if [[ "${#INSTANCE_IDS[@]}" -gt 0 ]]; then + printf ' matches: %s\n' "${INSTANCE_IDS[*]}" >&2 + fi + echo " set RUNNER_INSTANCE_ID=i-... to target an instance explicitly" >&2 + exit 1 + fi + INSTANCE_ID="${INSTANCE_IDS[0]}" fi echo " instance: $INSTANCE_ID" -ASSET_BASE="https://github.com/boxlite-ai/boxlite/releases/download/v${VERSION}" -ASSET_TARBALL="boxlite-runner-v${VERSION}-linux-amd64.tar.gz" +if [[ "$IS_DEV_VERSION" -eq 1 ]]; then + if [[ -z "$RUNNER_ARTIFACT_S3_URI" ]]; then + ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + RUNNER_ARTIFACT_S3_URI="s3://boxlite-${STAGE}-runner-builds/tmp/runner-rollouts/${ACCOUNT_ID}/${VERSION}/$(date -u +%Y%m%dT%H%M%SZ)" + fi + REMOTE_TARBALL_URL="${RUNNER_ARTIFACT_S3_URI%/}/$ASSET_TARBALL" + REMOTE_SHA_URL="$REMOTE_TARBALL_URL.sha256" + echo "==> Uploading local artifact to $RUNNER_ARTIFACT_S3_URI" + aws s3 cp --region "$AWS_REGION" "$LOCAL_TARBALL" "$REMOTE_TARBALL_URL" + aws s3 cp --region "$AWS_REGION" "$LOCAL_SHA" "$REMOTE_SHA_URL" + DOWNLOAD_TARBALL_URL=$(aws s3 presign --region "$AWS_REGION" "$REMOTE_TARBALL_URL" --expires-in 3600) + DOWNLOAD_SHA_URL=$(aws s3 presign --region "$AWS_REGION" "$REMOTE_SHA_URL" --expires-in 3600) +else + RELEASE_REPOSITORY="${BOXLITE_RELEASE_REPOSITORY:-boxlite-ai/boxlite}" + RELEASE_TAG="${BOXLITE_RELEASE_TAG:-v${VERSION}}" + RELEASE_BASE_URL="https://github.com/${RELEASE_REPOSITORY}/releases/download/${RELEASE_TAG}" + DOWNLOAD_TARBALL_URL="$RELEASE_BASE_URL/$ASSET_TARBALL" + DOWNLOAD_SHA_URL="$DOWNLOAD_TARBALL_URL.sha256" + echo "==> Target runner will download $DOWNLOAD_TARBALL_URL" +fi # Remote upgrade script. Mirrors the boot user-data's integrity policy and adds a # rollback: download + checksum-verify BEFORE stopping the unit (so a failed or -# corrupt fetch never takes the runner down), back up the live binary, swap it in, -# and restore the backup if the new binary fails to come up. +# corrupt fetch never takes the runner down), install the new binary into a +# versioned release directory, switch the live symlink, and switch back if the +# new binary fails to come up. read -r -d '' SCRIPT <&2 + exit 1 + fi + if [ "\$ENABLE_TLS" = true ]; then + RUNNER_BASE_URL="https://127.0.0.1:\$API_PORT" + CURL_TLS=(-k) + else + RUNNER_BASE_URL="http://127.0.0.1:\$API_PORT" + CURL_TLS=() + fi +} + +proc_start_time() { + awk '{print \$22}' "/proc/\$1/stat" 2>/dev/null || true +} + +proc_state() { + awk '{print \$3}' "/proc/\$1/stat" 2>/dev/null || true +} + +pid_alive() { + local pid="\$1" + [ -n "\$pid" ] || return 1 + kill -0 "\$pid" 2>/dev/null || return 1 + [ "\$(proc_state "\$pid")" != Z ] +} + +snapshot_live_detached_shims() { + : > "\$HOT_SNAPSHOT" + local boxes_dir="\$BOXLITE_HOME_DIR/boxes" + [ -d "\$boxes_dir" ] || return 0 + while IFS= read -r pid_file; do + local box_id pid start actual_start + box_id="\$(basename "\$(dirname "\$pid_file")")" + pid="\$(sed -n '1p' "\$pid_file" 2>/dev/null | tr -dc '0-9')" + start="\$(sed -n '2p' "\$pid_file" 2>/dev/null | tr -dc '0-9')" + pid_alive "\$pid" || continue + if [ -n "\$start" ]; then + actual_start="\$(proc_start_time "\$pid")" + [ "\$actual_start" = "\$start" ] || continue + fi + printf '%s %s %s\n' "\$box_id" "\$pid" "\$start" >> "\$HOT_SNAPSHOT" + done < <(find "\$boxes_dir" -mindepth 2 -maxdepth 2 -name shim.pid -type f 2>/dev/null | sort) +} + +wait_runner_ready() { + local i + for i in \$(seq 1 30); do + if curl -fsS "\${CURL_TLS[@]}" "\$RUNNER_BASE_URL/" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + echo "FATAL: runner API did not become ready at \$RUNNER_BASE_URL" >&2 + return 1 +} + +verify_runner_health_samples() { + local i + for i in 1 2; do + curl -fsS "\${CURL_TLS[@]}" \ + -H "Authorization: Bearer \$BOXLITE_RUNNER_TOKEN" \ + "\$RUNNER_BASE_URL/info" >/dev/null || { + echo "FATAL: runner health sample \$i failed at \$RUNNER_BASE_URL/info" >&2 + return 1 + } + echo "runner health sample \$i: ok" + [ "\$i" -eq 2 ] || sleep 2 + done +} + +runtime_cache_dirs() { + local version_dir="\${RUNTIME_CACHE_DIR_NAME:-v${RUNTIME_CACHE_VERSION}}" + local -a data_roots=() + local svc_user svc_home env_value + + svc_user=\$(systemctl show "\$SERVICE" --property=User --value 2>/dev/null || true) + if [ -z "\$svc_user" ]; then + svc_user=root + fi + + svc_home=\$(getent passwd "\$svc_user" 2>/dev/null | cut -d: -f6 || true) + [ -n "\$svc_home" ] && data_roots+=("\$svc_home/.local/share") + + while IFS= read -r env_value; do + case "\$env_value" in + XDG_DATA_HOME=*) data_roots+=("\${env_value#XDG_DATA_HOME=}") ;; + HOME=*) data_roots+=("\${env_value#HOME=}/.local/share") ;; + esac + done < <(systemctl show "\$SERVICE" --property=Environment --value | tr ' ' '\n') + + data_roots+=("/root/.local/share") + for home in /home/*; do + [ -d "\$home" ] && data_roots+=("\$home/.local/share") + done + + local seen=" " + local root cache_dir + for root in "\${data_roots[@]}"; do + [ -n "\$root" ] || continue + case "\$seen" in *" \$root "*) continue ;; esac + seen="\$seen\$root " + cache_dir="\$root/boxlite/runtimes/\$version_dir" + [ -d "\$cache_dir" ] && printf '%s\n' "\$cache_dir" + done +} + +primary_runtime_cache_dir() { + local version_dir="\${RUNTIME_CACHE_DIR_NAME:-v${RUNTIME_CACHE_VERSION}}" + local svc_user svc_home + + svc_user=\$(systemctl show "\$SERVICE" --property=User --value 2>/dev/null || true) + if [ -z "\$svc_user" ]; then + svc_user=root + fi + + svc_home=\$(getent passwd "\$svc_user" 2>/dev/null | cut -d: -f6 || true) + if [ -z "\$svc_home" ]; then + svc_home=/root + fi + + printf '%s\n' "\$svc_home/.local/share/boxlite/runtimes/\$version_dir" +} + +install_embedded_runtime_payload() { + local payload="\$1" + local cache_dir tmp_dir guest_hash + + if [ -z "\${GUEST_EXPECTED:-}" ]; then + echo "embedded runtime install skipped: no expected guest hash sidecar" + return 0 + fi + if [ ! -f "\$payload" ]; then + echo "embedded runtime install skipped: no runtime payload in tarball" + return 0 + fi + + cache_dir="\$(primary_runtime_cache_dir)" + tmp_dir="\${cache_dir}.tmp.\$\$" + rm -rf "\$tmp_dir" + mkdir -p "\$tmp_dir" + tar -xzf "\$payload" -C "\$tmp_dir" + + if [ ! -f "\$tmp_dir/boxlite-guest" ]; then + echo "FATAL: embedded runtime payload has no boxlite-guest" >&2 + rm -rf "\$tmp_dir" + return 1 + fi + + guest_hash=\$(sha256sum "\$tmp_dir/boxlite-guest" | awk '{print \$1}') + if [ "\$guest_hash" != "\$GUEST_EXPECTED" ]; then + echo "FATAL: embedded runtime payload guest hash mismatch (expected=\${GUEST_EXPECTED:0:12} actual=\${guest_hash:0:12})" >&2 + rm -rf "\$tmp_dir" + return 1 + fi + + mkdir -p "\$(dirname "\$cache_dir")" + rm -rf "\$cache_dir" + mv "\$tmp_dir" "\$cache_dir" + echo "embedded runtime payload installed: \${guest_hash:0:12} (\$cache_dir)" +} + +write_runtime_dir_override() { + local cache_dir + cache_dir="\$(primary_runtime_cache_dir)" + if [ ! -d "\$cache_dir" ]; then + echo "FATAL: runtime cache directory missing before start: \$cache_dir" >&2 + return 1 + fi + + mkdir -p "/etc/systemd/system/\$SERVICE.service.d" + cat > "/etc/systemd/system/\$SERVICE.service.d/runtime-dir.conf" <&2 + return 1 + fi + echo "embedded runtime guest hash verified: \${guest_hash:0:12} (\$cache_dir)" + done < <(runtime_cache_dirs) + + if [ "\$checked" -eq 0 ]; then + echo "FATAL: embedded runtime cache not found for \$RUNTIME_CACHE_DIR_NAME; refusing rollout before create/start can hit an old guest binary" >&2 + return 1 + fi +} + +verify_hot_adopted_shims() { + local count=0 + if [ ! -s "\$HOT_SNAPSHOT" ]; then + echo "hot rollout: no live detached shims to adopt" + return 0 + fi + + while read -r box_id before_pid before_start; do + [ -n "\$box_id" ] || continue + count=\$((count + 1)) + local pid_file now_pid now_start + pid_file="\$BOXLITE_HOME_DIR/boxes/\$box_id/shim.pid" + now_pid="\$(sed -n '1p' "\$pid_file" 2>/dev/null | tr -dc '0-9')" + now_start="\$(sed -n '2p' "\$pid_file" 2>/dev/null | tr -dc '0-9')" + [ "\$now_pid" = "\$before_pid" ] || { + echo "FATAL: box \$box_id shim pid changed across rollout (before=\$before_pid after=\$now_pid)" >&2 + return 1 + } + pid_alive "\$now_pid" || { + echo "FATAL: box \$box_id shim pid \$now_pid is not alive after rollout" >&2 + return 1 + } + if [ -n "\$before_start" ]; then + [ "\$now_start" = "\$before_start" ] || { + echo "FATAL: box \$box_id shim start-time changed across rollout" >&2 + return 1 + } + [ "\$(proc_start_time "\$now_pid")" = "\$before_start" ] || { + echo "FATAL: box \$box_id shim pid \$now_pid failed start-time verification" >&2 + return 1 + } + fi + + # This forces the new runner process through getOrFetchBox -> runtime.Get -> + # vmm_attach for boxes that only existed in the previous runner's memory. + curl -fsS "\${CURL_TLS[@]}" \ + -H "Authorization: Bearer \$BOXLITE_RUNNER_TOKEN" \ + "\$RUNNER_BASE_URL/v1/boxes/\$box_id/metrics" >/dev/null || { + echo "FATAL: new runner failed to attach/probe detached box \$box_id" >&2 + return 1 + } + echo "hot rollout: adopted \$box_id pid=\$now_pid" + done < "\$HOT_SNAPSHOT" + + echo "hot rollout: adopted \$count detached box(es)" +} + +prepare_release_target() { + local source_binary="\$1" + local release_id="\$2" + local release_dir="\$RELEASES_DIR/\$release_id" + mkdir -p "\$release_dir" + install -m 0755 "\$source_binary" "\$release_dir/boxlite-runner" + sha256sum "\$release_dir/boxlite-runner" | awk '{print \$1}' > "\$release_dir/boxlite-runner.sha256" + printf '%s\n' "\$release_dir/boxlite-runner" +} + +verify_release_target() { + local target="\$1" + local label="\$2" + local expected actual sha_file + + if [ ! -x "\$target" ]; then + echo "FATAL: \$label runner target is not executable: \$target" >&2 + return 1 + fi + + sha_file="\$(dirname "\$target")/boxlite-runner.sha256" + actual=\$(sha256sum "\$target" | awk '{print \$1}') + if [ -f "\$sha_file" ]; then + expected=\$(awk '{print \$1}' "\$sha_file") + if [ "\$expected" != "\$actual" ]; then + echo "FATAL: \$label runner target hash mismatch: \$target (expected=\${expected:0:12} actual=\${actual:0:12})" >&2 + return 1 + fi + else + echo "\$actual" > "\$sha_file" + echo "WARNING: \$label runner target had no hash sidecar; recorded current hash \${actual:0:12}" >&2 + fi + + echo "\$label runner target verified: \${actual:0:12} (\$target)" +} + +current_runner_target() { + if [ -L "\$RUNNER_BIN" ]; then + readlink -f "\$RUNNER_BIN" 2>/dev/null || true + elif [ -x "\$RUNNER_BIN" ]; then + local legacy_dir="\$RELEASES_DIR/legacy-\$(date +%Y%m%d%H%M%S)" + mkdir -p "\$legacy_dir" + cp -a "\$RUNNER_BIN" "\$legacy_dir/boxlite-runner" + sha256sum "\$legacy_dir/boxlite-runner" | awk '{print \$1}' > "\$legacy_dir/boxlite-runner.sha256" + printf '%s\n' "\$legacy_dir/boxlite-runner" + fi +} + +activate_runner_target() { + local target="\$1" + ln -sfn "\$target" "\$RUNNER_BIN" +} + +prune_old_releases() { + [ "\$KEEP_RELEASES" -gt 0 ] 2>/dev/null || return 0 + [ -d "\$RELEASES_DIR" ] || return 0 + find "\$RELEASES_DIR" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' \ + | sort -rn \ + | awk -v keep="\$KEEP_RELEASES" 'NR > keep { sub(/^[^ ]+ /, ""); print }' \ + | while IFS= read -r old_dir; do + rm -rf "\$old_dir" + done +} + +restart_with_target() { + local target="\$1" + systemctl stop "\$SERVICE" || true + activate_runner_target "\$target" || return 1 + write_runtime_dir_override || return 1 + systemctl start "\$SERVICE" || return 1 + sleep 2 + systemctl is-active --quiet "\$SERVICE" || return 1 + wait_runner_ready || return 1 + verify_runner_health_samples || return 1 + verify_embedded_runtime_hash || return 1 + verify_hot_adopted_shims || return 1 +} + +restart_rollback_target() { + local target="\$1" + local saved_guest_expected="\${GUEST_EXPECTED:-}" + local saved_runtime_suffix="\${RUNTIME_SUFFIX:-}" + local saved_runtime_cache_dir_name="\${RUNTIME_CACHE_DIR_NAME:-}" + + GUEST_EXPECTED="" + RUNTIME_SUFFIX="" + RUNTIME_CACHE_DIR_NAME="v${RUNTIME_CACHE_VERSION}" + restart_with_target "\$target" + local rc=\$? + + GUEST_EXPECTED="\$saved_guest_expected" + RUNTIME_SUFFIX="\$saved_runtime_suffix" + RUNTIME_CACHE_DIR_NAME="\$saved_runtime_cache_dir_name" + return "\$rc" +} + +load_runner_env +snapshot_live_detached_shims +echo "hot rollout: captured \$(wc -l < "\$HOT_SNAPSHOT" | tr -d ' ') live detached shim(s)" WORK=\$(mktemp -d) -trap 'rm -rf "\$WORK"' EXIT -curl -fsSL "${ASSET_BASE}/${ASSET_TARBALL}" -o "\$WORK/runner.tar.gz" -if curl -fsSL "${ASSET_BASE}/${ASSET_TARBALL}.sha256" -o "\$WORK/runner.sha256"; then +trap 'rm -rf "\$WORK"; rm -f "\$HOT_SNAPSHOT"' EXIT +curl -fL --show-error --silent \ + --retry 5 --retry-delay 2 --retry-connrefused \ + --connect-timeout "${RUNNER_DOWNLOAD_CONNECT_TIMEOUT_SECONDS}" \ + --max-time "${RUNNER_DOWNLOAD_MAX_TIME_SECONDS}" \ + "${DOWNLOAD_TARBALL_URL}" -o "\$WORK/runner.tar.gz" +if curl -fL --show-error --silent \ + --retry 3 --retry-delay 2 --retry-connrefused \ + --connect-timeout "${RUNNER_DOWNLOAD_CONNECT_TIMEOUT_SECONDS}" \ + --max-time "${RUNNER_CHECKSUM_DOWNLOAD_MAX_TIME_SECONDS}" \ + "${DOWNLOAD_SHA_URL}" -o "\$WORK/runner.sha256"; then EXPECTED=\$(awk '{print \$1}' "\$WORK/runner.sha256") ACTUAL=\$(sha256sum "\$WORK/runner.tar.gz" | awk '{print \$1}') [ "\$EXPECTED" = "\$ACTUAL" ] || { echo "FATAL: checksum mismatch (want \$EXPECTED got \$ACTUAL)" >&2; exit 1; } echo "checksum verified (\$ACTUAL)" else - echo "WARNING: no .sha256 published for v${VERSION}; installing without integrity verification" >&2 + echo "WARNING: no checksum available for runner tarball; installing without integrity verification" >&2 fi tar -xzf "\$WORK/runner.tar.gz" -C "\$WORK" test -x "\$WORK/boxlite-runner" || { echo "FATAL: tarball has no boxlite-runner binary" >&2; exit 1; } +if [ -f "\$WORK/boxlite-runner.guest.sha256" ]; then + GUEST_EXPECTED=\$(awk '{print \$1}' "\$WORK/boxlite-runner.guest.sha256") + echo "runner expected guest hash: \${GUEST_EXPECTED:0:12}" +else + GUEST_EXPECTED="" + echo "WARNING: tarball has no guest hash sidecar; skipping pre-install guest hash verification" >&2 +fi +if [ -f "\$WORK/boxlite-runner.runtime-suffix" ]; then + RUNTIME_SUFFIX=\$(tr -dc 'A-Za-z0-9._-' < "\$WORK/boxlite-runner.runtime-suffix") + if [ -n "\$RUNTIME_SUFFIX" ]; then + RUNTIME_CACHE_DIR_NAME="v${RUNTIME_CACHE_VERSION}-\$RUNTIME_SUFFIX" + echo "runner runtime cache key: \$RUNTIME_CACHE_DIR_NAME" + else + RUNTIME_CACHE_DIR_NAME="v${RUNTIME_CACHE_VERSION}" + echo "WARNING: runtime suffix sidecar is empty; using \$RUNTIME_CACHE_DIR_NAME" >&2 + fi +else + RUNTIME_CACHE_DIR_NAME="v${RUNTIME_CACHE_VERSION}" + echo "runner runtime cache key: \$RUNTIME_CACHE_DIR_NAME" +fi +install_embedded_runtime_payload "\$WORK/boxlite-runtime.tar.gz" || exit 1 -# Back up the live binary (if any) so a failed swap or start can roll back. -HAD_PREVIOUS=false -if [ -x /usr/local/bin/boxlite-runner ]; then - cp -a /usr/local/bin/boxlite-runner /usr/local/bin/boxlite-runner.bak - HAD_PREVIOUS=true +CURRENT_TARGET=\$(current_runner_target) +NEW_TARGET=\$(prepare_release_target "\$WORK/boxlite-runner" "v${VERSION}") +verify_release_target "\$NEW_TARGET" "new" || exit 1 +echo "install target: \$NEW_TARGET" +if [ -n "\$CURRENT_TARGET" ]; then + verify_release_target "\$CURRENT_TARGET" "rollback" || exit 1 + echo "rollback target: \$CURRENT_TARGET" +else + echo "rollback target: none" fi -systemctl stop boxlite-runner || true + # Swap + start + health as one guarded condition: a failing step here (install error, # start failure, or an unhealthy unit) routes to the rollback branch instead of aborting # the script under set -e — commands in an if-condition are exempt from set -e. -if install -m 0755 "\$WORK/boxlite-runner" /usr/local/bin/boxlite-runner && systemctl start boxlite-runner && sleep 2 && systemctl is-active --quiet boxlite-runner; then - [ "\$HAD_PREVIOUS" = true ] && rm -f /usr/local/bin/boxlite-runner.bak +if restart_with_target "\$NEW_TARGET"; then + prune_old_releases echo "systemd unit: active" - echo "new version:" - /usr/local/bin/boxlite-runner --version else - echo "upgrade failed; rolling back" >&2 - if [ "\$HAD_PREVIOUS" = true ]; then - mv -f /usr/local/bin/boxlite-runner.bak /usr/local/bin/boxlite-runner - systemctl restart boxlite-runner || true + echo "upgrade failed or detached-box adoption failed; rolling back" >&2 + if [ -n "\$CURRENT_TARGET" ]; then + if restart_rollback_target "\$CURRENT_TARGET"; then + echo "rollback complete" + else + echo "rollback failed" >&2 + fi fi - journalctl -u boxlite-runner --no-pager -n 50 || true + journalctl -u "\$SERVICE" --no-pager -n 50 || true exit 1 fi EOF @@ -108,10 +676,28 @@ CMD_ID=$(aws ssm send-command --region "$AWS_REGION" \ --query 'Command.CommandId' --output text) echo " command: $CMD_ID" -echo "==> Waiting for SSM command to finish..." +echo "==> Waiting for SSM command to finish (timeout=${SSM_WAIT_TIMEOUT_SECONDS}s)..." + +STATUS="" +DEADLINE=$((SECONDS + SSM_WAIT_TIMEOUT_SECONDS)) +while true; do + STATUS=$(aws ssm get-command-invocation --region "$AWS_REGION" \ + --command-id "$CMD_ID" --instance-id "$INSTANCE_ID" \ + --query 'Status' --output text 2>/dev/null || true) + + case "$STATUS" in + Success|Failed|Cancelled|TimedOut|Cancelling) + break + ;; + esac + + if (( SECONDS >= DEADLINE )); then + echo "error: SSM command still ${STATUS:-unknown} after ${SSM_WAIT_TIMEOUT_SECONDS}s" >&2 + exit 1 + fi -aws ssm wait command-executed --region "$AWS_REGION" \ - --command-id "$CMD_ID" --instance-id "$INSTANCE_ID" + sleep "$SSM_WAIT_POLL_SECONDS" +done STATUS=$(aws ssm get-command-invocation --region "$AWS_REGION" \ --command-id "$CMD_ID" --instance-id "$INSTANCE_ID" \ diff --git a/scripts/deploy/tests/parity/run.sh b/scripts/deploy/tests/parity/run.sh new file mode 100755 index 000000000..ad2280546 --- /dev/null +++ b/scripts/deploy/tests/parity/run.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Parity test: the JS twins of the runner deploy scripts must issue the same +# semantic command sequence and produce the same artifacts as the bash +# authorities. Run on any host (macOS ok) — every environment-dependent command +# is PATH-shimmed by setup-fixture.sh. +# +# Usage: scripts/deploy/tests/parity/run.sh +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +FIXTURE="$WORK/fixture" + +bash "$HERE/setup-fixture.sh" "$FIXTURE" >/dev/null + +export FIXTURE_ROOT="$FIXTURE" +export PATH="$FIXTURE/shims:$PATH" + +fail=0 +note() { printf '%s\n' "$*"; } +die_soft() { printf 'FAIL: %s\n' "$*"; fail=1; } + +# Normalize per-run randomness. macOS mktemp ignores an inherited TMPDIR for +# bare `mktemp -d`, and node's ESM loader realpaths /var → /private/var, so +# the two implementations legitimately print different-but-equivalent temp +# paths. Collapse them all before diffing. +normalize() { # log + local work_norm="${WORK#/private}" + sed -E \ + -e 's|/private/var|/var|g' \ + -e "s|$work_norm/out\.[a-z]+|OUT|g" \ + -e "s|$work_norm/u?tmp\.[a-z]+|ASSIGNEDTMP|g" \ + -e 's|(ASSIGNEDTMP\|/var/folders/[A-Za-z0-9/._+-]+/T\|/tmp)/tmp\.[A-Za-z0-9._-]+|TMP|g' \ + "$1" +} + +run_build() { # impl(sh|mjs) tag + local impl="$1" tag="$2" + local tmp="$WORK/tmp.$tag" out="$WORK/out.$tag" log="$WORK/calls.$tag.log" + mkdir -p "$tmp" "$out" + : > "$log" + # Reset fixture state the previous run may have consumed/produced. + rm -rf "$FIXTURE/target" "$FIXTURE/sdks/go/libboxlite.a" "$FIXTURE/apps/go.work" "$FIXTURE/apps/go.work.sum" "$FIXTURE/sdks/c/dist" + ( cd "$FIXTURE" \ + && CALL_LOG="$log" TMPDIR="$tmp" \ + "scripts/deploy/build-runner-binary.$impl" --output-dir "$out" --build-number 123 --skip-setup \ + > "$WORK/stdout.$tag.log" 2>&1 ) || { die_soft "build-runner-binary.$impl exited non-zero"; cat "$WORK/stdout.$tag.log"; return 1; } + normalize "$log" > "$log.norm" +} + +run_update() { # impl tag artifact_dir + local impl="$1" tag="$2" artifacts="$3" + local tmp="$WORK/utmp.$tag" log="$WORK/ucalls.$tag.log" + mkdir -p "$tmp" + : > "$log" + ( cd "$FIXTURE" \ + && CALL_LOG="$log" TMPDIR="$tmp" \ + STAGE=dev AWS_REGION=ap-southeast-1 \ + RUNNER_INSTANCE_ID=i-0fixture000000000 \ + RUNNER_ARTIFACT_DIR="$artifacts" \ + RUNNER_ARTIFACT_S3_URI="s3://fixture-bucket/tmp/runner-rollouts/fixed" \ + "scripts/deploy/runner-update-binary.$impl" "$VERSION_UNDER_TEST" \ + > "$WORK/ustdout.$tag.log" 2>&1 ) || { die_soft "runner-update-binary.$impl exited non-zero"; cat "$WORK/ustdout.$tag.log"; return 1; } + normalize "$log" > "$log.norm" +} + +# ── build parity ──────────────────────────────────────────────────────────── +note "==> build: bash authority" +run_build sh bash + +if [ ! -f "$FIXTURE/scripts/deploy/build-runner-binary.mjs" ]; then + die_soft "build-runner-binary.mjs missing — JS twin not implemented yet" +else + note "==> build: js twin" + run_build mjs js + + if ! diff -u "$WORK/calls.bash.log.norm" "$WORK/calls.js.log.norm"; then + die_soft "build call sequence differs (bash vs js)" + fi + + bash_tarball=$(ls "$WORK/out.bash"/boxlite-runner-v*-linux-amd64.tar.gz 2>/dev/null || true) + js_tarball=$(ls "$WORK/out.js"/boxlite-runner-v*-linux-amd64.tar.gz 2>/dev/null || true) + if [ -z "$bash_tarball" ] || [ -z "$js_tarball" ]; then + die_soft "missing output tarball (bash='$bash_tarball' js='$js_tarball')" + else + [ "$(basename "$bash_tarball")" = "$(basename "$js_tarball")" ] \ + || die_soft "tarball names differ: $(basename "$bash_tarball") vs $(basename "$js_tarball")" + if ! diff <(/usr/bin/tar -tzf "$bash_tarball" | sort) <(/usr/bin/tar -tzf "$js_tarball" | sort); then + die_soft "tarball member lists differ" + fi + for member in boxlite-runner.guest.sha256 boxlite-runner.runtime-suffix; do + b=$(/usr/bin/tar -xzOf "$bash_tarball" "$member" 2>/dev/null || /usr/bin/tar -xzOf "$bash_tarball" "./$member") + j=$(/usr/bin/tar -xzOf "$js_tarball" "$member" 2>/dev/null || /usr/bin/tar -xzOf "$js_tarball" "./$member") + [ "$b" = "$j" ] || die_soft "sidecar $member differs: '$b' vs '$j'" + done + fi +fi + +# ── update parity ─────────────────────────────────────────────────────────── +if [ -n "${bash_tarball:-}" ]; then + VERSION_UNDER_TEST=$(basename "$bash_tarball") + VERSION_UNDER_TEST="${VERSION_UNDER_TEST#boxlite-runner-v}" + VERSION_UNDER_TEST="${VERSION_UNDER_TEST%-linux-amd64.tar.gz}" + + note "==> update: bash authority (v$VERSION_UNDER_TEST)" + run_update sh bash "$WORK/out.bash" + + if [ ! -f "$FIXTURE/scripts/deploy/runner-update-binary.mjs" ]; then + die_soft "runner-update-binary.mjs missing — JS twin not implemented yet" + else + note "==> update: js twin" + run_update mjs js "$WORK/out.bash" # same artifact dir: compare orchestration, not build noise + + if ! diff -u "$WORK/ucalls.bash.log.norm" "$WORK/ucalls.js.log.norm"; then + die_soft "update call sequence differs (bash vs js)" + fi + + # The SSM payload is the highest-risk surface: extract the base64 arg from + # the recorded send-command call and byte-compare the decoded scripts. + extract_payload() { + grep '^aws ssm send-command' "$1" | sed -E 's/.*commands=\["echo ([A-Za-z0-9+/=]+) \| base64 -d \| bash"\].*/\1/' + } + pb=$(extract_payload "$WORK/ucalls.bash.log") + pj=$(extract_payload "$WORK/ucalls.js.log") + if [ -z "$pb" ] || [ -z "$pj" ]; then + die_soft "could not extract SSM payload (bash=${#pb}b js=${#pj}b)" + elif [ "$pb" != "$pj" ]; then + printf '%s' "$pb" | base64 -d > "$WORK/payload.bash.sh" || true + printf '%s' "$pj" | base64 -d > "$WORK/payload.js.sh" || true + diff -u "$WORK/payload.bash.sh" "$WORK/payload.js.sh" | head -80 || true + die_soft "SSM remote payload differs between implementations" + else + note " SSM payload byte-identical ($(printf '%s' "$pb" | wc -c | tr -d ' ') b64 chars)" + fi + fi +fi + +if [ "$fail" -ne 0 ]; then + note "PARITY: FAIL" + exit 1 +fi +note "PARITY: PASS" diff --git a/scripts/deploy/tests/parity/setup-fixture.sh b/scripts/deploy/tests/parity/setup-fixture.sh new file mode 100755 index 000000000..cccac468b --- /dev/null +++ b/scripts/deploy/tests/parity/setup-fixture.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# Build a throwaway fixture repo that exercises every filesystem path the +# runner deploy scripts touch, plus a PATH shim dir that records the semantic +# commands (build/publish/SSM) each implementation issues. The parity test +# runs the bash and JS implementations against the same fixture and diffs the +# recorded call logs — text utilities (sed/awk/grep/base64/sha256sum) are NOT +# recorded because the JS twin legitimately replaces them with node builtins. +# +# Usage: setup-fixture.sh +set -euo pipefail + +FIXTURE="${1:?usage: setup-fixture.sh }" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" + +mkdir -p "$FIXTURE"/{apps/runner,apps/common-go,apps/api-client-go,scripts/build,scripts/deploy,sdks/go,target/release,dist} + +cat > "$FIXTURE/Cargo.toml" <<'EOF' +[package] +name = "boxlite" +version = "0.9.7" +EOF + +cat > "$FIXTURE/apps/runner/go.mod" <<'EOF' +module github.com/boxlite-ai/runner + +go 1.24 +EOF + +# The real build scripts under test, copied so ROOT_DIR resolution inside them +# lands on the fixture instead of the actual repo. +cp "$REPO_ROOT/scripts/deploy/build-runner-binary.sh" "$FIXTURE/scripts/deploy/" +if [ -f "$REPO_ROOT/scripts/deploy/build-runner-binary.mjs" ]; then + cp "$REPO_ROOT/scripts/deploy/build-runner-binary.mjs" "$FIXTURE/scripts/deploy/" +fi +cp "$REPO_ROOT/scripts/deploy/runner-update-binary.sh" "$FIXTURE/scripts/deploy/" +if [ -f "$REPO_ROOT/scripts/deploy/runner-update-binary.mjs" ]; then + cp "$REPO_ROOT/scripts/deploy/runner-update-binary.mjs" "$FIXTURE/scripts/deploy/" +fi + +# Product build steps the deploy script shells out to. They log themselves and +# regenerate the inputs the deploy script cleaned beforehand. +cat > "$FIXTURE/scripts/build/build-shim.sh" <<'EOF' +#!/usr/bin/env bash +echo "build-shim.sh $*" >> "$CALL_LOG" +mkdir -p "$FIXTURE_ROOT/target/release" +printf 'shim-binary-v1' > "$FIXTURE_ROOT/target/release/boxlite-shim" +chmod +x "$FIXTURE_ROOT/target/release/boxlite-shim" +EOF + +cat > "$FIXTURE/scripts/build/build-guest.sh" <<'EOF' +#!/usr/bin/env bash +echo "build-guest.sh $*" >> "$CALL_LOG" +mkdir -p "$FIXTURE_ROOT/target/x86_64-unknown-linux-musl/release" +printf 'guest-binary-v1' > "$FIXTURE_ROOT/target/x86_64-unknown-linux-musl/release/boxlite-guest" +chmod +x "$FIXTURE_ROOT/target/x86_64-unknown-linux-musl/release/boxlite-guest" +EOF + +chmod +x "$FIXTURE"/scripts/build/*.sh "$FIXTURE"/scripts/deploy/*.sh +[ -f "$FIXTURE/scripts/deploy/build-runner-binary.mjs" ] && chmod +x "$FIXTURE"/scripts/deploy/*.mjs + +# ── PATH shims ────────────────────────────────────────────────────────────── +SHIMS="$FIXTURE/shims" +mkdir -p "$SHIMS" + +logged_shim() { # name [extra-body...] + local name="$1"; shift + { + echo '#!/usr/bin/env bash' + echo "echo \"$name \$*\" >> \"\$CALL_LOG\"" + printf '%s\n' "$@" + } > "$SHIMS/$name" + chmod +x "$SHIMS/$name" +} + +quiet_shim() { # name body... + local name="$1"; shift + { + echo '#!/usr/bin/env bash' + printf '%s\n' "$@" + } > "$SHIMS/$name" + chmod +x "$SHIMS/$name" +} + +logged_shim cargo ':' +logged_shim make ' +if [ "${1:-}" = "dist:c" ]; then + mkdir -p "$FIXTURE_ROOT/target/release/build/boxlite-fixture123/out/runtime" + cp "$FIXTURE_ROOT/target/x86_64-unknown-linux-musl/release/boxlite-guest" \ + "$FIXTURE_ROOT/target/release/build/boxlite-fixture123/out/runtime/boxlite-guest" + printf 'lib-v1' > "$FIXTURE_ROOT/target/release/libboxlite.a" +fi' +logged_shim go ' +prev="" +out="" +for a in "$@"; do + [ "$prev" = "-o" ] && out="$a" + prev="$a" +done +if [ -n "$out" ]; then + printf 'runner-binary-v1' > "$out" + chmod +x "$out" +fi' +logged_shim git ' +if [ "$1 $2" = "rev-parse --short" ]; then echo abc1234; fi' +logged_shim tar 'exec /usr/bin/tar "$@"' +logged_shim curl ':' +logged_shim aws ' +query="" +s3uri="" +prev="" +for a in "$@"; do + [ "$prev" = "--query" ] && query="$a" + case "$a" in s3://*) s3uri="$a" ;; esac + prev="$a" +done +case "$1 ${2:-}" in + "s3 presign") echo "https://presigned.example/${s3uri#s3://}" ;; + "s3 cp") : ;; + "sts get-caller-identity") echo 111122223333 ;; + "ssm send-command") echo cmd-fixture-0001 ;; + "ssm get-command-invocation") + case "$query" in + Status) echo Success ;; + StandardOutputContent) echo remote-ok ;; + StandardErrorContent) echo "" ;; + *) echo Success ;; + esac + ;; +esac' + +quiet_shim uname 'case "${1:-}" in -s) echo Linux ;; -m) echo x86_64 ;; *) echo Linux ;; esac' +quiet_shim sha256sum 'exec /usr/bin/shasum -a 256 "$@"' +quiet_shim sleep ':' + +echo "fixture ready: $FIXTURE"