diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index fc8f840..aef7f5b 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -29,6 +29,28 @@ jobs: bash -n digitalocean/lib/*.sh digitalocean/scripts/*.sh bash -n digitalocean/services/*/verify.sh + # Helper scripts a verify.sh evaluates inside a container. They only ever + # run on the production host, so a syntax error in one would surface as a + # failed health gate mid-deploy rather than here. + - name: Check service helper scripts + run: | + set -euo pipefail + shopt -s nullglob + for js in digitalocean/services/*/*.js; do + node --check "$js" + echo "node --check ok $js" + done + + # The Knoxx gate has to tell an OpenPlanner that was never deployed from + # one that is deployed and broken, and it gets that wrong in a way nothing + # else catches: too strict and every deploy fails (run 30758885732), too + # loose and a crashed upstream ships green. The probe carries its own + # classifier matrix; run it against the same file the gate executes. + - name: Self-test the OpenPlanner reachability classifier + env: + PROBE_SELFTEST: "1" + run: node digitalocean/services/knoxx/probe-openplanner.js + # A remote script piped to `bash -s` over ssh is parsed lazily from the # same stdin the script's own commands inherit. One `docker compose exec # -T` in a verify.sh was enough to swallow a health gate's failure diff --git a/digitalocean/services/knoxx/env.template b/digitalocean/services/knoxx/env.template index 34616a7..5228cc0 100644 --- a/digitalocean/services/knoxx/env.template +++ b/digitalocean/services/knoxx/env.template @@ -39,11 +39,14 @@ OPENPLANNER_API_KEY='${OPENPLANNER_API_KEY}' # Whether a host OpenPlanner HTTP service is expected on this host. # -# A refused connection cannot distinguish "deliberately not deployed" from +# A failed connection cannot distinguish "deliberately not deployed" from # "deployed and crashed", so the health gate is told which one to assume rather # than guessing. deploy-stack.yml supplies false because it does not deploy # OpenPlanner. Other callers can set this to true through extra_env, and a -# refused connection then fails the deployment. +# failed connection then fails the deployment. +# +# Note that on this host ufw denies incoming, so an absent listener times out +# rather than refusing; verify.sh classifies both as absent. KNOXX_EXPECT_OPENPLANNER_REST='${KNOXX_EXPECT_OPENPLANNER_REST}' PROXY_AUTH_TOKEN='${PROXY_AUTH_TOKEN}' diff --git a/digitalocean/services/knoxx/probe-openplanner.js b/digitalocean/services/knoxx/probe-openplanner.js new file mode 100644 index 0000000..c9cbf58 --- /dev/null +++ b/digitalocean/services/knoxx/probe-openplanner.js @@ -0,0 +1,136 @@ +'use strict'; + +// Probe the host OpenPlanner HTTP API from inside the Knoxx backend container. +// +// Delivered to the container by verify.sh as `node -e "$(cat ...)"`, so it runs +// where the container's own network view applies while staying a real file here +// — `node --check`-able, reviewable, and self-testable (see PROBE_SELFTEST at +// the bottom, which .github/workflows/code-quality.yml runs on every PR). +// +// It answers one question the health gate cannot answer any other way: is a +// host OpenPlanner deliberately absent, or deployed and broken? Both look like +// 502/503/504 from Knoxx's CMS routes, so the upstream has to be probed +// directly. +// +// Absence is decided at the connect phase and nowhere else. That distinction is +// the whole point, so the two phases are separated explicitly rather than +// inferred from a single fetch's error code: +// +// * A bare TCP connect is attempted first, with its own timeout. Failing to +// establish a connection is the only thing that can mean "absent". +// * The HTTP request runs only after a connection has demonstrably been +// established. Anything that goes wrong from there — including a timeout — +// is a deployed service failing, and must fail the gate. +// +// Deriving this from one fetch does not work. AbortSignal.timeout aborts the +// whole request, so a dropped connect and a hung response can surface as the +// same TimeoutError depending only on which timer wins, and undici's own +// 10-second connect timeout is not configurable through global fetch. + +const net = require('node:net'); + +// What a failure to establish a connection can look like. +// +// ECONNREFUSED is what a closed port answers on an unfiltered host. This host is +// not unfiltered: bootstrap-host.sh runs 'ufw default deny incoming', traffic +// from the bridge network to host-gateway traverses INPUT, and ufw DROPs it, so +// the attempt times out instead. Treating only ECONNREFUSED as absent failed +// every Knoxx deploy on this host (run 30758885732, 2026-08-02), which in turn +// skipped deploy-caddy and silently froze the ingress configuration. +// +// CONNECT_TIMEOUT is this script's own socket timeout; ETIMEDOUT is the kernel +// giving up on the handshake first. Both mean the same thing. +const ABSENT_CONNECT_CODES = new Set(['ECONNREFUSED', 'CONNECT_TIMEOUT', 'ETIMEDOUT']); + +// Everything else is an infrastructure failure rather than intentional absence: +// ENOTFOUND means the host.docker.internal mapping did not apply, and +// EHOSTUNREACH/ENETUNREACH mean the route is broken. +function classifyConnectFailure(code) { + return {reachable: false, phase: 'connect', code, absent: ABSENT_CONNECT_CODES.has(code)}; +} + +// Reached only after a connection was established, so never absent. +function classifyResponseFailure(code, error) { + return {reachable: false, phase: 'response', code, absent: false, error}; +} + +function targetOf(baseUrl) { + const url = new URL(baseUrl); + return { + hostname: url.hostname, + port: Number(url.port) || (url.protocol === 'https:' ? 443 : 80), + }; +} + +function connect({hostname, port}, timeoutMs) { + return new Promise((resolve) => { + const socket = net.connect({host: hostname, port}); + let settled = false; + const settle = (result) => { + if (settled) return; + settled = true; + socket.destroy(); + resolve(result); + }; + socket.setTimeout(timeoutMs); + socket.once('connect', () => settle({connected: true})); + socket.once('timeout', () => settle({connected: false, code: 'CONNECT_TIMEOUT'})); + socket.once('error', (e) => settle({connected: false, code: e.code || e.name || 'unknown'})); + }); +} + +async function probe(baseUrl, timeoutMs) { + let target; + try { + target = targetOf(baseUrl); + } catch (e) { + // An unparseable URL is a configuration error, not an absent service. + return classifyResponseFailure('ERR_INVALID_URL', String(e)); + } + + const attempt = await connect(target, timeoutMs); + if (!attempt.connected) return classifyConnectFailure(attempt.code); + + try { + const r = await fetch(baseUrl.replace(/\/+$/, '') + '/v1/health', { + signal: AbortSignal.timeout(timeoutMs), + }); + return {reachable: true, phase: 'response', status: r.status}; + } catch (e) { + const code = (e && e.cause && e.cause.code) || (e && e.name) || 'unknown'; + return classifyResponseFailure(code, String(e)); + } +} + +// ── self-test ──────────────────────────────────────────────── +// Runs the classifier matrix with no network. Kept in this file so the code the +// gate executes and the code CI asserts on cannot drift apart. +if (process.env.PROBE_SELFTEST === '1') { + const assert = require('node:assert/strict'); + + for (const code of ['ECONNREFUSED', 'CONNECT_TIMEOUT', 'ETIMEDOUT']) { + assert.equal(classifyConnectFailure(code).absent, true, `${code} must count as absent`); + } + for (const code of ['ENOTFOUND', 'EHOSTUNREACH', 'ENETUNREACH', 'unknown']) { + assert.equal(classifyConnectFailure(code).absent, false, `${code} must fail the gate`); + } + // A timeout after the connection was established is a hung deployed service. + for (const code of ['TimeoutError', 'AbortError', 'UND_ERR_SOCKET', 'ECONNRESET']) { + assert.equal(classifyResponseFailure(code).absent, false, + `${code} in the response phase must never be absent`); + } + assert.equal(classifyConnectFailure('ECONNREFUSED').reachable, false); + assert.equal(classifyResponseFailure('TimeoutError').phase, 'response'); + assert.deepEqual(targetOf('http://host.docker.internal:7777'), + {hostname: 'host.docker.internal', port: 7777}); + assert.equal(targetOf('http://host.docker.internal').port, 80); + assert.equal(targetOf('https://openplanner.example').port, 443); + + process.stdout.write('probe-openplanner: classifier matrix ok\n'); +} else { + const timeoutMs = Number(process.env.BACKEND_PROBE_TIMEOUT_MS) || 15000; + const baseUrl = process.env.OPENPLANNER_BASE_URL || ''; + probe(baseUrl, timeoutMs).then((result) => { + process.stdout.write(JSON.stringify(result)); + }); +} diff --git a/digitalocean/services/knoxx/verify.sh b/digitalocean/services/knoxx/verify.sh index 3b11f4a..eccefe5 100755 --- a/digitalocean/services/knoxx/verify.sh +++ b/digitalocean/services/knoxx/verify.sh @@ -103,43 +103,33 @@ openplanner_base=$(docker compose --project-name knoxx --env-file .env \ if [ -z "$openplanner_base" ]; then echo "knoxx: CMS surface skipped — OPENPLANNER_BASE_URL is unset" >&2 else + # probe-openplanner.js ships beside this script and is read here on the host, + # then evaluated inside the container so the container's network view applies. + # Keeping it a real file rather than an inline string is what lets CI run + # `node --check` and its classifier self-test against the same source the gate + # executes. Its contract: `absent` is true only for a failure to establish a + # TCP connection, so a hung deployed service can never be mistaken for one + # that was never deployed. upstream=$(docker compose --project-name knoxx --env-file .env \ exec -T -e BACKEND_PROBE_TIMEOUT_MS="$BACKEND_PROBE_TIMEOUT_MS" \ - knoxx-backend node -e " - const ms = Number(process.env.BACKEND_PROBE_TIMEOUT_MS) || 15000; - const base = (process.env.OPENPLANNER_BASE_URL || '').replace(/\/+\$/, ''); - // Only an undeployed listener counts as absent. The topology is fixed — - // host.docker.internal is mapped to host-gateway in compose — so nothing - // deployed produces ECONNREFUSED, while ENOTFOUND means that mapping did - // not apply and EHOSTUNREACH/ENETUNREACH/TimeoutError mean the route or - // the service is broken. Those are infrastructure failures, not an - // intentionally absent OpenPlanner, and must fail the gate. - const ABSENT = new Set(['ECONNREFUSED']); - fetch(base + '/v1/health', {signal: AbortSignal.timeout(ms)}) - .then(r => { process.stdout.write(JSON.stringify({reachable: true, status: r.status})); }) - .catch(e => { - const code = e && e.cause && e.cause.code; - const absent = ABSENT.has(code); - process.stdout.write(JSON.stringify({ - reachable: false, absent, code: code || e.name || 'unknown', error: String(e), - })); - }); - " &2 + echo "knoxx: CMS surface skipped — no host OpenPlanner API at ${openplanner_base} (${upstream_phase}/${upstream_code}), and KNOXX_EXPECT_OPENPLANNER_REST is not true" >&2 elif [ "$upstream_reachable" != "true" ]; then - # Either the host expects OpenPlanner and it is refusing connections — a - # crashed or stopped process — or it answered in a way that is not usable. + # Either the host expects OpenPlanner and nothing is accepting connections — + # a crashed or stopped process — or a connection was established and the + # service then failed to answer, which the probe reports as phase=response. # Both are failures rather than intentional absence. - echo "knoxx: host OpenPlanner API at ${openplanner_base} did not answer (${upstream_code}); expected=${KNOXX_EXPECT_OPENPLANNER_REST:-false}" >&2 + echo "knoxx: host OpenPlanner API at ${openplanner_base} did not answer (${upstream_phase}/${upstream_code}); expected=${KNOXX_EXPECT_OPENPLANNER_REST:-false}" >&2 printf '%s\n' "$upstream" >&2 exit 1 else