fix(deploy): count a dropped connection as an absent OpenPlanner

.github/workflows/code-quality.yml

@@ -29,6 +29,28 @@ jobs:
           bash -n digitalocean/lib/*.sh digitalocean/scripts/*.sh
           bash -n digitalocean/services/*/verify.sh
 
+      # Helper scripts a verify.sh evaluates inside a container. They only ever
+      # run on the production host, so a syntax error in one would surface as a
+      # failed health gate mid-deploy rather than here.
+      - name: Check service helper scripts
+        run: |
+          set -euo pipefail
+          shopt -s nullglob
+          for js in digitalocean/services/*/*.js; do
+            node --check "$js"
+            echo "node --check ok $js"
+          done
+
+      # The Knoxx gate has to tell an OpenPlanner that was never deployed from
+      # one that is deployed and broken, and it gets that wrong in a way nothing
+      # else catches: too strict and every deploy fails (run 30758885732), too
+      # loose and a crashed upstream ships green. The probe carries its own
+      # classifier matrix; run it against the same file the gate executes.
+      - name: Self-test the OpenPlanner reachability classifier
+        env:
+          PROBE_SELFTEST: "1"
+        run: node digitalocean/services/knoxx/probe-openplanner.js
+
       # A remote script piped to `bash -s` over ssh is parsed lazily from the
       # same stdin the script's own commands inherit. One `docker compose exec
       # -T` in a verify.sh was enough to swallow a health gate's failure

digitalocean/services/knoxx/env.template

@@ -39,11 +39,14 @@ OPENPLANNER_API_KEY='${OPENPLANNER_API_KEY}'
 
 # Whether a host OpenPlanner HTTP service is expected on this host.
 #
-# A refused connection cannot distinguish "deliberately not deployed" from
+# A failed connection cannot distinguish "deliberately not deployed" from
 # "deployed and crashed", so the health gate is told which one to assume rather
 # than guessing. deploy-stack.yml supplies false because it does not deploy
 # OpenPlanner. Other callers can set this to true through extra_env, and a
-# refused connection then fails the deployment.
+# failed connection then fails the deployment.
+#
+# Note that on this host ufw denies incoming, so an absent listener times out
+# rather than refusing; verify.sh classifies both as absent.
 KNOXX_EXPECT_OPENPLANNER_REST='${KNOXX_EXPECT_OPENPLANNER_REST}'
 
 PROXY_AUTH_TOKEN='${PROXY_AUTH_TOKEN}'

digitalocean/services/knoxx/probe-openplanner.js

@@ -0,0 +1,136 @@
+'use strict';
+
+// Probe the host OpenPlanner HTTP API from inside the Knoxx backend container.
+//
+// Delivered to the container by verify.sh as `node -e "$(cat ...)"`, so it runs
+// where the container's own network view applies while staying a real file here
+// — `node --check`-able, reviewable, and self-testable (see PROBE_SELFTEST at
+// the bottom, which .github/workflows/code-quality.yml runs on every PR).
+//
+// It answers one question the health gate cannot answer any other way: is a
+// host OpenPlanner deliberately absent, or deployed and broken? Both look like
+// 502/503/504 from Knoxx's CMS routes, so the upstream has to be probed
+// directly.
+//
+// Absence is decided at the connect phase and nowhere else. That distinction is
+// the whole point, so the two phases are separated explicitly rather than
+// inferred from a single fetch's error code:
+//
+//   * A bare TCP connect is attempted first, with its own timeout. Failing to
+//     establish a connection is the only thing that can mean "absent".
+//   * The HTTP request runs only after a connection has demonstrably been
+//     established. Anything that goes wrong from there — including a timeout —
+//     is a deployed service failing, and must fail the gate.
+//
+// Deriving this from one fetch does not work. AbortSignal.timeout aborts the
+// whole request, so a dropped connect and a hung response can surface as the
+// same TimeoutError depending only on which timer wins, and undici's own
+// 10-second connect timeout is not configurable through global fetch.
+
+const net = require('node:net');
+
+// What a failure to establish a connection can look like.
+//
+// ECONNREFUSED is what a closed port answers on an unfiltered host. This host is
+// not unfiltered: bootstrap-host.sh runs 'ufw default deny incoming', traffic
+// from the bridge network to host-gateway traverses INPUT, and ufw DROPs it, so
+// the attempt times out instead. Treating only ECONNREFUSED as absent failed
+// every Knoxx deploy on this host (run 30758885732, 2026-08-02), which in turn
+// skipped deploy-caddy and silently froze the ingress configuration.
+//
+// CONNECT_TIMEOUT is this script's own socket timeout; ETIMEDOUT is the kernel
+// giving up on the handshake first. Both mean the same thing.
+const ABSENT_CONNECT_CODES = new Set(['ECONNREFUSED', 'CONNECT_TIMEOUT', 'ETIMEDOUT']);
+
+// Everything else is an infrastructure failure rather than intentional absence:
+// ENOTFOUND means the host.docker.internal mapping did not apply, and
+// EHOSTUNREACH/ENETUNREACH mean the route is broken.
+function classifyConnectFailure(code) {
+  return {reachable: false, phase: 'connect', code, absent: ABSENT_CONNECT_CODES.has(code)};
+}
+
+// Reached only after a connection was established, so never absent.
+function classifyResponseFailure(code, error) {
+  return {reachable: false, phase: 'response', code, absent: false, error};
+}
+
+function targetOf(baseUrl) {
+  const url = new URL(baseUrl);
+  return {
+    hostname: url.hostname,
+    port: Number(url.port) || (url.protocol === 'https:' ? 443 : 80),
+  };
+}
+
+function connect({hostname, port}, timeoutMs) {
+  return new Promise((resolve) => {
+    const socket = net.connect({host: hostname, port});
+    let settled = false;
+    const settle = (result) => {
+      if (settled) return;
+      settled = true;
+      socket.destroy();
+      resolve(result);
+    };
+    socket.setTimeout(timeoutMs);
+    socket.once('connect', () => settle({connected: true}));
+    socket.once('timeout', () => settle({connected: false, code: 'CONNECT_TIMEOUT'}));
+    socket.once('error', (e) => settle({connected: false, code: e.code || e.name || 'unknown'}));
+  });
+}
+
+async function probe(baseUrl, timeoutMs) {
+  let target;
+  try {
+    target = targetOf(baseUrl);
+  } catch (e) {
+    // An unparseable URL is a configuration error, not an absent service.
+    return classifyResponseFailure('ERR_INVALID_URL', String(e));
+  }
+
+  const attempt = await connect(target, timeoutMs);
+  if (!attempt.connected) return classifyConnectFailure(attempt.code);
+
+  try {
+    const r = await fetch(baseUrl.replace(/\/+$/, '') + '/v1/health', {
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+    return {reachable: true, phase: 'response', status: r.status};
+  } catch (e) {
+    const code = (e && e.cause && e.cause.code) || (e && e.name) || 'unknown';
+    return classifyResponseFailure(code, String(e));
+  }
+}
+
+// ── self-test ────────────────────────────────────────────────
+// Runs the classifier matrix with no network. Kept in this file so the code the
+// gate executes and the code CI asserts on cannot drift apart.
+if (process.env.PROBE_SELFTEST === '1') {
+  const assert = require('node:assert/strict');
+
+  for (const code of ['ECONNREFUSED', 'CONNECT_TIMEOUT', 'ETIMEDOUT']) {
+    assert.equal(classifyConnectFailure(code).absent, true, `${code} must count as absent`);
+  }
+  for (const code of ['ENOTFOUND', 'EHOSTUNREACH', 'ENETUNREACH', 'unknown']) {
+    assert.equal(classifyConnectFailure(code).absent, false, `${code} must fail the gate`);
+  }
+  // A timeout after the connection was established is a hung deployed service.
+  for (const code of ['TimeoutError', 'AbortError', 'UND_ERR_SOCKET', 'ECONNRESET']) {
+    assert.equal(classifyResponseFailure(code).absent, false,
+      `${code} in the response phase must never be absent`);
+  }
+  assert.equal(classifyConnectFailure('ECONNREFUSED').reachable, false);
+  assert.equal(classifyResponseFailure('TimeoutError').phase, 'response');
+  assert.deepEqual(targetOf('http://host.docker.internal:7777'),
+    {hostname: 'host.docker.internal', port: 7777});
+  assert.equal(targetOf('http://host.docker.internal').port, 80);
+  assert.equal(targetOf('https://openplanner.example').port, 443);
+
+  process.stdout.write('probe-openplanner: classifier matrix ok\n');
+} else {
+  const timeoutMs = Number(process.env.BACKEND_PROBE_TIMEOUT_MS) || 15000;
+  const baseUrl = process.env.OPENPLANNER_BASE_URL || '';
+  probe(baseUrl, timeoutMs).then((result) => {
+    process.stdout.write(JSON.stringify(result));
+  });
+}

digitalocean/services/knoxx/verify.sh

@@ -103,43 +103,33 @@ openplanner_base=$(docker compose --project-name knoxx --env-file .env \
 if [ -z "$openplanner_base" ]; then
   echo "knoxx: CMS surface skipped — OPENPLANNER_BASE_URL is unset" >&2
 else
+  # probe-openplanner.js ships beside this script and is read here on the host,
+  # then evaluated inside the container so the container's network view applies.
+  # Keeping it a real file rather than an inline string is what lets CI run
+  # `node --check` and its classifier self-test against the same source the gate
+  # executes. Its contract: `absent` is true only for a failure to establish a
+  # TCP connection, so a hung deployed service can never be mistaken for one
+  # that was never deployed.
   upstream=$(docker compose --project-name knoxx --env-file .env \
     exec -T -e BACKEND_PROBE_TIMEOUT_MS="$BACKEND_PROBE_TIMEOUT_MS" \
-    knoxx-backend node -e "
-      const ms = Number(process.env.BACKEND_PROBE_TIMEOUT_MS) || 15000;
-      const base = (process.env.OPENPLANNER_BASE_URL || '').replace(/\/+\$/, '');
-      // Only an undeployed listener counts as absent. The topology is fixed —
-      // host.docker.internal is mapped to host-gateway in compose — so nothing
-      // deployed produces ECONNREFUSED, while ENOTFOUND means that mapping did
-      // not apply and EHOSTUNREACH/ENETUNREACH/TimeoutError mean the route or
-      // the service is broken. Those are infrastructure failures, not an
-      // intentionally absent OpenPlanner, and must fail the gate.
-      const ABSENT = new Set(['ECONNREFUSED']);
-      fetch(base + '/v1/health', {signal: AbortSignal.timeout(ms)})
-        .then(r => { process.stdout.write(JSON.stringify({reachable: true, status: r.status})); })
-        .catch(e => {
-          const code = e && e.cause && e.cause.code;
-          const absent = ABSENT.has(code);
-          process.stdout.write(JSON.stringify({
-            reachable: false, absent, code: code || e.name || 'unknown', error: String(e),
-          }));
-        });
-    " </dev/null)
+    knoxx-backend node -e "$(cat ./probe-openplanner.js)" </dev/null)
   upstream_reachable=$(printf '%s' "$upstream" | jq -r '.reachable // false')
 
   upstream_absent=$(printf '%s' "$upstream" | jq -r '.absent // false')
   upstream_code=$(printf '%s' "$upstream" | jq -r '.code // "unknown"')
+  upstream_phase=$(printf '%s' "$upstream" | jq -r '.phase // "unknown"')
 
   if [ "$upstream_reachable" != "true" ] && [ "$upstream_absent" = "true" ] \
      && [ "${KNOXX_EXPECT_OPENPLANNER_REST:-false}" != "true" ]; then
     # Nothing listening, and this host does not expect anything to be. REST-only
     # compatibility operations stay degraded until OpenPlanner exists.
-    echo "knoxx: CMS surface skipped — no host OpenPlanner API at ${openplanner_base} (${upstream_code}), and KNOXX_EXPECT_OPENPLANNER_REST is not true" >&2
+    echo "knoxx: CMS surface skipped — no host OpenPlanner API at ${openplanner_base} (${upstream_phase}/${upstream_code}), and KNOXX_EXPECT_OPENPLANNER_REST is not true" >&2
   elif [ "$upstream_reachable" != "true" ]; then
-    # Either the host expects OpenPlanner and it is refusing connections — a
-    # crashed or stopped process — or it answered in a way that is not usable.
+    # Either the host expects OpenPlanner and nothing is accepting connections —
+    # a crashed or stopped process — or a connection was established and the
+    # service then failed to answer, which the probe reports as phase=response.
     # Both are failures rather than intentional absence.
-    echo "knoxx: host OpenPlanner API at ${openplanner_base} did not answer (${upstream_code}); expected=${KNOXX_EXPECT_OPENPLANNER_REST:-false}" >&2
+    echo "knoxx: host OpenPlanner API at ${openplanner_base} did not answer (${upstream_phase}/${upstream_code}); expected=${KNOXX_EXPECT_OPENPLANNER_REST:-false}" >&2
     printf '%s\n' "$upstream" >&2
     exit 1
   else