JSONbored · JSONbored · Jul 5, 2026 · Jul 5, 2026
@@ -247,7 +247,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "ORB_BROKER_URL",
-    firstReference: "src/server.ts:968",
+    firstReference: "src/server.ts:974",
   },
   {
     name: "ORB_COLLECTOR_TOKEN",
@@ -263,7 +263,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "ORB_RELAY_MODE",
-    firstReference: "src/server.ts:970",
+    firstReference: "src/server.ts:962",
   },
   {
     name: "OTEL_EXPORTER_OTLP_ENDPOINT",
@@ -450,11 +450,11 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `ORB_AIR_GAP` | `src/selfhost/orb-collector.ts:161` |",
   "| `ORB_ANONYMIZE` | `src/selfhost/orb-collector.ts:174` |",
   "| `ORB_APP_ID` | `src/selfhost/orb-collector.ts:59` |",
-  "| `ORB_BROKER_URL` | `src/server.ts:968` |",
+  "| `ORB_BROKER_URL` | `src/server.ts:974` |",
   "| `ORB_COLLECTOR_TOKEN` | `src/selfhost/orb-collector.ts:205` |",
   "| `ORB_COLLECTOR_URL` | `src/selfhost/orb-collector.ts:172` |",
   "| `ORB_ENROLLMENT_SECRET` | `src/selfhost/orb-collector.ts:165` |",
-  "| `ORB_RELAY_MODE` | `src/server.ts:970` |",
+  "| `ORB_RELAY_MODE` | `src/server.ts:962` |",
   "| `OTEL_EXPORTER_OTLP_ENDPOINT` | `src/selfhost/otel.ts:47` |",
   "| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | `src/selfhost/otel.ts:45` |",
   "| `OTEL_SERVICE_ENVIRONMENT` | `src/selfhost/otel.ts:60` |",

@@ -2637,9 +2637,47 @@
         }
       ]
     },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "custom": { "lineWidth": 2, "fillOpacity": 10 },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "seconds since last pull-mode drain" },
+            "properties": [{ "id": "unit", "value": "s" }]
+          }
+        ]
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 170 },
+      "id": 158,
+      "options": {
+        "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "title": "Orb Relay Registration: Streak vs Drain Progress (one hiccup vs actually stuck)",
+      "type": "timeseries",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "gittensory_orb_relay_register_consecutive_failures or vector(0)",
+          "legendFormat": "consecutive registration failures",
+          "refId": "A"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "gittensory_orb_relay_drain_seconds_since_last or vector(0)",
+          "legendFormat": "seconds since last pull-mode drain",
+          "refId": "B"
+        }
+      ]
+    },
     {
       "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 171 },
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 179 },
       "id": 145,
       "panels": [],
       "title": "Foreground Liveness (#selfhost-queue-liveness)",
@@ -2648,7 +2686,7 @@
     {
       "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
       "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } },
-      "gridPos": { "h": 4, "w": 4, "x": 0, "y": 172 },
+      "gridPos": { "h": 4, "w": 4, "x": 0, "y": 180 },
       "id": 146,
       "options": {
         "colorMode": "background",
@@ -2671,7 +2709,7 @@
     {
       "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
       "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } },
-      "gridPos": { "h": 4, "w": 4, "x": 4, "y": 172 },
+      "gridPos": { "h": 4, "w": 4, "x": 4, "y": 180 },
       "id": 147,
       "options": {
         "colorMode": "background",
@@ -2694,7 +2732,7 @@
     {
       "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
       "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } },
-      "gridPos": { "h": 4, "w": 4, "x": 8, "y": 172 },
+      "gridPos": { "h": 4, "w": 4, "x": 8, "y": 180 },
       "id": 148,
       "options": {
         "colorMode": "background",
@@ -2730,7 +2768,7 @@
           "unit": "s"
         }
       },
-      "gridPos": { "h": 4, "w": 4, "x": 12, "y": 172 },
+      "gridPos": { "h": 4, "w": 4, "x": 12, "y": 180 },
       "id": 149,
       "options": {
         "colorMode": "background",
@@ -2753,7 +2791,7 @@
     {
       "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
       "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } },
-      "gridPos": { "h": 4, "w": 4, "x": 16, "y": 172 },
+      "gridPos": { "h": 4, "w": 4, "x": 16, "y": 180 },
       "id": 150,
       "options": {
         "colorMode": "background",
@@ -2778,7 +2816,7 @@
       "fieldConfig": {
         "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "short" }
       },
-      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 176 },
+      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 184 },
       "id": 151,
       "options": {
         "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" },
@@ -2797,7 +2835,7 @@
     },
     {
       "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 182 },
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 190 },
       "id": 152,
       "title": "Backlog-vs-Fresh-Intake Lane Fairness (#selfhost-lane-observability)",
       "type": "row"
@@ -2818,7 +2856,7 @@
           "unit": "short"
         }
       },
-      "gridPos": { "h": 4, "w": 4, "x": 0, "y": 183 },
+      "gridPos": { "h": 4, "w": 4, "x": 0, "y": 191 },
       "id": 153,
       "options": {
         "colorMode": "background",
@@ -2854,7 +2892,7 @@
           "unit": "short"
         }
       },
-      "gridPos": { "h": 4, "w": 4, "x": 4, "y": 183 },
+      "gridPos": { "h": 4, "w": 4, "x": 4, "y": 191 },
       "id": 154,
       "options": {
         "colorMode": "background",
@@ -2879,7 +2917,7 @@
       "fieldConfig": {
         "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "short" }
       },
-      "gridPos": { "h": 6, "w": 16, "x": 8, "y": 183 },
+      "gridPos": { "h": 6, "w": 16, "x": 8, "y": 191 },
       "id": 155,
       "options": {
         "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" },
@@ -2902,7 +2940,7 @@
       "fieldConfig": {
         "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "ops" }
       },
-      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 189 },
+      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 197 },
       "id": 156,
       "options": {
         "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" },
@@ -2922,7 +2960,7 @@
     {
       "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
       "fieldConfig": { "defaults": { "unit": "short" } },
-      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 189 },
+      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 197 },
       "id": 157,
       "options": {
         "showHeader": true,

@@ -389,6 +389,20 @@ groups:
           description: "Webhook enqueue failures are occurring for over 2m. Labels event/action show which GitHub event class is failing."
           runbook: "Check WEBHOOKS queue availability and the durable queue backend. If this coincides with Orb relay drains, inspect gittensory_orb_webhook_total{result=\"enqueue_failed\"}."
 
+      - alert: GittensoryOrbRelayRegistrationStuck
+        # #selfhost-runtime-drift follow-up: mirrors isOrbRelayRegistrationAlerting's own gate (3 consecutive
+        # registration failures, OR the pull-mode drain loop gone quiet for 30m) so Prometheus escalates on
+        # EXACTLY the same evidence the app itself uses to decide "actually stuck" vs "one hiccup, still
+        # draining fine". A single transient registration timeout will not trip this on its own.
+        expr: gittensory_orb_relay_register_consecutive_failures >= 3 or gittensory_orb_relay_drain_seconds_since_last > 1800
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "gittensory orb relay registration looks actually stuck, not just a transient hiccup"
+          description: "Either the consecutive registration-failure streak has reached {{ $value | printf \"%.0f\" }}, or the pull-mode drain loop hasn't completed in over 30m. A lone registration timeout alone would not trigger this."
+          runbook: "Check gittensory_orb_relay_register_total{result=\"failed\"} by mode for the failure pattern, and confirm ORB_BROKER_URL / ORB_ENROLLMENT_SECRET are still valid. If pull mode, verify the drain loop itself isn't crash-looping (selfhost_orb_relay_register_failed logs at level=error)."
+
   # ── HTTP serving health (uses the PLANNED status label + duration histogram) ─
   # NOTE: gittensory_http_requests_total is gaining a status="2xx|3xx|4xx|5xx" label,
   # and gittensory_http_request_duration_seconds (a histogram) is being added. Both

@@ -174,12 +174,23 @@ export async function registerOrbRelayTarget(
   }
 }
 
-export type OrbRelayRegistrationState = { registered: boolean; lastAttemptAtMs: number | null; attempts: number };
+// `attempts` (below) is a lifetime total, never reset -- it answers "did this recover after prior
+// failures" (see registerOrbRelayWithMonitor) but can't tell "is it CURRENTLY stuck" from "it hiccuped
+// once years ago". `consecutiveFailures` is the complementary streak: it resets to 0 on any success, so
+// only a SUSTAINED run of back-to-back failures grows it -- the same shape as ai.ts's
+// `aiConsecutiveFailures` / `AI_UNHEALTHY_FAILURE_STREAK`.
+export type OrbRelayRegistrationState = { registered: boolean; lastAttemptAtMs: number | null; attempts: number; consecutiveFailures: number };
 
 export function createOrbRelayRegistrationState(): OrbRelayRegistrationState {
-  return { registered: false, lastAttemptAtMs: null, attempts: 0 };
+  return { registered: false, lastAttemptAtMs: null, attempts: 0, consecutiveFailures: 0 };
 }
 
+// Mirrors AI_UNHEALTHY_FAILURE_STREAK's shape (src/selfhost/ai.ts): one bad registration attempt is
+// routine (the broker had a slow tick, a deploy in flight, a momentary network blip) and must not alert
+// on its own -- only a SUSTAINED run of consecutive failures indicates the broker link is actually
+// stuck rather than just having hiccuped once (#selfhost-runtime-drift follow-up).
+export const ORB_RELAY_REGISTER_UNHEALTHY_FAILURE_STREAK = 3;
+
 // Mirrors RELAY_RETRY_BACKOFF_MINUTES (src/orb/relay.ts) for the same reason: a sustained broker outage must not
 // re-attempt registration on every ~1min tick -- fleet-wide, that is a synchronized retry storm against a
 // central Orb that is already degraded.
@@ -206,11 +217,13 @@ export async function registerOrbRelayTargetWithRetry(
   state.lastAttemptAtMs = nowMs;
   state.attempts += 1;
   const result = await registerOrbRelayTarget(env, fetchImpl);
-  if (result.status === "skipped") return { status: "skipped" };
+  if (result.status === "skipped") return { status: "skipped" }; // intentional no-op, not a broker failure -- leaves the streak untouched
   if (result.status === "registered") {
     state.registered = true;
+    state.consecutiveFailures = 0;
     return { status: "registered" };
   }
+  state.consecutiveFailures += 1;
   /* v8 ignore next -- registerOrbRelayTarget's own "failed" returns always set a string reason (http_NNN or an
    * error message); the undefined arm only satisfies exactOptionalPropertyTypes for the shared result shape. */
   return result.reason !== undefined ? { status: "failed", reason: result.reason } : { status: "failed" };

@@ -97,6 +97,8 @@ const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [
   ["gittensory_orb_events_exported_total", { help: "Orb events exported from the self-host runtime.", type: "counter" }],
   ["gittensory_orb_export_errors_total", { help: "Orb event export errors.", type: "counter" }],
   ["gittensory_orb_relay_drains_total", { help: "Orb relay drain outcomes.", type: "counter" }],
+  ["gittensory_orb_relay_register_consecutive_failures", { help: "Current consecutive orb relay registration failure streak, reset to 0 on any success.", type: "gauge" }],
+  ["gittensory_orb_relay_drain_seconds_since_last", { help: "Seconds since the pull-mode orb relay drain loop last completed successfully, or -1 if never (or in push mode).", type: "gauge" }],
   ["gittensory_orb_webhook_total", { help: "Orb webhook outcomes.", type: "counter" }],
   ["gittensory_ai_requests_total", { help: "AI provider request outcomes.", type: "counter" }],
   ["gittensory_ai_cost_usd_total", { help: "Estimated AI provider cost in USD.", type: "counter" }],