From 0ad19c5c36357deeee1ddefc1312e99c28d670f7 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 10:47:22 -0700 Subject: [PATCH 01/38] test(ha): define clean install qualification --- deployment-files/ha/QUALIFICATION.md | 80 ++++++++++++++++++++++++++++ deployment-files/ha/README.md | 3 +- 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 deployment-files/ha/QUALIFICATION.md diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md new file mode 100644 index 0000000000..45bbb01473 --- /dev/null +++ b/deployment-files/ha/QUALIFICATION.md @@ -0,0 +1,80 @@ +# Proto Fleet HA qualification + +The tested release artifact and architecture are supported only after they pass +every gate in this document on three clean, same-L2 Debian hosts. End each +measurement at the gate's actual recovery signal: VIP health for routing, +writable SQL for database recovery, durable command/device state for commands, +and telemetry or independent power measurement for curtailment. Do not include +addresses, certificates, passwords, device names, or customer data in the +committed report. + +## Test identity + +| Field | Value | +| --- | --- | +| Release version | Pending | +| Commit SHA | Pending | +| Architecture | arm64 | +| Operating system | Debian 13 | +| Page size | 4096 bytes | +| Started | Pending | +| Completed | Pending | + +## Clean installation + +1. Start with three freshly provisioned Debian 13 hosts. Do not qualify by + deleting directories from a previous HA installation; reimage the hosts so + no old services, containers, firewall rules, VIP, or data remain. +2. Copy the same release, host-specific `node.env`, and only that node's + matching secret directory to each host. Never copy the generated `offline` + directory to a running host. Copy the etcd root password separately to + `ha-a` only. +3. Run `fleet-ha install` concurrently on `ha-a`, `ha-b`, and `ha-c` as + described in [README.md](README.md). +4. Reboot all three hosts. On both database hosts, run + `sudo /opt/proto-fleet/deployment/ha/fleet-ha status + /etc/proto-fleet/ha/node.env --check`. +5. Confirm exactly one active Fleet, one passive Fleet, one Patroni primary, + one Patroni replica, three etcd members, and one VIP owner. + +## Failure matrix + +Restore full readiness before starting each row. Record the observed recovery +time and a short redacted evidence reference. + +| Gate | Required result | Duration | Result | Evidence | +| --- | --- | --- | --- | --- | +| Kill active Fleet process | Peer serves VIP within 15s | Pending | Pending | Pending | +| Power off active host | Peer serves VIP within 15s | Pending | Pending | Pending | +| Stop database primary | Writable primary recovers within 30s | Pending | Pending | Pending | +| Stop database standby | Service remains usable; failover readiness is degraded | Pending | Pending | Pending | +| Stop etcd witness | Service remains usable; failover readiness is degraded | Pending | Pending | Pending | +| Break active DCS path | Old active stops serving; peer takes over | Pending | Pending | Pending | +| Break active database path | Old active stops serving; peer takes over | Pending | Pending | Pending | +| Remove active VIP/interface path | Old active stops serving; peer takes over | Pending | Pending | Pending | +| Fail over with PENDING command | New active dispatches the command | Pending | Pending | Pending | +| Fail over with PROCESSING command | Interrupted attempt fails and later work resumes | Pending | Pending | Pending | +| Fail over during firmware command | Transitional device state is cleared | Pending | Pending | Pending | +| Send command after failover | Command succeeds on the new active | Pending | Pending | Pending | +| Fail over during curtailment | Telemetry or independent power measurement proves load shedding within 180s | Pending | Pending | Pending | + +## Repetition and soak + +| Gate | Required result | Result | Evidence | +| --- | --- | --- | --- | +| Application failover | 5 consecutive passes | Pending | Pending | +| Database failover | 3 consecutive passes | Pending | Pending | +| Soak | 24h with no split ownership or lost failover readiness | Pending | Pending | + +During the soak, sample both hosts at least every two seconds. Record local +`sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env --json --check`, +direct active/passive health, database lease ownership, and VIP interface +ownership with timestamps. Fail the soak on any dual-active, dual-VIP, or +lost-readiness sample. Redact the retained evidence before committing it. + +## Verdict + +**Pending.** Do not describe an artifact and architecture as supported until +every result above is `PASS` and the report records the tested release version +and commit. This initial Raspberry Pi run qualifies arm64 only; amd64 remains +unqualified until the same gates pass for its release artifact. diff --git a/deployment-files/ha/README.md b/deployment-files/ha/README.md index ee9869ed29..7cb87c7be4 100644 --- a/deployment-files/ha/README.md +++ b/deployment-files/ha/README.md @@ -196,4 +196,5 @@ go test ./server/internal/ha -run '^TestProductionHAProfile$' -v Repeat on `ha-b` without `HA_PROFILE_MIGRATE`. The emitted `HA_PROFILE_EVIDENCE` line proves that the deployment artifacts, etcd leader, Patroni primary, and connected PostgreSQL writer agree. -The qualification workflow owns the complete failure matrix and evidence. +The [qualification procedure](QUALIFICATION.md) owns the release qualification +matrix and redacted evidence. From 0e160a7c76ea892b00d2a1482932695065d6d0fd Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 15:56:29 -0700 Subject: [PATCH 02/38] Close HA qualification coverage gaps (#889) --- deployment-files/ha/QUALIFICATION.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 45bbb01473..ebc57308c2 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -8,6 +8,9 @@ and telemetry or independent power measurement for curtailment. Do not include addresses, certificates, passwords, device names, or customer data in the committed report. +Use a fourth, non-peer host on the same L2 network only for the firewall gate; +it must not be one of the three configured HA addresses. + ## Test identity | Field | Value | @@ -47,6 +50,7 @@ time and a short redacted evidence reference. | Kill active Fleet process | Peer serves VIP within 15s | Pending | Pending | Pending | | Power off active host | Peer serves VIP within 15s | Pending | Pending | Pending | | Stop database primary | Writable primary recovers within 30s | Pending | Pending | Pending | +| Commit uniquely identified state, then abruptly stop its acknowledged primary | The exact state exists on the promoted writer | Pending | Pending | Pending | | Stop database standby | Service remains usable; failover readiness is degraded | Pending | Pending | Pending | | Stop etcd witness | Service remains usable; failover readiness is degraded | Pending | Pending | Pending | | Break active DCS path | Old active stops serving; peer takes over | Pending | Pending | Pending | @@ -54,8 +58,11 @@ time and a short redacted evidence reference. | Remove active VIP/interface path | Old active stops serving; peer takes over | Pending | Pending | Pending | | Fail over with PENDING command | New active dispatches the command | Pending | Pending | Pending | | Fail over with PROCESSING command | Interrupted attempt fails and later work resumes | Pending | Pending | Pending | +| Stall a PROCESSING device call, break the active DCS path, wait for takeover, then release the old call | The stale transition is rejected; one device effect and one terminal result remain | Pending | Pending | Pending | | Fail over during firmware command | Transitional device state is cleared | Pending | Pending | Pending | | Send command after failover | Command succeeds on the new active | Pending | Pending | Pending | +| Send active-only API and streaming traffic directly to the passive host | Requests are rejected as not active; streams accepted before demotion close after ownership loss | Pending | Pending | Pending | +| Probe etcd, Patroni, and PostgreSQL from a non-peer host before and after reboot | Every management port rejects the non-peer while configured peers remain connected | Pending | Pending | Pending | | Fail over during curtailment | Telemetry or independent power measurement proves load shedding within 180s | Pending | Pending | Pending | ## Repetition and soak From d37faf5054293f54481b55283c0d89fa67182e3b Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 16:09:31 -0700 Subject: [PATCH 03/38] Make HA qualification gates layer-specific (#889) --- deployment-files/ha/QUALIFICATION.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index ebc57308c2..f980e5cb9b 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -61,8 +61,10 @@ time and a short redacted evidence reference. | Stall a PROCESSING device call, break the active DCS path, wait for takeover, then release the old call | The stale transition is rejected; one device effect and one terminal result remain | Pending | Pending | Pending | | Fail over during firmware command | Transitional device state is cleared | Pending | Pending | Pending | | Send command after failover | Command succeeds on the new active | Pending | Pending | Pending | -| Send active-only API and streaming traffic directly to the passive host | Requests are rejected as not active; streams accepted before demotion close after ownership loss | Pending | Pending | Pending | -| Probe etcd, Patroni, and PostgreSQL from a non-peer host before and after reboot | Every management port rejects the non-peer while configured peers remain connected | Pending | Pending | Pending | +| Pin an authenticated active-only API request to the passive host while retaining the VIP URL and TLS SNI | Fleet returns its machine-readable `NOT_ACTIVE` response, not a TLS or transport error | Pending | Pending | Pending | +| Open an authenticated long-lived stream on the active host, then induce demotion | The established stream closes promptly after ownership loss | Pending | Pending | Pending | +| Attempt TCP connections from a non-peer to ports 2379, 2380, 5432, and 8008 before and after reboot | TCP establishment fails at the firewall while the configured peer connects to every required port | Pending | Pending | Pending | +| Send an unauthorized VRRP protocol-112 advertisement from the non-peer | The firewall drops it and VIP ownership does not change | Pending | Pending | Pending | | Fail over during curtailment | Telemetry or independent power measurement proves load shedding within 180s | Pending | Pending | Pending | ## Repetition and soak From 9fc6ad811e38b75c0d55f2882efe39f0f9ad4c9f Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 16:14:37 -0700 Subject: [PATCH 04/38] Qualify HA debugger-port isolation (#889) --- deployment-files/ha/QUALIFICATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index f980e5cb9b..c2b2d98ef0 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -63,7 +63,7 @@ time and a short redacted evidence reference. | Send command after failover | Command succeeds on the new active | Pending | Pending | Pending | | Pin an authenticated active-only API request to the passive host while retaining the VIP URL and TLS SNI | Fleet returns its machine-readable `NOT_ACTIVE` response, not a TLS or transport error | Pending | Pending | Pending | | Open an authenticated long-lived stream on the active host, then induce demotion | The established stream closes promptly after ownership loss | Pending | Pending | Pending | -| Attempt TCP connections from a non-peer to ports 2379, 2380, 5432, and 8008 before and after reboot | TCP establishment fails at the firewall while the configured peer connects to every required port | Pending | Pending | Pending | +| Attempt TCP connections from a non-peer to ports 2379, 2380, 5432, 8008, and 40000 before and after reboot | TCP establishment fails against the installed firewall while the configured peer connects to every required HA port | Pending | Pending | Pending | | Send an unauthorized VRRP protocol-112 advertisement from the non-peer | The firewall drops it and VIP ownership does not change | Pending | Pending | Pending | | Fail over during curtailment | Telemetry or independent power measurement proves load shedding within 180s | Pending | Pending | Pending | From de6211147d6edf476a9ccc097bf9b40903c0aec4 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 16:20:29 -0700 Subject: [PATCH 05/38] Qualify HA database and MQTT recovery (#889) --- deployment-files/ha/QUALIFICATION.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index c2b2d98ef0..b6c557ead9 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -51,6 +51,7 @@ time and a short redacted evidence reference. | Power off active host | Peer serves VIP within 15s | Pending | Pending | Pending | | Stop database primary | Writable primary recovers within 30s | Pending | Pending | Pending | | Commit uniquely identified state, then abruptly stop its acknowledged primary | The exact state exists on the promoted writer | Pending | Pending | Pending | +| Partition the current Patroni primary from DCS quorum while continuously probing pinned connections to both database hosts | At most one host accepts writes; the isolated primary is read-only or unreachable before promotion; it rejoins without divergent acknowledged state | Pending | Pending | Pending | | Stop database standby | Service remains usable; failover readiness is degraded | Pending | Pending | Pending | | Stop etcd witness | Service remains usable; failover readiness is degraded | Pending | Pending | Pending | | Break active DCS path | Old active stops serving; peer takes over | Pending | Pending | Pending | @@ -66,6 +67,7 @@ time and a short redacted evidence reference. | Attempt TCP connections from a non-peer to ports 2379, 2380, 5432, 8008, and 40000 before and after reboot | TCP establishment fails against the installed firewall while the configured peer connects to every required HA port | Pending | Pending | Pending | | Send an unauthorized VRRP protocol-112 advertisement from the non-peer | The firewall drops it and VIP ownership does not change | Pending | Pending | Pending | | Fail over during curtailment | Telemetry or independent power measurement proves load shedding within 180s | Pending | Pending | Pending | +| After takeover, publish a uniquely identified MQTT curtailment target and then its restoration target | The new active persists both source updates and physical measurement proves shedding and restoration within 180s | Pending | Pending | Pending | ## Repetition and soak From 13ca7e8e66f991d7971a44f430eaf71c9a5dd938 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 17:05:38 -0700 Subject: [PATCH 06/38] Clarify HA host-failure qualification (#889) --- deployment-files/ha/QUALIFICATION.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index b6c557ead9..919f5d0a05 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -48,7 +48,8 @@ time and a short redacted evidence reference. | Gate | Required result | Duration | Result | Evidence | | --- | --- | --- | --- | --- | | Kill active Fleet process | Peer serves VIP within 15s | Pending | Pending | Pending | -| Power off active host | Peer serves VIP within 15s | Pending | Pending | Pending | +| Power off the active Fleet host after confirming it is the Patroni replica | Peer serves VIP within 15s | Pending | Pending | Pending | +| Power off a host that is both active Fleet and Patroni primary | Writable Fleet service recovers through the VIP within 45s | Pending | Pending | Pending | | Stop database primary | Writable primary recovers within 30s | Pending | Pending | Pending | | Commit uniquely identified state, then abruptly stop its acknowledged primary | The exact state exists on the promoted writer | Pending | Pending | Pending | | Partition the current Patroni primary from DCS quorum while continuously probing pinned connections to both database hosts | At most one host accepts writes; the isolated primary is read-only or unreachable before promotion; it rejoins without divergent acknowledged state | Pending | Pending | Pending | From 9b4880b2e73ca747d4bd078b366be180371cdcb3 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 17:40:59 -0700 Subject: [PATCH 07/38] Strengthen HA qualification evidence --- deployment-files/ha/QUALIFICATION.md | 17 +++++++++++------ deployment-files/ha/firewall.nft.tmpl | 12 ++++++------ deployment-files/ha/tests/test-profile.sh | 2 +- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 919f5d0a05..836c79fbbf 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -65,8 +65,8 @@ time and a short redacted evidence reference. | Send command after failover | Command succeeds on the new active | Pending | Pending | Pending | | Pin an authenticated active-only API request to the passive host while retaining the VIP URL and TLS SNI | Fleet returns its machine-readable `NOT_ACTIVE` response, not a TLS or transport error | Pending | Pending | Pending | | Open an authenticated long-lived stream on the active host, then induce demotion | The established stream closes promptly after ownership loss | Pending | Pending | Pending | -| Attempt TCP connections from a non-peer to ports 2379, 2380, 5432, 8008, and 40000 before and after reboot | TCP establishment fails against the installed firewall while the configured peer connects to every required HA port | Pending | Pending | Pending | -| Send an unauthorized VRRP protocol-112 advertisement from the non-peer | The firewall drops it and VIP ownership does not change | Pending | Pending | Pending | +| With listeners confirmed, attempt TCP connections from a non-peer to ports 2379, 2380, 5432, 8008, and 40000 before and after reboot | Required HA ports accept their configured peers but reject the non-peer. For the intentionally closed 40000 port, a temporary listener accepts loopback but rejects the non-peer. The matching nftables drop counters increase. | Pending | Pending | Pending | +| Send an unauthorized VRRP protocol-112 advertisement from the non-peer | The VRRP drop counter increases and VIP ownership does not change | Pending | Pending | Pending | | Fail over during curtailment | Telemetry or independent power measurement proves load shedding within 180s | Pending | Pending | Pending | | After takeover, publish a uniquely identified MQTT curtailment target and then its restoration target | The new active persists both source updates and physical measurement proves shedding and restoration within 180s | Pending | Pending | Pending | @@ -78,11 +78,16 @@ time and a short redacted evidence reference. | Database failover | 3 consecutive passes | Pending | Pending | | Soak | 24h with no split ownership or lost failover readiness | Pending | Pending | -During the soak, sample both hosts at least every two seconds. Record local +During the soak, sample both hosts at least every two seconds to measure +availability. Separately poll the database lease below the three-second renewal +interval, recording database time, holder, epoch, and expiry, and retain both +hosts' timestamped service start/exit journals. Reconstruct each active proof +interval and fail on overlap. Record local `sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env --json --check`, -direct active/passive health, database lease ownership, and VIP interface -ownership with timestamps. Fail the soak on any dual-active, dual-VIP, or -lost-readiness sample. Redact the retained evidence before committing it. +direct active/passive health, and VIP interface ownership with timestamps. Fail +the soak on any dual-active, dual-VIP, or lost-readiness sample. The two-second +samples alone are not proof against shorter ownership overlap. Redact the +retained evidence before committing it. ## Verdict diff --git a/deployment-files/ha/firewall.nft.tmpl b/deployment-files/ha/firewall.nft.tmpl index 2f70770911..e7b6489f98 100644 --- a/deployment-files/ha/firewall.nft.tmpl +++ b/deployment-files/ha/firewall.nft.tmpl @@ -14,22 +14,22 @@ table inet proto_fleet_ha { # Trust VRRP only between Fleet-capable hosts on the configured HA network. ip protocol vrrp iifname "${HA_NETWORK_INTERFACE}" ip saddr @database_nodes ip daddr ${HA_NODE_IP} accept - ip protocol vrrp drop + ip protocol vrrp counter drop # Clients enter through nginx; fleet-api remains local-only. iifname "lo" tcp dport 4000 accept - tcp dport 4000 drop + tcp dport 4000 counter drop # The HA image runs fleetd directly; never expose the image's debug port. - tcp dport 40000 drop + tcp dport 40000 counter drop tcp dport 2380 ip saddr @ha_nodes accept - tcp dport 2380 drop + tcp dport 2380 counter drop tcp dport 2379 ip saddr @ha_nodes accept - tcp dport 2379 drop + tcp dport 2379 counter drop tcp dport { 5432, 8008 } ip saddr @database_nodes accept - tcp dport { 5432, 8008 } drop + tcp dport { 5432, 8008 } counter drop } } diff --git a/deployment-files/ha/tests/test-profile.sh b/deployment-files/ha/tests/test-profile.sh index 6bf189dc85..0b28dab1bc 100755 --- a/deployment-files/ha/tests/test-profile.sh +++ b/deployment-files/ha/tests/test-profile.sh @@ -145,7 +145,7 @@ test_fleet_ha_contract() { assert_contains "${HA_DIR}/proto-fleet-ha-keepalived.conf" "Wants=keepalived.service" assert_contains "${HA_DIR}/keepalived-systemd.conf.tmpl" 'ExecStopPost=/usr/sbin/ip address flush to ${HA_VIRTUAL_IP}/32 dev ${HA_NETWORK_INTERFACE}' assert_not_contains "${HA_DIR}/firewall.nft.tmpl" "destroy table" - assert_contains "${HA_DIR}/firewall.nft.tmpl" "tcp dport 40000 drop" + assert_contains "${HA_DIR}/firewall.nft.tmpl" "tcp dport 40000 counter drop" assert_contains "${HA_DIR}/proto-fleet-ha-firewall.service" "ExecStart=/usr/sbin/nft -f /etc/proto-fleet/ha/firewall.nft" assert_contains "${HA_DIR}/proto-fleet-ha-firewall.service" "ExecStartPre=/usr/sbin/nft -c -f /etc/proto-fleet/ha/firewall.nft" assert_contains "${HA_DIR}/proto-fleet-ha-firewall.service" "ExecStartPre=-/usr/sbin/nft delete table inet proto_fleet_ha" From c4dea903b3c441e77a63e7b8626059a308ba0772 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 17:46:31 -0700 Subject: [PATCH 08/38] Correct debugger firewall qualification --- deployment-files/ha/QUALIFICATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 836c79fbbf..ab12788d51 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -65,7 +65,7 @@ time and a short redacted evidence reference. | Send command after failover | Command succeeds on the new active | Pending | Pending | Pending | | Pin an authenticated active-only API request to the passive host while retaining the VIP URL and TLS SNI | Fleet returns its machine-readable `NOT_ACTIVE` response, not a TLS or transport error | Pending | Pending | Pending | | Open an authenticated long-lived stream on the active host, then induce demotion | The established stream closes promptly after ownership loss | Pending | Pending | Pending | -| With listeners confirmed, attempt TCP connections from a non-peer to ports 2379, 2380, 5432, 8008, and 40000 before and after reboot | Required HA ports accept their configured peers but reject the non-peer. For the intentionally closed 40000 port, a temporary listener accepts loopback but rejects the non-peer. The matching nftables drop counters increase. | Pending | Pending | Pending | +| With listeners confirmed, attempt TCP connections from a non-peer to ports 2379, 2380, 5432, 8008, and 40000 before and after reboot | Required HA ports accept their configured peers but reject the non-peer. For the intentionally closed 40000 port, `ss` confirms a temporary listener exists while the non-peer is rejected. The matching nftables drop counters increase. | Pending | Pending | Pending | | Send an unauthorized VRRP protocol-112 advertisement from the non-peer | The VRRP drop counter increases and VIP ownership does not change | Pending | Pending | Pending | | Fail over during curtailment | Telemetry or independent power measurement proves load shedding within 180s | Pending | Pending | Pending | | After takeover, publish a uniquely identified MQTT curtailment target and then its restoration target | The new active persists both source updates and physical measurement proves shedding and restoration within 180s | Pending | Pending | Pending | From b8501c2f4753b367df53c0426a932911520800b6 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 17:58:54 -0700 Subject: [PATCH 09/38] Scope HA soak evidence precisely --- deployment-files/ha/QUALIFICATION.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index ab12788d51..00e44c8c73 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -76,18 +76,19 @@ time and a short redacted evidence reference. | --- | --- | --- | --- | | Application failover | 5 consecutive passes | Pending | Pending | | Database failover | 3 consecutive passes | Pending | Pending | -| Soak | 24h with no split ownership or lost failover readiness | Pending | Pending | +| Soak | 24h with no observed dual-active state or lost failover readiness | Pending | Pending | During the soak, sample both hosts at least every two seconds to measure availability. Separately poll the database lease below the three-second renewal interval, recording database time, holder, epoch, and expiry, and retain both -hosts' timestamped service start/exit journals. Reconstruct each active proof -interval and fail on overlap. Record local +hosts' timestamped service start/exit journals. Record local `sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env --json --check`, direct active/passive health, and VIP interface ownership with timestamps. Fail -the soak on any dual-active, dual-VIP, or lost-readiness sample. The two-second -samples alone are not proof against shorter ownership overlap. Redact the -retained evidence before committing it. +the soak on any dual-active, dual-VIP, or lost-readiness sample, and repeat the +stale-transition gate after every application failover. These observations do +not constitute an event-complete admission history; a durable activation-event +ledger is outside this initial qualification. Redact the retained evidence +before committing it. ## Verdict From 5b408ad841d620d0ff9b4dc19406534cea92c1e7 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 19:24:27 -0700 Subject: [PATCH 10/38] Simplify HA qualification gates --- deployment-files/ha/QUALIFICATION.md | 24 +++--------------------- deployment-files/ha/firewall.nft.tmpl | 12 ++++++------ 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 00e44c8c73..07b084e5a8 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -8,9 +8,6 @@ and telemetry or independent power measurement for curtailment. Do not include addresses, certificates, passwords, device names, or customer data in the committed report. -Use a fourth, non-peer host on the same L2 network only for the firewall gate; -it must not be one of the three configured HA addresses. - ## Test identity | Field | Value | @@ -49,10 +46,7 @@ time and a short redacted evidence reference. | --- | --- | --- | --- | --- | | Kill active Fleet process | Peer serves VIP within 15s | Pending | Pending | Pending | | Power off the active Fleet host after confirming it is the Patroni replica | Peer serves VIP within 15s | Pending | Pending | Pending | -| Power off a host that is both active Fleet and Patroni primary | Writable Fleet service recovers through the VIP within 45s | Pending | Pending | Pending | | Stop database primary | Writable primary recovers within 30s | Pending | Pending | Pending | -| Commit uniquely identified state, then abruptly stop its acknowledged primary | The exact state exists on the promoted writer | Pending | Pending | Pending | -| Partition the current Patroni primary from DCS quorum while continuously probing pinned connections to both database hosts | At most one host accepts writes; the isolated primary is read-only or unreachable before promotion; it rejoins without divergent acknowledged state | Pending | Pending | Pending | | Stop database standby | Service remains usable; failover readiness is degraded | Pending | Pending | Pending | | Stop etcd witness | Service remains usable; failover readiness is degraded | Pending | Pending | Pending | | Break active DCS path | Old active stops serving; peer takes over | Pending | Pending | Pending | @@ -60,15 +54,9 @@ time and a short redacted evidence reference. | Remove active VIP/interface path | Old active stops serving; peer takes over | Pending | Pending | Pending | | Fail over with PENDING command | New active dispatches the command | Pending | Pending | Pending | | Fail over with PROCESSING command | Interrupted attempt fails and later work resumes | Pending | Pending | Pending | -| Stall a PROCESSING device call, break the active DCS path, wait for takeover, then release the old call | The stale transition is rejected; one device effect and one terminal result remain | Pending | Pending | Pending | | Fail over during firmware command | Transitional device state is cleared | Pending | Pending | Pending | | Send command after failover | Command succeeds on the new active | Pending | Pending | Pending | -| Pin an authenticated active-only API request to the passive host while retaining the VIP URL and TLS SNI | Fleet returns its machine-readable `NOT_ACTIVE` response, not a TLS or transport error | Pending | Pending | Pending | -| Open an authenticated long-lived stream on the active host, then induce demotion | The established stream closes promptly after ownership loss | Pending | Pending | Pending | -| With listeners confirmed, attempt TCP connections from a non-peer to ports 2379, 2380, 5432, 8008, and 40000 before and after reboot | Required HA ports accept their configured peers but reject the non-peer. For the intentionally closed 40000 port, `ss` confirms a temporary listener exists while the non-peer is rejected. The matching nftables drop counters increase. | Pending | Pending | Pending | -| Send an unauthorized VRRP protocol-112 advertisement from the non-peer | The VRRP drop counter increases and VIP ownership does not change | Pending | Pending | Pending | | Fail over during curtailment | Telemetry or independent power measurement proves load shedding within 180s | Pending | Pending | Pending | -| After takeover, publish a uniquely identified MQTT curtailment target and then its restoration target | The new active persists both source updates and physical measurement proves shedding and restoration within 180s | Pending | Pending | Pending | ## Repetition and soak @@ -78,17 +66,11 @@ time and a short redacted evidence reference. | Database failover | 3 consecutive passes | Pending | Pending | | Soak | 24h with no observed dual-active state or lost failover readiness | Pending | Pending | -During the soak, sample both hosts at least every two seconds to measure -availability. Separately poll the database lease below the three-second renewal -interval, recording database time, holder, epoch, and expiry, and retain both -hosts' timestamped service start/exit journals. Record local +During the soak, sample both hosts at least every two seconds. Record local `sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env --json --check`, direct active/passive health, and VIP interface ownership with timestamps. Fail -the soak on any dual-active, dual-VIP, or lost-readiness sample, and repeat the -stale-transition gate after every application failover. These observations do -not constitute an event-complete admission history; a durable activation-event -ledger is outside this initial qualification. Redact the retained evidence -before committing it. +the soak on any dual-active, dual-VIP, or lost-readiness sample. Redact the +retained evidence before committing it. ## Verdict diff --git a/deployment-files/ha/firewall.nft.tmpl b/deployment-files/ha/firewall.nft.tmpl index e7b6489f98..2f70770911 100644 --- a/deployment-files/ha/firewall.nft.tmpl +++ b/deployment-files/ha/firewall.nft.tmpl @@ -14,22 +14,22 @@ table inet proto_fleet_ha { # Trust VRRP only between Fleet-capable hosts on the configured HA network. ip protocol vrrp iifname "${HA_NETWORK_INTERFACE}" ip saddr @database_nodes ip daddr ${HA_NODE_IP} accept - ip protocol vrrp counter drop + ip protocol vrrp drop # Clients enter through nginx; fleet-api remains local-only. iifname "lo" tcp dport 4000 accept - tcp dport 4000 counter drop + tcp dport 4000 drop # The HA image runs fleetd directly; never expose the image's debug port. - tcp dport 40000 counter drop + tcp dport 40000 drop tcp dport 2380 ip saddr @ha_nodes accept - tcp dport 2380 counter drop + tcp dport 2380 drop tcp dport 2379 ip saddr @ha_nodes accept - tcp dport 2379 counter drop + tcp dport 2379 drop tcp dport { 5432, 8008 } ip saddr @database_nodes accept - tcp dport { 5432, 8008 } counter drop + tcp dport { 5432, 8008 } drop } } From 29d302b462cbaf7f3707bc9ead04b821d73c20b8 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 20:45:56 -0700 Subject: [PATCH 11/38] Strengthen HA qualification gates --- deployment-files/ha/QUALIFICATION.md | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 07b084e5a8..0f483f9d49 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -1,7 +1,8 @@ # Proto Fleet HA qualification -The tested release artifact and architecture are supported only after they pass -every gate in this document on three clean, same-L2 Debian hosts. End each +The tested release artifact, host package versions, and architecture are +supported only after they pass every gate in this document on three clean, +same-L2 Debian hosts. End each measurement at the gate's actual recovery signal: VIP health for routing, writable SQL for database recovery, durable command/device state for commands, and telemetry or independent power measurement for curtailment. Do not include @@ -17,6 +18,7 @@ committed report. | Architecture | arm64 | | Operating system | Debian 13 | | Page size | 4096 bytes | +| Docker, containerd, keepalived, nftables, and arping package versions | Pending | | Started | Pending | | Completed | Pending | @@ -46,17 +48,26 @@ time and a short redacted evidence reference. | --- | --- | --- | --- | --- | | Kill active Fleet process | Peer serves VIP within 15s | Pending | Pending | Pending | | Power off the active Fleet host after confirming it is the Patroni replica | Peer serves VIP within 15s | Pending | Pending | Pending | -| Stop database primary | Writable primary recovers within 30s | Pending | Pending | Pending | +| Power off a host that is both active Fleet and Patroni primary | Peer restores writable VIP service within 30s | Pending | Pending | Pending | +| Abruptly lose the database primary after an acknowledged uniquely identified write | Writable primary recovers within 30s and retains the write | Pending | Pending | Pending | +| Isolate the database primary from DCS quorum | At most one primary accepts writes; the old primary rejoins without divergence | Pending | Pending | Pending | | Stop database standby | Service remains usable; failover readiness is degraded | Pending | Pending | Pending | | Stop etcd witness | Service remains usable; failover readiness is degraded | Pending | Pending | Pending | | Break active DCS path | Old active stops serving; peer takes over | Pending | Pending | Pending | | Break active database path | Old active stops serving; peer takes over | Pending | Pending | Pending | | Remove active VIP/interface path | Old active stops serving; peer takes over | Pending | Pending | Pending | +| Send an active-only request directly to the passive host | Passive rejects it as not active | Pending | Pending | Pending | +| Hold an active-only request across demotion | The old active cancels it before the peer serves active traffic | Pending | Pending | Pending | +| Probe HA-only ports from a non-peer before and after reboot | Protected ports reject the probe while peer traffic remains healthy | Pending | Pending | Pending | +| Send a VRRP advertisement from a non-peer | The packet is dropped and VIP ownership does not change | Pending | Pending | Pending | | Fail over with PENDING command | New active dispatches the command | Pending | Pending | Pending | | Fail over with PROCESSING command | Interrupted attempt fails and later work resumes | Pending | Pending | Pending | | Fail over during firmware command | Transitional device state is cleared | Pending | Pending | Pending | | Send command after failover | Command succeeds on the new active | Pending | Pending | Pending | -| Fail over during curtailment | Telemetry or independent power measurement proves load shedding within 180s | Pending | Pending | Pending | +| Publish unique curtailment and restoration targets after failover | Both inputs persist and measured load follows them within 180s | Pending | Pending | Pending | + +The PROCESSING-command gate proves server-side recovery, not exactly-once +device effects. Device-side fencing is outside this profile's support claim. ## Repetition and soak @@ -75,6 +86,7 @@ retained evidence before committing it. ## Verdict **Pending.** Do not describe an artifact and architecture as supported until -every result above is `PASS` and the report records the tested release version -and commit. This initial Raspberry Pi run qualifies arm64 only; amd64 remains -unqualified until the same gates pass for its release artifact. +every result above is `PASS` and the report records the tested release, commit, +and host package versions. A package-version change requires requalification. +This initial Raspberry Pi run qualifies arm64 only; amd64 remains unqualified +until the same gates pass for its release artifact. From c271f14afd2089b83a10514afd313adcf0617afa Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 21:31:43 -0700 Subject: [PATCH 12/38] Tighten HA qualification evidence --- deployment-files/ha/QUALIFICATION.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 0f483f9d49..87a60ee306 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -61,10 +61,10 @@ time and a short redacted evidence reference. | Probe HA-only ports from a non-peer before and after reboot | Protected ports reject the probe while peer traffic remains healthy | Pending | Pending | Pending | | Send a VRRP advertisement from a non-peer | The packet is dropped and VIP ownership does not change | Pending | Pending | Pending | | Fail over with PENDING command | New active dispatches the command | Pending | Pending | Pending | -| Fail over with PROCESSING command | Interrupted attempt fails and later work resumes | Pending | Pending | Pending | +| Demote a live old active with a stalled PROCESSING plugin call, recover on the peer, then release the old call | The stale transition is rejected, exactly one terminal database result remains, and later work resumes | Pending | Pending | Pending | | Fail over during firmware command | Transitional device state is cleared | Pending | Pending | Pending | | Send command after failover | Command succeeds on the new active | Pending | Pending | Pending | -| Publish unique curtailment and restoration targets after failover | Both inputs persist and measured load follows them within 180s | Pending | Pending | Pending | +| Publish a unique curtailment target, confirm shedding starts, fail the active before completion, then restore after takeover | The peer retains or reasserts curtailment within 180s, then measured load follows the restoration target | Pending | Pending | Pending | The PROCESSING-command gate proves server-side recovery, not exactly-once device effects. Device-side fencing is outside this profile's support claim. @@ -77,11 +77,16 @@ device effects. Device-side fencing is outside this profile's support claim. | Database failover | 3 consecutive passes | Pending | Pending | | Soak | 24h with no observed dual-active state or lost failover readiness | Pending | Pending | -During the soak, sample both hosts at least every two seconds. Record local +Before the soak, install a qualification-only database trigger that appends +every `fleet_runtime_lease` insert and update to a separate audit table with +database time, DCS cluster ID, writer generation, lease epoch, holder ID, and +expiry. During the soak, retain that event-complete lease history and sample +both hosts at least every two seconds for availability. Record local `sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env --json --check`, direct active/passive health, and VIP interface ownership with timestamps. Fail -the soak on any dual-active, dual-VIP, or lost-readiness sample. Redact the -retained evidence before committing it. +on overlap between reconstructed holder/epoch terms, a missing audit interval, +dual-VIP, or lost readiness. Export redacted evidence, then remove the trigger +and audit table. ## Verdict From 09337b8d516f9ec9741b851bb34fbf57f1b632f4 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 21:40:15 -0700 Subject: [PATCH 13/38] Clarify HA qualification support boundary --- deployment-files/ha/QUALIFICATION.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 87a60ee306..9cee9a86b8 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -9,6 +9,11 @@ and telemetry or independent power measurement for curtailment. Do not include addresses, certificates, passwords, device names, or customer data in the committed report. +This report qualifies the fixed Fleet application, database, DCS, and VIP +profile only. Adjacent application updates require +[UPDATE_QUALIFICATION.md](UPDATE_QUALIFICATION.md). Fleet Node HA, reconnect +scale, and alert delivery remain outside this support claim. + ## Test identity | Field | Value | @@ -56,15 +61,18 @@ time and a short redacted evidence reference. | Break active DCS path | Old active stops serving; peer takes over | Pending | Pending | Pending | | Break active database path | Old active stops serving; peer takes over | Pending | Pending | Pending | | Remove active VIP/interface path | Old active stops serving; peer takes over | Pending | Pending | Pending | -| Send an active-only request directly to the passive host | Passive rejects it as not active | Pending | Pending | Pending | +| Fail one active-runtime job without killing Fleet | Active health fails, the process exits, and the peer takes over | Pending | Pending | Pending | +| Send Connect RPC, non-RPC HTTP, and ControlStream traffic directly to the passive host | Each product transport rejects it as not active; only health and local status remain available | Pending | Pending | Pending | | Hold an active-only request across demotion | The old active cancels it before the peer serves active traffic | Pending | Pending | Pending | +| Probe public health through the VIP | Public responses reveal no HA topology and `/health/ha` returns 404 | Pending | Pending | Pending | +| With Grafana and telemetry ingestion stopped, stop the etcd witness | Active health remains usable and local HA status still reports degraded failover readiness | Pending | Pending | Pending | | Probe HA-only ports from a non-peer before and after reboot | Protected ports reject the probe while peer traffic remains healthy | Pending | Pending | Pending | | Send a VRRP advertisement from a non-peer | The packet is dropped and VIP ownership does not change | Pending | Pending | Pending | | Fail over with PENDING command | New active dispatches the command | Pending | Pending | Pending | | Demote a live old active with a stalled PROCESSING plugin call, recover on the peer, then release the old call | The stale transition is rejected, exactly one terminal database result remains, and later work resumes | Pending | Pending | Pending | | Fail over during firmware command | Transitional device state is cleared | Pending | Pending | Pending | | Send command after failover | Command succeeds on the new active | Pending | Pending | Pending | -| Publish a unique curtailment target, confirm shedding starts, fail the active before completion, then restore after takeover | The peer retains or reasserts curtailment within 180s, then measured load follows the restoration target | Pending | Pending | Pending | +| Publish a unique MQTT curtailment target, confirm shedding starts, fail the active before completion, then restore after takeover | The peer resumes MQTT intake and retains or reasserts curtailment within 180s, then measured load follows the restoration target | Pending | Pending | Pending | The PROCESSING-command gate proves server-side recovery, not exactly-once device effects. Device-side fencing is outside this profile's support claim. @@ -80,11 +88,13 @@ device effects. Device-side fencing is outside this profile's support claim. Before the soak, install a qualification-only database trigger that appends every `fleet_runtime_lease` insert and update to a separate audit table with database time, DCS cluster ID, writer generation, lease epoch, holder ID, and -expiry. During the soak, retain that event-complete lease history and sample -both hosts at least every two seconds for availability. Record local +expiry. From one controller, continuously stream interface-address events and +direct active-health results from both hosts with synchronized monotonic +timestamps at 100 ms or faster. Also sample both hosts at least every two +seconds for availability. Record local `sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env --json --check`, -direct active/passive health, and VIP interface ownership with timestamps. Fail -on overlap between reconstructed holder/epoch terms, a missing audit interval, +and retain the lease, health, and address streams. Fail on overlap between +reconstructed holder/epoch terms, a missing audit interval, dual-active, dual-VIP, or lost readiness. Export redacted evidence, then remove the trigger and audit table. From 7a9e6abcd8d8ee3209d09835fecc01fe69a6c327 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 21:47:43 -0700 Subject: [PATCH 14/38] Keep qualification evidence external --- deployment-files/ha/QUALIFICATION.md | 29 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 9cee9a86b8..6abd4c9d45 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -10,9 +10,8 @@ addresses, certificates, passwords, device names, or customer data in the committed report. This report qualifies the fixed Fleet application, database, DCS, and VIP -profile only. Adjacent application updates require -[UPDATE_QUALIFICATION.md](UPDATE_QUALIFICATION.md). Fleet Node HA, reconnect -scale, and alert delivery remain outside this support claim. +profile only. Adjacent application updates are not qualified here. Fleet Node +HA, reconnect scale, and alert delivery remain outside this support claim. ## Test identity @@ -64,7 +63,7 @@ time and a short redacted evidence reference. | Fail one active-runtime job without killing Fleet | Active health fails, the process exits, and the peer takes over | Pending | Pending | Pending | | Send Connect RPC, non-RPC HTTP, and ControlStream traffic directly to the passive host | Each product transport rejects it as not active; only health and local status remain available | Pending | Pending | Pending | | Hold an active-only request across demotion | The old active cancels it before the peer serves active traffic | Pending | Pending | Pending | -| Probe public health through the VIP | Public responses reveal no HA topology and `/health/ha` returns 404 | Pending | Pending | Pending | +| Probe public health through the VIP | Public responses reveal no HA topology and `/api-proxy/health/ha` returns 404 | Pending | Pending | Pending | | With Grafana and telemetry ingestion stopped, stop the etcd witness | Active health remains usable and local HA status still reports degraded failover readiness | Pending | Pending | Pending | | Probe HA-only ports from a non-peer before and after reboot | Protected ports reject the probe while peer traffic remains healthy | Pending | Pending | Pending | | Send a VRRP advertisement from a non-peer | The packet is dropped and VIP ownership does not change | Pending | Pending | Pending | @@ -85,18 +84,18 @@ device effects. Device-side fencing is outside this profile's support claim. | Database failover | 3 consecutive passes | Pending | Pending | | Soak | 24h with no observed dual-active state or lost failover readiness | Pending | Pending | -Before the soak, install a qualification-only database trigger that appends -every `fleet_runtime_lease` insert and update to a separate audit table with -database time, DCS cluster ID, writer generation, lease epoch, holder ID, and -expiry. From one controller, continuously stream interface-address events and -direct active-health results from both hosts with synchronized monotonic -timestamps at 100 ms or faster. Also sample both hosts at least every two -seconds for availability. Record local +Before the soak, start an external append-only recorder on a separate +controller. Continuously record each host's direct active-health result and +interface-address events with synchronized monotonic timestamps at 100 ms or +faster; evidence stored only inside the HA database is insufficient. Treat an +unreachable host, write failure, or gap longer than 250 ms as a failed soak. +Also sample both hosts at least every two seconds for availability. Record local `sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env --json --check`, -and retain the lease, health, and address streams. Fail on overlap between -reconstructed holder/epoch terms, a missing audit interval, dual-active, -dual-VIP, or lost readiness. Export redacted evidence, then remove the trigger -and audit table. +and retain the status, health, and address streams. Reconstruct each active +interval from the first active result through the first passive result, and +each VIP interval from address-add through address-delete. Fail on overlapping +host intervals, any collection gap, or lost readiness. Export redacted +evidence after the soak. ## Verdict From 24d72a99aca9a157d7e9cdfb94b847a8d131e4d6 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 21:55:35 -0700 Subject: [PATCH 15/38] Align HA qualification with initial scope --- deployment-files/ha/QUALIFICATION.md | 15 ++++++------ docs/rfcs/0002-active-passive-fleet-ha.md | 30 +++++++++++------------ 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 6abd4c9d45..0031fffc98 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -65,8 +65,8 @@ time and a short redacted evidence reference. | Hold an active-only request across demotion | The old active cancels it before the peer serves active traffic | Pending | Pending | Pending | | Probe public health through the VIP | Public responses reveal no HA topology and `/api-proxy/health/ha` returns 404 | Pending | Pending | Pending | | With Grafana and telemetry ingestion stopped, stop the etcd witness | Active health remains usable and local HA status still reports degraded failover readiness | Pending | Pending | Pending | -| Probe HA-only ports from a non-peer before and after reboot | Protected ports reject the probe while peer traffic remains healthy | Pending | Pending | Pending | -| Send a VRRP advertisement from a non-peer | The packet is dropped and VIP ownership does not change | Pending | Pending | Pending | +| From a fourth non-peer, raw-probe each HA-only port before and after reboot after verifying its listener from a peer | The matching nftables drop counter increases while peer traffic remains healthy; use a temporary listener for the normally closed debug port | Pending | Pending | Pending | +| Send a valid winning VRRP advertisement from a non-peer | The protocol-112 drop counter increases and VIP ownership does not change | Pending | Pending | Pending | | Fail over with PENDING command | New active dispatches the command | Pending | Pending | Pending | | Demote a live old active with a stalled PROCESSING plugin call, recover on the peer, then release the old call | The stale transition is rejected, exactly one terminal database result remains, and later work resumes | Pending | Pending | Pending | | Fail over during firmware command | Transitional device state is cleared | Pending | Pending | Pending | @@ -91,11 +91,12 @@ faster; evidence stored only inside the HA database is insufficient. Treat an unreachable host, write failure, or gap longer than 250 ms as a failed soak. Also sample both hosts at least every two seconds for availability. Record local `sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env --json --check`, -and retain the status, health, and address streams. Reconstruct each active -interval from the first active result through the first passive result, and -each VIP interval from address-add through address-delete. Fail on overlapping -host intervals, any collection gap, or lost readiness. Export redacted -evidence after the soak. +and retain the status, health, and address streams. Conservatively reconstruct +each active interval from its last preceding passive sample through its first +following passive sample, and each VIP interval from address-add through +address-delete. Fail when these uncertainty-inclusive host intervals could +overlap, or on any collection gap or lost readiness. Export redacted evidence +after the soak. ## Verdict diff --git a/docs/rfcs/0002-active-passive-fleet-ha.md b/docs/rfcs/0002-active-passive-fleet-ha.md index 1e50940a1b..0eeee5f62e 100644 --- a/docs/rfcs/0002-active-passive-fleet-ha.md +++ b/docs/rfcs/0002-active-passive-fleet-ha.md @@ -3,13 +3,13 @@ - **Status**: approved - **Author(s)**: Ankit Goswami (@ankitgoswami) - **Created**: 2026-07-13 -- **Last updated**: 2026-08-04 +- **Last updated**: 2026-08-07 ## Summary Add a supported high-availability install mode for Proto Fleet where two warm Fleet app hosts share a self-managed Postgres/TimescaleDB HA cluster, but exactly one Fleet app instance is active for real-time control at a time. The active instance is selected by a Fleet-owned, epoch-fenced database lease. Postgres failover is handled by Patroni over a three-member quorum, and Fleet connects to the current writable database through a pgx/libpq-style multi-host DSN rather than a DB proxy. -This RFC deliberately scopes the HA promise to the real-time control plane. Curtailment dispatch, command execution, schedules, Fleet Node ControlStreams, MQTT curtailment intake, and the database state required for those flows must recover automatically after a single failure. Live HA alerting for failover readiness and control-plane health remains in scope. Historical telemetry, Grafana dashboards, alert history, logs, and cache-like artifacts may be stale, delayed, unavailable, or partially lost in degraded mode. +This RFC deliberately scopes the initial HA promise to Fleet application control. Curtailment dispatch, command execution, schedules, MQTT curtailment intake, and the database state required for those flows must recover automatically after a single failure. Local HA status must expose failover readiness and control-plane health. Fleet Node HA and reconnect-scale qualification, alert delivery, historical telemetry, Grafana dashboards, alert history, logs, and cache-like artifacts remain outside the initial supported profile. ## Decision summary @@ -42,8 +42,8 @@ The supported HA contract is intentionally narrower than "every subsystem stays | Class | Examples | HA guarantee | | ---- | -------- | ------------ | | Critical control state | Curtailment events and targets, command queue/status, schedules, Fleet Node auth/pairing state, MQTT curtailment source config and runtime edge state, active Fleet lease | Durable while replication is healthy; required for RTO | -| Real-time runtime | Active Fleet app, Fleet Node ControlStreams, command executor, curtailment reconciler, scheduler, MQTT subscriber | One active instance; resumes automatically on another Fleet app host | -| Live HA alert state | Failover readiness, active Fleet holder count, DB primary/standby health, quorum, replication lag, VIP/load-balancer target | Must be emitted from control-plane HA status, independent of Grafana history | +| Real-time runtime | Active Fleet app, command executor, curtailment reconciler, scheduler, MQTT subscriber | One active instance; resumes automatically on another Fleet app host | +| Local HA status | Failover readiness, active Fleet holder count, DB primary/standby health, quorum, replication lag, VIP/load-balancer target | Must be emitted independently of Grafana history; alert delivery is deferred | | Best-effort history | Raw telemetry samples, rollups, notification metric samples, Grafana dashboards, alert history, logs | May be stale, delayed, unavailable, or partially lost | | Local artifacts | Firmware files, command artifacts, cached downloads | Not v1 HA unless explicitly promoted to critical storage | @@ -102,13 +102,13 @@ For on-prem installs, the supported endpoint is a Fleet VIP managed by keepalive - both Fleet app hosts run keepalived with the same VIP; - keepalived advertises the VIP only on the host whose local Fleet app passes `/health/active`; - the VIP moves to the peer after the old active fails health and the peer acquires the Fleet lease; -- existing long-lived UI and ControlStream connections reconnect to the same stable endpoint after VIP movement. +- existing UI and API clients reconnect to the same stable endpoint after VIP movement. The VIP is an endpoint routing mechanism, not the correctness authority. Fleet's database lease remains the only source of truth for app activeness. The VIP must follow `/health/active`; it must not decide which Fleet app is allowed to dispatch commands or curtailment. Endpoint-adapter failures that make the active Fleet endpoint unavailable are active-readiness failures. In the on-prem VIP profile, if the current active host cannot maintain local VIP ownership or advertisement, it must fail `/health/active` and stop renewing or relinquish the Fleet lease, or provide an equivalent endpoint-adapter fencing mechanism that lets the peer take over. An active Fleet app that is no longer reachable through the supported stable endpoint must not keep the active lease indefinitely. -The stable endpoint must preserve Fleet's client-facing identity and network-security expectations in every environment. VIP movement must not make UI/API, Fleet Node, or ControlStream traffic reachable outside the intended private network. The supported on-prem VIP profile can rely on the site VPN/private network plus VIP ownership controls that restrict advertisement to the intended Fleet app hosts. Cloud or otherwise untrusted network paths require environment-appropriate transport security and server identity. +The stable endpoint must preserve Fleet's client-facing identity and network-security expectations in every environment. VIP movement must not make UI/API traffic reachable outside the intended private network. The supported on-prem VIP profile can rely on the site VPN/private network plus VIP ownership controls that restrict advertisement to the intended Fleet app hosts. Cloud or otherwise untrusted network paths require environment-appropriate transport security and server identity. MQTT curtailment intake is not Fleet VIP traffic. It is an active runtime responsibility: the old active subscriber must quiesce on failover, and the new active Fleet app must subscribe to the configured broker within the RTO target. @@ -244,7 +244,7 @@ Always-on services: Activation must trigger required control reconciliation promptly enough to meet the RTO target. -### Request and ControlStream gating +### Request gating Add a single active-mode request gate for product traffic: @@ -252,14 +252,13 @@ Add a single active-mode request gate for product traffic: - In passive mode, Connect/gRPC product traffic fails with `Unavailable` and a machine-readable `not-active` detail; other product transports use the equivalent retryable status. - Load-balancer-safe health endpoints and loopback-only HA diagnostics bypass active gating. -Fleet Node ControlStreams are explicitly gated: +Fleet Node ControlStreams are also rejected on a passive Fleet instance as a +safety boundary: - Passive Fleet rejects ControlStreams with `not-active`. - Lease loss cancels active-scoped request contexts and closes already accepted product streams with `not-active`. -- Fleet Node clients treat `not-active` as a cheap redirect signal and reconnect quickly, with bounded jitter so large fleets do not reconnect in lockstep. -- Fleet Node transport must detect dead streams quickly enough to meet the RTO target. - -This preserves the RFC 0001 model where Fleet Nodes connect outbound to the server/Fleet app endpoint, while ensuring only the active Fleet app host owns command routing state. +Fleet Node failover, reconnect timing, and fleet-scale reconnect behavior are +not qualified by the initial supported profile. ## Health and operator status @@ -325,7 +324,7 @@ The exact installer flags, templates, compose files, and runbook commands belong | On-prem active Fleet host loses health | keepalived stops advertising the VIP; the peer advertises the same VIP after acquiring the Fleet lease and passing `/health/active`. | | On-prem active Fleet host loses VIP ownership | Active readiness fails or equivalent endpoint-adapter fencing triggers; the peer can acquire the Fleet lease and advertise the VIP. | | Endpoint routes to passive | Passive fails `/health/active` and rejects product traffic and ControlStreams. | -| Grafana fails | Real-time control continues; `/health/active` remains based on control readiness only; HA alert generation continues from control-plane status. | +| Grafana fails | Real-time control continues; `/health/active` remains based on control readiness only; local HA status remains available. | | Historical telemetry ingestion stalls | Real-time control continues; history is stale/lossy and HA status/alerts still report control-plane degraded state. | ## Validation gates @@ -341,7 +340,6 @@ Activation and fencing: - Passive mode rejects all product traffic, including non-RPC HTTP routes, while preserving the explicit health and operator-status bypasses. - Lease loss terminates already accepted product streams and active-scoped request work. - A stalled active process cannot overwrite terminal command state or renew an expired lease with its old epoch. -- Fleet-scale reconnect tests avoid synchronized ControlStream reconnect storms during failover. Database and durability: @@ -363,7 +361,7 @@ Deployment and diagnostics: - Same-subnet on-prem validation proves the configured VIP is unused, moves only to the Fleet app host passing `/health/active`, remains reachable only through the intended private network, and resists unintended VIP ownership by non-HA hosts. - Same-subnet on-prem failover tests prove loss of local VIP ownership on the active host triggers active-readiness failure, lease release/expiry, or equivalent endpoint-adapter fencing so the peer can take over within the RTO target. - HA management-plane ports are restricted to HA peers and approved operator diagnostics paths; cloud or untrusted network profiles add authenticated transport. -- HA alerts fire from control-plane status for standby loss, quorum loss, active holder anomalies, replication lag, and endpoint targeting failures even when Grafana or telemetry history is unavailable. +- Local control-plane status reports standby loss, quorum loss, active holder anomalies, replication lag, and endpoint targeting failures even when Grafana or telemetry history is unavailable. - Telemetry/Grafana failures do not fail `/health/active`. - Public health endpoints do not expose HA topology, and `/health/ha` is reachable only on loopback. @@ -405,7 +403,7 @@ Deployment and diagnostics: | 1 | DB writer routing | Multi-host DSN, read-write targeting, stale pooled connection discard, failover retry classification | Fleet can reconnect to the current DB writer without HAProxy. | | 2 | Active lease | `fleet_runtime_lease`, sqlc queries, `ha.Coordinator`, `/health/active` | Fleet can decide active/passive state safely. | | 3 | Runtime supervision | Move active-only services behind a supervisor; immediate reconciler tick on activation | Passive Fleet stays warm but does not dispatch or mutate control state. | -| 4 | Passive gating and Fleet Node retry | Active-mode request gate, ControlStream `not-active`, Fleet Node fast retry and stream liveness tuning | Traffic can safely route only to active Fleet. | +| 4 | Passive gating | Active-mode request gate and ControlStream `not-active` rejection | Product traffic can safely route only to active Fleet; Fleet Node reconnect qualification is deferred. | | 5 | HA substrate | Patroni image/config, etcd, keepalived/VRRP VIP templates, peer-connectivity preflight, install/join flow | Operators can install the on-prem HA profile. | | 6 | Degraded-mode observability | Local status for standby loss, quorum loss, active count, replication lag, and failover readiness | Operators can distinguish full HA from control-only/degraded operation. | | 7 | Lab and cloud references | Repeated failover tests, partition tests, runbook, cloud deployment reference | The install mode can be marked supported after validation gates pass. | From 75bef358d14ad20f58a382824c6aeef5a7e92218 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 22:00:31 -0700 Subject: [PATCH 16/38] Exercise paused stale command completion --- deployment-files/ha/QUALIFICATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 0031fffc98..ebf1a29aa1 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -68,7 +68,7 @@ time and a short redacted evidence reference. | From a fourth non-peer, raw-probe each HA-only port before and after reboot after verifying its listener from a peer | The matching nftables drop counter increases while peer traffic remains healthy; use a temporary listener for the normally closed debug port | Pending | Pending | Pending | | Send a valid winning VRRP advertisement from a non-peer | The protocol-112 drop counter increases and VIP ownership does not change | Pending | Pending | Pending | | Fail over with PENDING command | New active dispatches the command | Pending | Pending | Pending | -| Demote a live old active with a stalled PROCESSING plugin call, recover on the peer, then release the old call | The stale transition is rejected, exactly one terminal database result remains, and later work resumes | Pending | Pending | Pending | +| SIGSTOP the old active with a stalled PROCESSING plugin call, let the peer recover it, queue the old result, then SIGCONT | The resumed stale transition is rejected, exactly one terminal database result remains, and later work resumes | Pending | Pending | Pending | | Fail over during firmware command | Transitional device state is cleared | Pending | Pending | Pending | | Send command after failover | Command succeeds on the new active | Pending | Pending | Pending | | Publish a unique MQTT curtailment target, confirm shedding starts, fail the active before completion, then restore after takeover | The peer resumes MQTT intake and retains or reasserts curtailment within 180s, then measured load follows the restoration target | Pending | Pending | Pending | From e46ba8d3b586e23c0b2ccdb3b64a1101c7bd4bc2 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 22:07:51 -0700 Subject: [PATCH 17/38] Bind HA qualification to release evidence --- deployment-files/ha/QUALIFICATION.md | 14 ++++++++++++-- docs/rfcs/0002-active-passive-fleet-ha.md | 8 ++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index ebf1a29aa1..aca95db27a 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -19,6 +19,9 @@ HA, reconnect scale, and alert delivery remain outside this support claim. | --- | --- | | Release version | Pending | | Commit SHA | Pending | +| Release bundle SHA-256 | Pending | +| `deployment-manifest.sha256` SHA-256 | Pending | +| Deployed API, client, and Patroni image IDs/digests on every host | Pending | | Architecture | arm64 | | Operating system | Debian 13 | | Page size | 4096 bytes | @@ -42,11 +45,18 @@ HA, reconnect scale, and alert delivery remain outside this support claim. /etc/proto-fleet/ha/node.env --check`. 5. Confirm exactly one active Fleet, one passive Fleet, one Patroni primary, one Patroni replica, three etcd members, and one VIP owner. +6. Confirm every host uses the recorded deployment manifest and container image + identities from the qualified release bundle. ## Failure matrix -Restore full readiness before starting each row. Record the observed recovery -time and a short redacted evidence reference. +Restore full readiness before starting each row. Run the external append-only +recorder described under Repetition and soak from before fault injection through +full recovery. Against a test device, continuously submit uniquely identified +idempotent command probes and retain database results. Fail any row on possible +active, VIP, or writable-primary overlap, a collection gap, duplicate terminal +results, or a stale transition. Record the observed recovery time and a short +redacted evidence reference. | Gate | Required result | Duration | Result | Evidence | | --- | --- | --- | --- | --- | diff --git a/docs/rfcs/0002-active-passive-fleet-ha.md b/docs/rfcs/0002-active-passive-fleet-ha.md index 0eeee5f62e..aae26ad864 100644 --- a/docs/rfcs/0002-active-passive-fleet-ha.md +++ b/docs/rfcs/0002-active-passive-fleet-ha.md @@ -9,7 +9,7 @@ Add a supported high-availability install mode for Proto Fleet where two warm Fleet app hosts share a self-managed Postgres/TimescaleDB HA cluster, but exactly one Fleet app instance is active for real-time control at a time. The active instance is selected by a Fleet-owned, epoch-fenced database lease. Postgres failover is handled by Patroni over a three-member quorum, and Fleet connects to the current writable database through a pgx/libpq-style multi-host DSN rather than a DB proxy. -This RFC deliberately scopes the initial HA promise to Fleet application control. Curtailment dispatch, command execution, schedules, MQTT curtailment intake, and the database state required for those flows must recover automatically after a single failure. Local HA status must expose failover readiness and control-plane health. Fleet Node HA and reconnect-scale qualification, alert delivery, historical telemetry, Grafana dashboards, alert history, logs, and cache-like artifacts remain outside the initial supported profile. +This RFC deliberately scopes the initial HA promise to Fleet application control. Curtailment dispatch, command execution, MQTT curtailment intake, and the database state required for those flows must recover automatically after a single failure. Local HA status must expose failover readiness and control-plane health. Schedule failover recovery, Fleet Node HA and reconnect-scale qualification, alert delivery, historical telemetry, Grafana dashboards, alert history, logs, and cache-like artifacts remain outside the initial supported profile. ## Decision summary @@ -41,8 +41,8 @@ The supported HA contract is intentionally narrower than "every subsystem stays | Class | Examples | HA guarantee | | ---- | -------- | ------------ | -| Critical control state | Curtailment events and targets, command queue/status, schedules, Fleet Node auth/pairing state, MQTT curtailment source config and runtime edge state, active Fleet lease | Durable while replication is healthy; required for RTO | -| Real-time runtime | Active Fleet app, command executor, curtailment reconciler, scheduler, MQTT subscriber | One active instance; resumes automatically on another Fleet app host | +| Critical control state | Curtailment events and targets, command queue/status, Fleet Node auth/pairing state, MQTT curtailment source config and runtime edge state, active Fleet lease | Durable while replication is healthy; required for RTO | +| Real-time runtime | Active Fleet app, command executor, curtailment reconciler, MQTT subscriber | One active instance; resumes automatically on another Fleet app host | | Local HA status | Failover readiness, active Fleet holder count, DB primary/standby health, quorum, replication lag, VIP/load-balancer target | Must be emitted independently of Grafana history; alert delivery is deferred | | Best-effort history | Raw telemetry samples, rollups, notification metric samples, Grafana dashboards, alert history, logs | May be stale, delayed, unavailable, or partially lost | | Local artifacts | Firmware files, command artifacts, cached downloads | Not v1 HA unless explicitly promoted to critical storage | @@ -232,7 +232,7 @@ Active-only work in v1 includes: - command execution and command-state repair; - telemetry polling and discovery work that feeds active control; -- schedule processing; +- schedule processing, with failover recovery qualification deferred; - curtailment reconciliation and MQTT intake; - cleanup/sweep work that mutates shared control state or external command artifacts. From b382c48282decc5fcce2e71b4d6a9860e98059d1 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 22:26:01 -0700 Subject: [PATCH 18/38] Make HA qualification evidence executable --- deployment-files/ha/QUALIFICATION.md | 8 +++++--- docs/rfcs/0002-active-passive-fleet-ha.md | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index aca95db27a..3bc6f45e7d 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -56,7 +56,9 @@ full recovery. Against a test device, continuously submit uniquely identified idempotent command probes and retain database results. Fail any row on possible active, VIP, or writable-primary overlap, a collection gap, duplicate terminal results, or a stale transition. Record the observed recovery time and a short -redacted evidence reference. +redacted evidence reference. For database isolation and failover rows, also +record host-pinned writable SQL probe results against both database hosts at +100 ms or faster; overlapping successful writes fail the gate. | Gate | Required result | Duration | Result | Evidence | | --- | --- | --- | --- | --- | @@ -75,8 +77,8 @@ redacted evidence reference. | Hold an active-only request across demotion | The old active cancels it before the peer serves active traffic | Pending | Pending | Pending | | Probe public health through the VIP | Public responses reveal no HA topology and `/api-proxy/health/ha` returns 404 | Pending | Pending | Pending | | With Grafana and telemetry ingestion stopped, stop the etcd witness | Active health remains usable and local HA status still reports degraded failover readiness | Pending | Pending | Pending | -| From a fourth non-peer, raw-probe each HA-only port before and after reboot after verifying its listener from a peer | The matching nftables drop counter increases while peer traffic remains healthy; use a temporary listener for the normally closed debug port | Pending | Pending | Pending | -| Send a valid winning VRRP advertisement from a non-peer | The protocol-112 drop counter increases and VIP ownership does not change | Pending | Pending | Pending | +| From a fourth non-peer, probe each listening HA-only service port before and after reboot | Peer traffic remains healthy and every non-peer connection is blocked | Pending | Pending | Pending | +| Send a valid winning VRRP advertisement from a non-peer | VIP ownership does not change | Pending | Pending | Pending | | Fail over with PENDING command | New active dispatches the command | Pending | Pending | Pending | | SIGSTOP the old active with a stalled PROCESSING plugin call, let the peer recover it, queue the old result, then SIGCONT | The resumed stale transition is rejected, exactly one terminal database result remains, and later work resumes | Pending | Pending | Pending | | Fail over during firmware command | Transitional device state is cleared | Pending | Pending | Pending | diff --git a/docs/rfcs/0002-active-passive-fleet-ha.md b/docs/rfcs/0002-active-passive-fleet-ha.md index aae26ad864..3e2959937f 100644 --- a/docs/rfcs/0002-active-passive-fleet-ha.md +++ b/docs/rfcs/0002-active-passive-fleet-ha.md @@ -325,7 +325,7 @@ The exact installer flags, templates, compose files, and runbook commands belong | On-prem active Fleet host loses VIP ownership | Active readiness fails or equivalent endpoint-adapter fencing triggers; the peer can acquire the Fleet lease and advertise the VIP. | | Endpoint routes to passive | Passive fails `/health/active` and rejects product traffic and ControlStreams. | | Grafana fails | Real-time control continues; `/health/active` remains based on control readiness only; local HA status remains available. | -| Historical telemetry ingestion stalls | Real-time control continues; history is stale/lossy and HA status/alerts still report control-plane degraded state. | +| Historical telemetry ingestion stalls | Real-time control continues; history is stale/lossy and local HA status reports control-plane degraded state. | ## Validation gates From b954db183ea0244f0af66b7c55a79c27a094d7b5 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 23:03:27 -0700 Subject: [PATCH 19/38] Make power-loss qualification evidence explicit --- deployment-files/ha/QUALIFICATION.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 3bc6f45e7d..28b42c98ba 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -101,6 +101,10 @@ controller. Continuously record each host's direct active-health result and interface-address events with synchronized monotonic timestamps at 100 ms or faster; evidence stored only inside the HA database is insufficient. Treat an unreachable host, write failure, or gap longer than 250 ms as a failed soak. +For a power-off gate, also record the switched power outlet or an independent +power monitor on the same clock. Confirmed power removal closes that host's +possible active interval; network unreachability alone does not. Fail the gate +if independent power-state evidence is unavailable. Also sample both hosts at least every two seconds for availability. Record local `sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env --json --check`, and retain the status, health, and address streams. Conservatively reconstruct From 9768a98e30f47284e0cec6c517262674a472fa99 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 23:13:55 -0700 Subject: [PATCH 20/38] Close power-loss ownership intervals explicitly --- deployment-files/ha/QUALIFICATION.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 28b42c98ba..afd2a7fa3c 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -102,9 +102,10 @@ interface-address events with synchronized monotonic timestamps at 100 ms or faster; evidence stored only inside the HA database is insufficient. Treat an unreachable host, write failure, or gap longer than 250 ms as a failed soak. For a power-off gate, also record the switched power outlet or an independent -power monitor on the same clock. Confirmed power removal closes that host's -possible active interval; network unreachability alone does not. Fail the gate -if independent power-state evidence is unavailable. +power monitor on the same clock. The first confirmed power-off sample closes +that host's possible active and VIP intervals; network unreachability alone +does not. Stream termination after that sample is expected, not a collection +gap. Fail the gate if independent power-state evidence is unavailable. Also sample both hosts at least every two seconds for availability. Record local `sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env --json --check`, and retain the status, health, and address streams. Conservatively reconstruct From e6eb2586b6c7dc669e127b3ab1ee71b8936fbb55 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 23:27:05 -0700 Subject: [PATCH 21/38] Seed HA qualification ownership intervals --- deployment-files/ha/QUALIFICATION.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index afd2a7fa3c..0a7319937a 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -99,7 +99,12 @@ device effects. Device-side fencing is outside this profile's support claim. Before the soak, start an external append-only recorder on a separate controller. Continuously record each host's direct active-health result and interface-address events with synchronized monotonic timestamps at 100 ms or -faster; evidence stored only inside the HA database is insufficient. Treat an +faster; evidence stored only inside the HA database is insufficient. Before +accepting events, record a timestamped snapshot of both hosts' active health +and configured-interface addresses. Seed intervals owned at that snapshot from +the observation-window start, and extend intervals still owned at the final +snapshot through the observation-window end. Fail if either boundary state is +unknown. Treat an unreachable host, write failure, or gap longer than 250 ms as a failed soak. For a power-off gate, also record the switched power outlet or an independent power monitor on the same clock. The first confirmed power-off sample closes From 355135858fdfd08213140d840388b176721aa3a7 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 23:35:15 -0700 Subject: [PATCH 22/38] Bound HA qualification evidence uncertainty --- deployment-files/ha/QUALIFICATION.md | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 0a7319937a..69b5d3823e 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -11,7 +11,8 @@ committed report. This report qualifies the fixed Fleet application, database, DCS, and VIP profile only. Adjacent application updates are not qualified here. Fleet Node -HA, reconnect scale, and alert delivery remain outside this support claim. +HA, reconnect scale, schedule recovery during failover, and alert delivery +remain outside this support claim. ## Test identity @@ -58,7 +59,11 @@ active, VIP, or writable-primary overlap, a collection gap, duplicate terminal results, or a stale transition. Record the observed recovery time and a short redacted evidence reference. For database isolation and failover rows, also record host-pinned writable SQL probe results against both database hosts at -100 ms or faster; overlapping successful writes fail the gate. +100 ms or faster. Record every transaction's start, commit, and failure on the +controller's monotonic clock, conservatively treat the full start-to-result +window as writable, and fail if those uncertainty-inclusive intervals overlap. +After the old primary rejoins, verify every committed probe identifier exists +exactly once. | Gate | Required result | Duration | Result | Evidence | | --- | --- | --- | --- | --- | @@ -97,12 +102,14 @@ device effects. Device-side fencing is outside this profile's support claim. | Soak | 24h with no observed dual-active state or lost failover readiness | Pending | Pending | Before the soak, start an external append-only recorder on a separate -controller. Continuously record each host's direct active-health result and -interface-address events with synchronized monotonic timestamps at 100 ms or -faster; evidence stored only inside the HA database is insufficient. Before -accepting events, record a timestamped snapshot of both hosts' active health -and configured-interface addresses. Seed intervals owned at that snapshot from -the observation-window start, and extend intervals still owned at the final +controller. Start and confirm both host subscriptions first, buffering active +health samples and interface-address events with synchronized monotonic +timestamps at 100 ms or faster. Then take timestamped state snapshots, replay +all buffered events at or after each snapshot, and begin the observation +window. Fail if the collector cannot prove that subscription, snapshot, and +replay were gap-free. Evidence stored only inside the HA database is +insufficient. Seed intervals owned at the initial snapshot from the +observation-window start, and extend intervals still owned at the final snapshot through the observation-window end. Fail if either boundary state is unknown. Treat an unreachable host, write failure, or gap longer than 250 ms as a failed soak. From c48d08cd2401d10e0ab707b6aa78338fbbf65b97 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 23:45:03 -0700 Subject: [PATCH 23/38] Coordinate HA qualification ownership probes --- deployment-files/ha/QUALIFICATION.md | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 69b5d3823e..c52ca92666 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -59,11 +59,13 @@ active, VIP, or writable-primary overlap, a collection gap, duplicate terminal results, or a stale transition. Record the observed recovery time and a short redacted evidence reference. For database isolation and failover rows, also record host-pinned writable SQL probe results against both database hosts at -100 ms or faster. Record every transaction's start, commit, and failure on the -controller's monotonic clock, conservatively treat the full start-to-result -window as writable, and fail if those uncertainty-inclusive intervals overlap. -After the old primary rejoins, verify every committed probe identifier exists -exactly once. +100 ms or faster. In each cycle, use a controller barrier to begin transactions +on both hosts and hold them open concurrently before attempting uniquely +identified writes. Treat an ambiguous result as potentially committed and fail +if both hosts can commit in the same overlapping cycle. Record transaction +start, commit, and failure bounds on the controller clock. After the old primary +rejoins, verify every possibly committed probe identifier exists at most once +and every acknowledged identifier exists exactly once. | Gate | Required result | Duration | Result | Evidence | | --- | --- | --- | --- | --- | @@ -103,11 +105,15 @@ device effects. Device-side fencing is outside this profile's support claim. Before the soak, start an external append-only recorder on a separate controller. Start and confirm both host subscriptions first, buffering active -health samples and interface-address events with synchronized monotonic -timestamps at 100 ms or faster. Then take timestamped state snapshots, replay -all buffered events at or after each snapshot, and begin the observation -window. Fail if the collector cannot prove that subscription, snapshot, and -replay were gap-free. Evidence stored only inside the HA database is +health samples and interface-address events at 100 ms or faster. Timestamp all +probe sends, responses, and event arrivals on that controller's monotonic clock; +do not compare host clocks. Treat ownership as beginning no later than the last +preceding non-owner probe started and ending no earlier than the first following +non-owner probe completed. Include measured request and delivery error in those +bounds. Then take timestamped state snapshots, replay all buffered +events that can intersect each snapshot, and begin the observation window. Fail +if the collector cannot prove that subscription, snapshot, replay, and timing +bounds were gap-free. Evidence stored only inside the HA database is insufficient. Seed intervals owned at the initial snapshot from the observation-window start, and extend intervals still owned at the final snapshot through the observation-window end. Fail if either boundary state is From 8aefcd7238b2280c5a2365fbc24caf1400b8b791 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 23:54:27 -0700 Subject: [PATCH 24/38] Tighten HA qualification boundaries --- deployment-files/ha/QUALIFICATION.md | 29 ++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index c52ca92666..6a361e2e52 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -10,9 +10,10 @@ addresses, certificates, passwords, device names, or customer data in the committed report. This report qualifies the fixed Fleet application, database, DCS, and VIP -profile only. Adjacent application updates are not qualified here. Fleet Node -HA, reconnect scale, schedule recovery during failover, and alert delivery -remain outside this support claim. +profile at the recorded fleet scale and topology only. Adjacent application +updates are not qualified here. Fleet Node HA, larger fleets, reconnect scale, +schedule recovery during failover, and alert delivery remain outside this +support claim. ## Test identity @@ -27,6 +28,9 @@ remain outside this support claim. | Operating system | Debian 13 | | Page size | 4096 bytes | | Docker, containerd, keepalived, nftables, and arping package versions | Pending | +| Test miner count and plugin mix | Pending | +| Miner connection topology | Pending | +| Command backlog at curtailment | Pending | | Started | Pending | | Completed | Pending | @@ -59,13 +63,15 @@ active, VIP, or writable-primary overlap, a collection gap, duplicate terminal results, or a stale transition. Record the observed recovery time and a short redacted evidence reference. For database isolation and failover rows, also record host-pinned writable SQL probe results against both database hosts at -100 ms or faster. In each cycle, use a controller barrier to begin transactions -on both hosts and hold them open concurrently before attempting uniquely -identified writes. Treat an ambiguous result as potentially committed and fail -if both hosts can commit in the same overlapping cycle. Record transaction -start, commit, and failure bounds on the controller clock. After the old primary -rejoins, verify every possibly committed probe identifier exists at most once -and every acknowledged identifier exists exactly once. +100 ms or faster. Before isolating the old primary, begin a writable transaction +on it, write a unique probe identifier, and hold the transaction open. After the +peer is confirmed promoted and writable, begin a fresh transaction there and +write a different identifier. Use a controller barrier to attempt both commits +concurrently. Treat an ambiguous result as potentially committed and fail if +both transactions can commit. Record transaction start, promotion, commit, and +failure bounds on the controller clock. After the old primary rejoins, verify +every possibly committed probe identifier exists at most once and every +acknowledged identifier exists exactly once. | Gate | Required result | Duration | Result | Evidence | | --- | --- | --- | --- | --- | @@ -92,6 +98,9 @@ and every acknowledged identifier exists exactly once. | Send command after failover | Command succeeds on the new active | Pending | Pending | Pending | | Publish a unique MQTT curtailment target, confirm shedding starts, fail the active before completion, then restore after takeover | The peer resumes MQTT intake and retains or reasserts curtailment within 180s, then measured load follows the restoration target | Pending | Pending | Pending | +The 180-second curtailment result applies only to the miner count, plugin mix, +connection topology, and command backlog recorded above. + The PROCESSING-command gate proves server-side recovery, not exactly-once device effects. Device-side fencing is outside this profile's support claim. From 473fc3bff8c52c03a8f03153e48e7b26d63414a6 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 00:05:53 -0700 Subject: [PATCH 25/38] State HA network trust boundary --- deployment-files/ha/QUALIFICATION.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 6a361e2e52..8104c2844d 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -15,6 +15,11 @@ updates are not qualified here. Fleet Node HA, larger fleets, reconnect scale, schedule recovery during failover, and alert delivery remain outside this support claim. +The HA segment must be restricted to the three HA hosts and trusted network +infrastructure. This profile does not defend against a compromised same-L2 +host spoofing a peer address or claiming the VIP through ARP; environments +without an enforced trusted segment are unsupported. + ## Test identity | Field | Value | @@ -28,6 +33,7 @@ support claim. | Operating system | Debian 13 | | Page size | 4096 bytes | | Docker, containerd, keepalived, nftables, and arping package versions | Pending | +| HA segment isolation control | Pending | | Test miner count and plugin mix | Pending | | Miner connection topology | Pending | | Command backlog at curtailment | Pending | @@ -91,7 +97,7 @@ acknowledged identifier exists exactly once. | Probe public health through the VIP | Public responses reveal no HA topology and `/api-proxy/health/ha` returns 404 | Pending | Pending | Pending | | With Grafana and telemetry ingestion stopped, stop the etcd witness | Active health remains usable and local HA status still reports degraded failover readiness | Pending | Pending | Pending | | From a fourth non-peer, probe each listening HA-only service port before and after reboot | Peer traffic remains healthy and every non-peer connection is blocked | Pending | Pending | Pending | -| Send a valid winning VRRP advertisement from a non-peer | VIP ownership does not change | Pending | Pending | Pending | +| Send a valid winning VRRP advertisement from a non-peer's assigned address | The host source filter rejects it and VIP ownership does not change | Pending | Pending | Pending | | Fail over with PENDING command | New active dispatches the command | Pending | Pending | Pending | | SIGSTOP the old active with a stalled PROCESSING plugin call, let the peer recover it, queue the old result, then SIGCONT | The resumed stale transition is rejected, exactly one terminal database result remains, and later work resumes | Pending | Pending | Pending | | Fail over during firmware command | Transitional device state is cleared | Pending | Pending | Pending | From 8992c61b1cf95fc3125e4beecf9ab3f918b3115a Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 02:32:07 -0700 Subject: [PATCH 26/38] Scope HA qualification evidence --- deployment-files/ha/QUALIFICATION.md | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 8104c2844d..27ec019b45 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -10,10 +10,10 @@ addresses, certificates, passwords, device names, or customer data in the committed report. This report qualifies the fixed Fleet application, database, DCS, and VIP -profile at the recorded fleet scale and topology only. Adjacent application -updates are not qualified here. Fleet Node HA, larger fleets, reconnect scale, -schedule recovery during failover, and alert delivery remain outside this -support claim. +profile only on the recorded host hardware, fleet scale, and topology. +Adjacent application updates are not qualified here. Other host hardware, +Fleet Node HA, larger fleets, reconnect scale, schedule recovery during +failover, and alert delivery remain outside this support claim. The HA segment must be restricted to the three HA hosts and trusted network infrastructure. This profile does not defend against a compromised same-L2 @@ -30,7 +30,11 @@ without an enforced trusted segment are unsupported. | `deployment-manifest.sha256` SHA-256 | Pending | | Deployed API, client, and Patroni image IDs/digests on every host | Pending | | Architecture | arm64 | +| Host model and board revision | Pending | +| CPU, memory, and boot storage | Pending | +| Ethernet controller and driver | Pending | | Operating system | Debian 13 | +| Kernel and firmware versions | Pending | | Page size | 4096 bytes | | Docker, containerd, keepalived, nftables, and arping package versions | Pending | | HA segment isolation control | Pending | @@ -102,7 +106,10 @@ acknowledged identifier exists exactly once. | SIGSTOP the old active with a stalled PROCESSING plugin call, let the peer recover it, queue the old result, then SIGCONT | The resumed stale transition is rejected, exactly one terminal database result remains, and later work resumes | Pending | Pending | Pending | | Fail over during firmware command | Transitional device state is cleared | Pending | Pending | Pending | | Send command after failover | Command succeeds on the new active | Pending | Pending | Pending | -| Publish a unique MQTT curtailment target, confirm shedding starts, fail the active before completion, then restore after takeover | The peer resumes MQTT intake and retains or reasserts curtailment within 180s, then measured load follows the restoration target | Pending | Pending | Pending | +| Publish a unique MQTT curtailment target, confirm shedding starts, then kill the active Fleet process before completion | The peer resumes MQTT intake and retains or reasserts curtailment within 180s | Pending | Pending | Pending | +| Publish a unique MQTT curtailment target, confirm shedding starts, then abruptly lose the database primary while Fleet remains active | A writable primary recovers and Fleet retains or reasserts curtailment within 180s | Pending | Pending | Pending | +| Publish a unique MQTT curtailment target, confirm shedding starts, then power off a host that is both active Fleet and Patroni primary | The peer restores writable VIP service and retains or reasserts curtailment within 180s | Pending | Pending | Pending | +| Restore the MQTT curtailment target after each failure above | Measured load follows the restoration target | Pending | Pending | Pending | The 180-second curtailment result applies only to the miner count, plugin mix, connection topology, and command backlog recorded above. @@ -150,8 +157,9 @@ after the soak. ## Verdict -**Pending.** Do not describe an artifact and architecture as supported until -every result above is `PASS` and the report records the tested release, commit, -and host package versions. A package-version change requires requalification. -This initial Raspberry Pi run qualifies arm64 only; amd64 remains unqualified -until the same gates pass for its release artifact. +**Pending.** Do not describe an artifact and hardware profile as supported +until every result above is `PASS` and the report records the tested release, +commit, host hardware, and package versions. A hardware or package-version +change requires requalification. This initial Raspberry Pi run does not make +an architecture-wide arm64 claim; amd64 and other arm64 hosts remain +unqualified until the same gates pass on that hardware. From f7e152cbd178a80f443830df77324c165ffad13d Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 02:59:29 -0700 Subject: [PATCH 27/38] Require durable command qualification evidence --- deployment-files/ha/QUALIFICATION.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 27ec019b45..9ed2f389b3 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -68,10 +68,15 @@ without an enforced trusted segment are unsupported. Restore full readiness before starting each row. Run the external append-only recorder described under Repetition and soak from before fault injection through full recovery. Against a test device, continuously submit uniquely identified -idempotent command probes and retain database results. Fail any row on possible -active, VIP, or writable-primary overlap, a collection gap, duplicate terminal -results, or a stale transition. Record the observed recovery time and a short -redacted evidence reference. For database isolation and failover rows, also +idempotent command probes and keep a controller-side ledger of every +acknowledged or durably observed command ID. Require each ID to retain exactly +one database record and reach SUCCESS or a documented failover-related FAILED +state within 60 seconds. Treat missing, duplicate, PENDING, or PROCESSING rows +after that bound as failures; record ambiguous submissions separately rather +than counting them as acknowledged. Also fail any row on possible active, VIP, +or writable-primary overlap, a collection gap, or a stale transition. Record +the observed recovery time and a short redacted evidence reference. For +database isolation and failover rows, also record host-pinned writable SQL probe results against both database hosts at 100 ms or faster. Before isolating the old primary, begin a writable transaction on it, write a unique probe identifier, and hold the transaction open. After the From a0aea9a2473388250dffc1796724060bb54432fa Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 03:36:38 -0700 Subject: [PATCH 28/38] Tighten command recovery qualification --- deployment-files/ha/QUALIFICATION.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 9ed2f389b3..9c774df958 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -69,11 +69,13 @@ Restore full readiness before starting each row. Run the external append-only recorder described under Repetition and soak from before fault injection through full recovery. Against a test device, continuously submit uniquely identified idempotent command probes and keep a controller-side ledger of every -acknowledged or durably observed command ID. Require each ID to retain exactly -one database record and reach SUCCESS or a documented failover-related FAILED -state within 60 seconds. Treat missing, duplicate, PENDING, or PROCESSING rows -after that bound as failures; record ambiguous submissions separately rather -than counting them as acknowledged. Also fail any row on possible active, VIP, +acknowledged or durably observed command ID, including its state at fault +injection. Require every PENDING ID to reach SUCCESS within 60 seconds. Permit +FAILED only for an ID already PROCESSING at fault injection and only with the +expected interruption reason. Treat missing, duplicate, PENDING, or PROCESSING +rows after that bound as failures; record ambiguous submissions separately +rather than counting them as acknowledged. Also fail any row on possible +active, VIP, or writable-primary overlap, a collection gap, or a stale transition. Record the observed recovery time and a short redacted evidence reference. For database isolation and failover rows, also From 0befe9c03c18ebc289c19dc0dd9becf98fcb3667 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 03:48:43 -0700 Subject: [PATCH 29/38] Observe active work during HA qualification --- deployment-files/ha/QUALIFICATION.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 9c774df958..dd1a266b23 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -78,6 +78,10 @@ rather than counting them as acknowledged. Also fail any row on possible active, VIP, or writable-primary overlap, a collection gap, or a stale transition. Record the observed recovery time and a short redacted evidence reference. For +the controlled test device, independently record every command, plugin, and +curtailment request from acceptance through response or connection close, +including its source host. Fail if work from the old holder remains in flight +when work from the new holder begins. For database isolation and failover rows, also record host-pinned writable SQL probe results against both database hosts at 100 ms or faster. Before isolating the old primary, begin a writable transaction @@ -105,6 +109,7 @@ acknowledged identifier exists exactly once. | Fail one active-runtime job without killing Fleet | Active health fails, the process exits, and the peer takes over | Pending | Pending | Pending | | Send Connect RPC, non-RPC HTTP, and ControlStream traffic directly to the passive host | Each product transport rejects it as not active; only health and local status remain available | Pending | Pending | Pending | | Hold an active-only request across demotion | The old active cancels it before the peer serves active traffic | Pending | Pending | Pending | +| Record active-only device work across failover | No old-holder request remains in flight when new-holder work begins | Pending | Pending | Pending | | Probe public health through the VIP | Public responses reveal no HA topology and `/api-proxy/health/ha` returns 404 | Pending | Pending | Pending | | With Grafana and telemetry ingestion stopped, stop the etcd witness | Active health remains usable and local HA status still reports degraded failover readiness | Pending | Pending | Pending | | From a fourth non-peer, probe each listening HA-only service port before and after reboot | Peer traffic remains healthy and every non-peer connection is blocked | Pending | Pending | Pending | @@ -159,8 +164,9 @@ and retain the status, health, and address streams. Conservatively reconstruct each active interval from its last preceding passive sample through its first following passive sample, and each VIP interval from address-add through address-delete. Fail when these uncertainty-inclusive host intervals could -overlap, or on any collection gap or lost readiness. Export redacted evidence -after the soak. +overlap, or on any collection gap or lost readiness. Keep the controlled test +device recorder running and apply the same old-holder versus new-holder work +interval rule throughout the soak. Export redacted evidence after the soak. ## Verdict From 02cc55e33b7b88e2e91e7465efb659e3cc8f4cd1 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 04:11:22 -0700 Subject: [PATCH 30/38] Qualify HA effect exclusivity without polling claims --- deployment-files/ha/QUALIFICATION.md | 41 +++++++++++++--------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index dd1a266b23..ea855130ea 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -74,9 +74,9 @@ injection. Require every PENDING ID to reach SUCCESS within 60 seconds. Permit FAILED only for an ID already PROCESSING at fault injection and only with the expected interruption reason. Treat missing, duplicate, PENDING, or PROCESSING rows after that bound as failures; record ambiguous submissions separately -rather than counting them as acknowledged. Also fail any row on possible -active, VIP, -or writable-primary overlap, a collection gap, or a stale transition. Record +rather than counting them as acknowledged. Also fail any row on possible VIP +or writable-primary overlap, overlapping active-only device work, a collection +gap, or a stale transition. Record the observed recovery time and a short redacted evidence reference. For the controlled test device, independently record every command, plugin, and curtailment request from acceptance through response or connection close, @@ -135,24 +135,21 @@ device effects. Device-side fencing is outside this profile's support claim. | --- | --- | --- | --- | | Application failover | 5 consecutive passes | Pending | Pending | | Database failover | 3 consecutive passes | Pending | Pending | -| Soak | 24h with no observed dual-active state or lost failover readiness | Pending | Pending | +| Soak | 24h with no overlapping active-only device work, VIP overlap, or lost failover readiness | Pending | Pending | Before the soak, start an external append-only recorder on a separate -controller. Start and confirm both host subscriptions first, buffering active -health samples and interface-address events at 100 ms or faster. Timestamp all +controller. Start and confirm both host subscriptions first, buffering +interface-address events. Timestamp all probe sends, responses, and event arrivals on that controller's monotonic clock; -do not compare host clocks. Treat ownership as beginning no later than the last -preceding non-owner probe started and ending no earlier than the first following -non-owner probe completed. Include measured request and delivery error in those -bounds. Then take timestamped state snapshots, replay all buffered -events that can intersect each snapshot, and begin the observation window. Fail -if the collector cannot prove that subscription, snapshot, replay, and timing -bounds were gap-free. Evidence stored only inside the HA database is +do not compare host clocks. Then take timestamped state snapshots, replay all +buffered events that can intersect each snapshot, and begin the observation +window. Fail if the collector cannot prove that subscription, snapshot, replay, +and timing bounds were gap-free. Evidence stored only inside the HA database is insufficient. Seed intervals owned at the initial snapshot from the observation-window start, and extend intervals still owned at the final snapshot through the observation-window end. Fail if either boundary state is -unknown. Treat an -unreachable host, write failure, or gap longer than 250 ms as a failed soak. +unknown. Treat an unreachable host, write failure, or address-event collection +gap as a failed soak. For a power-off gate, also record the switched power outlet or an independent power monitor on the same clock. The first confirmed power-off sample closes that host's possible active and VIP intervals; network unreachability alone @@ -160,13 +157,13 @@ does not. Stream termination after that sample is expected, not a collection gap. Fail the gate if independent power-state evidence is unavailable. Also sample both hosts at least every two seconds for availability. Record local `sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env --json --check`, -and retain the status, health, and address streams. Conservatively reconstruct -each active interval from its last preceding passive sample through its first -following passive sample, and each VIP interval from address-add through -address-delete. Fail when these uncertainty-inclusive host intervals could -overlap, or on any collection gap or lost readiness. Keep the controlled test -device recorder running and apply the same old-holder versus new-holder work -interval rule throughout the soak. Export redacted evidence after the soak. +and retain the status, health, and address streams. Reconstruct each VIP +interval from address-add through address-delete and fail if host intervals +overlap. Health polling is availability evidence, not proof that every runtime +transition was observed. Prove effect exclusivity with the controlled test +device recorder instead: it must capture every request, and no old-holder work +may remain in flight when new-holder work begins. Any recorder gap fails the +soak. Export redacted evidence after the soak. ## Verdict From fb6ce57844c8a2197f35bf9d6ef0e565739319f2 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 04:19:08 -0700 Subject: [PATCH 31/38] Bound post-failover curtailment restoration --- deployment-files/ha/QUALIFICATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index ea855130ea..9fc199a256 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -121,7 +121,7 @@ acknowledged identifier exists exactly once. | Publish a unique MQTT curtailment target, confirm shedding starts, then kill the active Fleet process before completion | The peer resumes MQTT intake and retains or reasserts curtailment within 180s | Pending | Pending | Pending | | Publish a unique MQTT curtailment target, confirm shedding starts, then abruptly lose the database primary while Fleet remains active | A writable primary recovers and Fleet retains or reasserts curtailment within 180s | Pending | Pending | Pending | | Publish a unique MQTT curtailment target, confirm shedding starts, then power off a host that is both active Fleet and Patroni primary | The peer restores writable VIP service and retains or reasserts curtailment within 180s | Pending | Pending | Pending | -| Restore the MQTT curtailment target after each failure above | Measured load follows the restoration target | Pending | Pending | Pending | +| Publish a distinct MQTT restoration target after each failure above | Measured load follows the new target within 180s | Pending | Pending | Pending | The 180-second curtailment result applies only to the miner count, plugin mix, connection topology, and command backlog recorded above. From 12d82b355d7b3715eee6bd838d8ab76e5c702f46 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 04:30:43 -0700 Subject: [PATCH 32/38] Keep HA qualification evidence gap-free --- deployment-files/ha/QUALIFICATION.md | 15 +++++++++------ docs/rfcs/0002-active-passive-fleet-ha.md | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 9fc199a256..63f7deab15 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -157,12 +157,15 @@ does not. Stream termination after that sample is expected, not a collection gap. Fail the gate if independent power-state evidence is unavailable. Also sample both hosts at least every two seconds for availability. Record local `sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env --json --check`, -and retain the status, health, and address streams. Reconstruct each VIP -interval from address-add through address-delete and fail if host intervals -overlap. Health polling is availability evidence, not proof that every runtime -transition was observed. Prove effect exclusivity with the controlled test -device recorder instead: it must capture every request, and no old-holder work -may remain in flight when new-holder work begins. Any recorder gap fails the +and retain the status, health, and address streams. Run this probe outside shell +error-exit handling: capture stdout, stderr, and exit status, then continue +after the expected nonzero result for degraded failover readiness. Missing or +invalid JSON is a collection failure, not expected degradation. Reconstruct +each VIP interval from address-add through address-delete and fail if host +intervals overlap. Health polling is availability evidence, not proof that +every runtime transition was observed. Prove effect exclusivity with the +controlled test device recorder instead: it must capture every request, and no +old-holder work may remain in flight when new-holder work begins. Any recorder gap fails the soak. Export redacted evidence after the soak. ## Verdict diff --git a/docs/rfcs/0002-active-passive-fleet-ha.md b/docs/rfcs/0002-active-passive-fleet-ha.md index 3e2959937f..11babacd17 100644 --- a/docs/rfcs/0002-active-passive-fleet-ha.md +++ b/docs/rfcs/0002-active-passive-fleet-ha.md @@ -43,7 +43,7 @@ The supported HA contract is intentionally narrower than "every subsystem stays | ---- | -------- | ------------ | | Critical control state | Curtailment events and targets, command queue/status, Fleet Node auth/pairing state, MQTT curtailment source config and runtime edge state, active Fleet lease | Durable while replication is healthy; required for RTO | | Real-time runtime | Active Fleet app, command executor, curtailment reconciler, MQTT subscriber | One active instance; resumes automatically on another Fleet app host | -| Local HA status | Failover readiness, active Fleet holder count, DB primary/standby health, quorum, replication lag, VIP/load-balancer target | Must be emitted independently of Grafana history; alert delivery is deferred | +| Local HA status | Runtime role, observation freshness, endpoint health, control and failover readiness, and generic reason codes | Must be emitted independently of Grafana history; alert delivery is deferred | | Best-effort history | Raw telemetry samples, rollups, notification metric samples, Grafana dashboards, alert history, logs | May be stale, delayed, unavailable, or partially lost | | Local artifacts | Firmware files, command artifacts, cached downloads | Not v1 HA unless explicitly promoted to critical storage | From a06edbb93aacc8645df5493f0ff608c38777cafd Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 07:07:29 -0700 Subject: [PATCH 33/38] Qualify curtailment across every failover --- deployment-files/ha/QUALIFICATION.md | 37 +++++++++++++++------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 63f7deab15..4ebad9d0d7 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -67,8 +67,8 @@ without an enforced trusted segment are unsupported. Restore full readiness before starting each row. Run the external append-only recorder described under Repetition and soak from before fault injection through -full recovery. Against a test device, continuously submit uniquely identified -idempotent command probes and keep a controller-side ledger of every +full recovery. Across the complete qualified test fleet, continuously submit +uniquely identified idempotent command probes and keep a controller-side ledger of every acknowledged or durably observed command ID, including its state at fault injection. Require every PENDING ID to reach SUCCESS within 60 seconds. Permit FAILED only for an ID already PROCESSING at fault injection and only with the @@ -76,13 +76,20 @@ expected interruption reason. Treat missing, duplicate, PENDING, or PROCESSING rows after that bound as failures; record ambiguous submissions separately rather than counting them as acknowledged. Also fail any row on possible VIP or writable-primary overlap, overlapping active-only device work, a collection -gap, or a stale transition. Record -the observed recovery time and a short redacted evidence reference. For -the controlled test device, independently record every command, plugin, and -curtailment request from acceptance through response or connection close, -including its source host. Fail if work from the old holder remains in flight -when work from the new holder begins. For -database isolation and failover rows, also +gap, or a stale transition. Record the observed recovery time and a short +redacted evidence reference. Route or instrument every qualified test miner so +the external recorder captures every command, plugin, and curtailment request +from acceptance through response or connection close, including the device and +source host. Fail on a recorder gap or if work from the old holder remains in +flight when work from the new holder begins anywhere in the test fleet. + +For every row that requires Fleet takeover or PostgreSQL writer promotion, +publish a unique MQTT curtailment target and confirm shedding has started before +injecting the fault. Require MQTT intake to resume, the target to be retained or +reasserted, and measured load to reach the exact target within 180 seconds. +Then publish a distinct restoration target and require measured load to follow +it within 180 seconds. Apply these bounds in addition to the row's service +recovery bound. For database isolation and failover rows, also record host-pinned writable SQL probe results against both database hosts at 100 ms or faster. Before isolating the old primary, begin a writable transaction on it, write a unique probe identifier, and hold the transaction open. After the @@ -118,11 +125,6 @@ acknowledged identifier exists exactly once. | SIGSTOP the old active with a stalled PROCESSING plugin call, let the peer recover it, queue the old result, then SIGCONT | The resumed stale transition is rejected, exactly one terminal database result remains, and later work resumes | Pending | Pending | Pending | | Fail over during firmware command | Transitional device state is cleared | Pending | Pending | Pending | | Send command after failover | Command succeeds on the new active | Pending | Pending | Pending | -| Publish a unique MQTT curtailment target, confirm shedding starts, then kill the active Fleet process before completion | The peer resumes MQTT intake and retains or reasserts curtailment within 180s | Pending | Pending | Pending | -| Publish a unique MQTT curtailment target, confirm shedding starts, then abruptly lose the database primary while Fleet remains active | A writable primary recovers and Fleet retains or reasserts curtailment within 180s | Pending | Pending | Pending | -| Publish a unique MQTT curtailment target, confirm shedding starts, then power off a host that is both active Fleet and Patroni primary | The peer restores writable VIP service and retains or reasserts curtailment within 180s | Pending | Pending | Pending | -| Publish a distinct MQTT restoration target after each failure above | Measured load follows the new target within 180s | Pending | Pending | Pending | - The 180-second curtailment result applies only to the miner count, plugin mix, connection topology, and command backlog recorded above. @@ -164,9 +166,10 @@ invalid JSON is a collection failure, not expected degradation. Reconstruct each VIP interval from address-add through address-delete and fail if host intervals overlap. Health polling is availability evidence, not proof that every runtime transition was observed. Prove effect exclusivity with the -controlled test device recorder instead: it must capture every request, and no -old-holder work may remain in flight when new-holder work begins. Any recorder gap fails the -soak. Export redacted evidence after the soak. +fleet-wide recorder instead: it must capture every request for every qualified +test miner, and no old-holder work may remain in flight when new-holder work +begins anywhere in the fleet. Any recorder gap fails the soak. Export redacted +evidence after the soak. ## Verdict From 4599d9609257ced9dc0b565c00fa5f8c65f2b613 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 07:13:07 -0700 Subject: [PATCH 34/38] Qualify curtailment during degraded operation --- deployment-files/ha/QUALIFICATION.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 4ebad9d0d7..87f753d5da 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -83,13 +83,15 @@ from acceptance through response or connection close, including the device and source host. Fail on a recorder gap or if work from the old holder remains in flight when work from the new holder begins anywhere in the test fleet. -For every row that requires Fleet takeover or PostgreSQL writer promotion, -publish a unique MQTT curtailment target and confirm shedding has started before -injecting the fault. Require MQTT intake to resume, the target to be retained or -reasserted, and measured load to reach the exact target within 180 seconds. -Then publish a distinct restoration target and require measured load to follow -it within 180 seconds. Apply these bounds in addition to the row's service -recovery bound. For database isolation and failover rows, also +For every row that injects a Fleet, PostgreSQL, etcd, or VIP failure, publish a +unique MQTT curtailment target and confirm shedding has started before injecting +the fault. Keep the fault active while requiring MQTT intake to continue or +resume, the target to be retained or reasserted, and measured load to reach the +exact target within 180 seconds. Then publish a distinct restoration target and +require measured load to follow it within 180 seconds. Apply these bounds +whether the fault causes takeover, writer promotion, or degraded readiness, and +in addition to the row's service recovery bound. For database isolation and +failover rows, also record host-pinned writable SQL probe results against both database hosts at 100 ms or faster. Before isolating the old primary, begin a writable transaction on it, write a unique probe identifier, and hold the transaction open. After the From fb83e15f93587c58c8a7b4eeacb11779513b8a21 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 07:53:12 -0700 Subject: [PATCH 35/38] Make HA qualification claims conservative --- deployment-files/ha/QUALIFICATION.md | 38 ++++++++++++++++------------ 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index 87f753d5da..aa160a62fa 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -143,31 +143,37 @@ device effects. Device-side fencing is outside this profile's support claim. Before the soak, start an external append-only recorder on a separate controller. Start and confirm both host subscriptions first, buffering -interface-address events. Timestamp all -probe sends, responses, and event arrivals on that controller's monotonic clock; -do not compare host clocks. Then take timestamped state snapshots, replay all -buffered events that can intersect each snapshot, and begin the observation -window. Fail if the collector cannot prove that subscription, snapshot, replay, -and timing bounds were gap-free. Evidence stored only inside the HA database is -insufficient. Seed intervals owned at the initial snapshot from the -observation-window start, and extend intervals still owned at the final -snapshot through the observation-window end. Fail if either boundary state is -unknown. Treat an unreachable host, write failure, or address-event collection -gap as a failed soak. +interface-address events. Timestamp every probe send, response, and event +arrival on that controller's monotonic clock; do not compare host clocks. Then +take timestamped state snapshots, replay all buffered events that can intersect +each snapshot, and begin the observation window. Fail if the collector cannot +prove that subscription, snapshot, replay, and timing bounds were gap-free. +Evidence stored only inside the HA database is insufficient. An event's receipt +time is not its occurrence time. For each host, conservatively bound every +possible VIP-owned interval from the send time of the last probe that confirmed +the VIP absent through the receipt time of the first later probe or event that +confirmed it absent again. Seed an initially owned interval at the observation +window start and extend a finally owned interval through the observation window +end. During each injected failure and VIP move, probe both hosts at least every +100 milliseconds; uncertainty that makes the two hosts' possible-owned +intervals overlap fails the gate rather than being resolved by event arrival +order. Fail if either boundary state is unknown. Treat an unreachable host, +write failure, or address-event collection gap as a failed soak. For a power-off gate, also record the switched power outlet or an independent power monitor on the same clock. The first confirmed power-off sample closes that host's possible active and VIP intervals; network unreachability alone does not. Stream termination after that sample is expected, not a collection gap. Fail the gate if independent power-state evidence is unavailable. -Also sample both hosts at least every two seconds for availability. Record local +Outside injected transitions, sample both hosts at least every two seconds for +availability. Record local `sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env --json --check`, and retain the status, health, and address streams. Run this probe outside shell error-exit handling: capture stdout, stderr, and exit status, then continue after the expected nonzero result for degraded failover readiness. Missing or -invalid JSON is a collection failure, not expected degradation. Reconstruct -each VIP interval from address-add through address-delete and fail if host -intervals overlap. Health polling is availability evidence, not proof that -every runtime transition was observed. Prove effect exclusivity with the +invalid JSON is a collection failure, not expected degradation. Fail if the +conservative possible-owned intervals overlap. Health polling is availability +evidence, not proof that every runtime transition was observed. Prove effect +exclusivity with the fleet-wide recorder instead: it must capture every request for every qualified test miner, and no old-holder work may remain in flight when new-holder work begins anywhere in the fleet. Any recorder gap fails the soak. Export redacted From 858702451403b2d4702ef1229fa85f0494d0dcad Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Mon, 10 Aug 2026 12:18:38 -0700 Subject: [PATCH 36/38] Qualify supported HA install platforms --- deployment-files/ha/QUALIFICATION.md | 32 +++++++++++++++++----------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/deployment-files/ha/QUALIFICATION.md b/deployment-files/ha/QUALIFICATION.md index aa160a62fa..b15862f38e 100644 --- a/deployment-files/ha/QUALIFICATION.md +++ b/deployment-files/ha/QUALIFICATION.md @@ -2,8 +2,17 @@ The tested release artifact, host package versions, and architecture are supported only after they pass every gate in this document on three clean, -same-L2 Debian hosts. End each -measurement at the gate's actual recovery signal: VIP health for routing, +same-L2 hosts. Complete and publish one report for each required platform: + +| Operating system | Architecture | Hardware | +| --- | --- | --- | +| Debian 12 or 13 | arm64 | Raspberry Pi | +| Ubuntu 22.04 or 24.04 | amd64 | Qualified host model | +| 64-bit Raspberry Pi OS based on Debian 12 or 13 | arm64 | Raspberry Pi | + +Other derivatives remain installer-compatible but unqualified. Do not combine +results across platform reports. End each measurement at the gate's actual +recovery signal: VIP health for routing, writable SQL for database recovery, durable command/device state for commands, and telemetry or independent power measurement for curtailment. Do not include addresses, certificates, passwords, device names, or customer data in the @@ -29,11 +38,11 @@ without an enforced trusted segment are unsupported. | Release bundle SHA-256 | Pending | | `deployment-manifest.sha256` SHA-256 | Pending | | Deployed API, client, and Patroni image IDs/digests on every host | Pending | -| Architecture | arm64 | +| Architecture | Pending | | Host model and board revision | Pending | | CPU, memory, and boot storage | Pending | | Ethernet controller and driver | Pending | -| Operating system | Debian 13 | +| Operating system | Pending | | Kernel and firmware versions | Pending | | Page size | 4096 bytes | | Docker, containerd, keepalived, nftables, and arping package versions | Pending | @@ -46,9 +55,9 @@ without an enforced trusted segment are unsupported. ## Clean installation -1. Start with three freshly provisioned Debian 13 hosts. Do not qualify by - deleting directories from a previous HA installation; reimage the hosts so - no old services, containers, firewall rules, VIP, or data remain. +1. Start with three freshly provisioned hosts for the report's platform. Do not + qualify by deleting directories from a previous HA installation; reimage + the hosts so no old services, containers, firewall rules, VIP, or data remain. 2. Copy the same release, host-specific `node.env`, and only that node's matching secret directory to each host. Never copy the generated `offline` directory to a running host. Copy the etcd root password separately to @@ -57,7 +66,7 @@ without an enforced trusted segment are unsupported. described in [README.md](README.md). 4. Reboot all three hosts. On both database hosts, run `sudo /opt/proto-fleet/deployment/ha/fleet-ha status - /etc/proto-fleet/ha/node.env --check`. + /etc/proto-fleet/ha/node.env`. 5. Confirm exactly one active Fleet, one passive Fleet, one Patroni primary, one Patroni replica, three etcd members, and one VIP owner. 6. Confirm every host uses the recorded deployment manifest and container image @@ -166,7 +175,7 @@ does not. Stream termination after that sample is expected, not a collection gap. Fail the gate if independent power-state evidence is unavailable. Outside injected transitions, sample both hosts at least every two seconds for availability. Record local -`sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env --json --check`, +`sudo /opt/proto-fleet/deployment/ha/fleet-ha status /etc/proto-fleet/ha/node.env`, and retain the status, health, and address streams. Run this probe outside shell error-exit handling: capture stdout, stderr, and exit status, then continue after the expected nonzero result for degraded failover readiness. Missing or @@ -184,6 +193,5 @@ evidence after the soak. **Pending.** Do not describe an artifact and hardware profile as supported until every result above is `PASS` and the report records the tested release, commit, host hardware, and package versions. A hardware or package-version -change requires requalification. This initial Raspberry Pi run does not make -an architecture-wide arm64 claim; amd64 and other arm64 hosts remain -unqualified until the same gates pass on that hardware. +change requires requalification. Passing one required platform does not +qualify the other platforms or make an architecture-wide support claim. From f2644600e6ee2d213b74cda03b69174fb27d7309 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Mon, 10 Aug 2026 16:08:25 -0700 Subject: [PATCH 37/38] Align HA qualification with simplified firewall --- deployment-files/ha/tests/test-profile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment-files/ha/tests/test-profile.sh b/deployment-files/ha/tests/test-profile.sh index 0b28dab1bc..6bf189dc85 100755 --- a/deployment-files/ha/tests/test-profile.sh +++ b/deployment-files/ha/tests/test-profile.sh @@ -145,7 +145,7 @@ test_fleet_ha_contract() { assert_contains "${HA_DIR}/proto-fleet-ha-keepalived.conf" "Wants=keepalived.service" assert_contains "${HA_DIR}/keepalived-systemd.conf.tmpl" 'ExecStopPost=/usr/sbin/ip address flush to ${HA_VIRTUAL_IP}/32 dev ${HA_NETWORK_INTERFACE}' assert_not_contains "${HA_DIR}/firewall.nft.tmpl" "destroy table" - assert_contains "${HA_DIR}/firewall.nft.tmpl" "tcp dport 40000 counter drop" + assert_contains "${HA_DIR}/firewall.nft.tmpl" "tcp dport 40000 drop" assert_contains "${HA_DIR}/proto-fleet-ha-firewall.service" "ExecStart=/usr/sbin/nft -f /etc/proto-fleet/ha/firewall.nft" assert_contains "${HA_DIR}/proto-fleet-ha-firewall.service" "ExecStartPre=/usr/sbin/nft -c -f /etc/proto-fleet/ha/firewall.nft" assert_contains "${HA_DIR}/proto-fleet-ha-firewall.service" "ExecStartPre=-/usr/sbin/nft delete table inet proto_fleet_ha" From b4255da76bcb565339124a0f0e3e2a2cc31ec68a Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Tue, 11 Aug 2026 13:01:51 -0700 Subject: [PATCH 38/38] Pin qualified HA release images (#889) --- deployment-files/client/Dockerfile | 2 +- deployment-files/server/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment-files/client/Dockerfile b/deployment-files/client/Dockerfile index 82c3c52e86..1097b5dff0 100644 --- a/deployment-files/client/Dockerfile +++ b/deployment-files/client/Dockerfile @@ -1,5 +1,5 @@ # Production stage using Nginx -FROM nginx:1.29.0-alpine +FROM nginx:1.29.0-alpine@sha256:d67ea0d64d518b1bb04acde3b00f722ac3e9764b3209a9b0a98924ba35e4b779 # Copy the prebuilt client files COPY protoFleet /usr/share/nginx/html diff --git a/deployment-files/server/Dockerfile b/deployment-files/server/Dockerfile index 0e80a1cf2d..b7e78e8abc 100644 --- a/deployment-files/server/Dockerfile +++ b/deployment-files/server/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.22.0 +FROM alpine:3.22.0@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715 RUN apk add --no-cache nmap gcompat