From 9f521f138f5c67c45bcc91eb4625ebeb24852137 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 11:20:07 -0700 Subject: [PATCH 01/40] feat(ha): complete updates through failover --- deployment-files/ha/README.md | 41 +++++ server/cmd/fleet-ha/main.go | 57 +++++-- server/cmd/fleet-ha/main_test.go | 43 ++++- .../internal/domain/updates/executor_test.go | 25 +++ server/internal/ha/deployment/update.go | 148 ++++++++++++++++-- server/internal/ha/deployment/update_test.go | 34 ++++ server/internal/updater/manager.go | 60 +++++-- server/internal/updater/manager_test.go | 128 ++++++++++++++- server/internal/updater/server.go | 7 +- server/internal/updaterapi/client.go | 10 +- server/internal/updaterapi/types.go | 1 + 11 files changed, 502 insertions(+), 52 deletions(-) diff --git a/deployment-files/ha/README.md b/deployment-files/ha/README.md index d4f45b4f4d..09ec00c522 100644 --- a/deployment-files/ha/README.md +++ b/deployment-files/ha/README.md @@ -165,6 +165,47 @@ The Docker repository setup follows the official instructions for [Ubuntu](https://docs.docker.com/engine/install/ubuntu/), and [64-bit Raspberry Pi OS](https://docs.docker.com/engine/install/raspberry-pi-os/). +## Update a passive Fleet host + +HA disables application-triggered updates. On the passive database host, run: + +```bash +sudo /opt/proto-fleet/deployment/ha/fleet-ha update v0.2.11 +``` + +The local updater downloads the release from the fixed Proto Fleet GitHub +release origin, verifies its SHA256 checksum, builds and persists the staged +Fleet images, then rechecks that this host is passive. It stops and replaces +only `fleet-api` and `fleet-client`; etcd, Patroni, PostgreSQL, and keepalived +remain running. The command returns only after the target version is healthy +and passive. + +The old active and new passive share the database during this rolling window. +Every migration in the target release must be expand-only and remain compatible +with the previous release; destructive contract migrations belong in a later +release after both HA hosts have advanced. + +After the peer is confirmed on the target release, complete the update from +the old active host: + +```bash +sudo /opt/proto-fleet/deployment/ha/fleet-ha update v0.2.11 --complete +``` + +If the updated host has already become active, the old host is now passive. +Run the ordinary `fleet-ha update v0.2.11` command on that passive host instead. + +The source release must already contain this HA update protocol. HA is being +introduced for new deployments, so an older experimental HA installation that +predates `update --complete` must be reinstalled at the supported baseline +rather than upgraded through this workflow. + +The updater stages everything first, stops the local Fleet containers, and +waits for the updated peer to serve the VIP with the target version. Only then +does it swap and restart the local application as passive. If takeover does +not complete within 15 seconds, it restarts the old local release without +swapping. This is a bounded interruption, not a zero-downtime update. + ## Qualification The distributions above are installer-compatible targets. The HA profile is diff --git a/server/cmd/fleet-ha/main.go b/server/cmd/fleet-ha/main.go index 515b2f66c5..287ddbb46e 100644 --- a/server/cmd/fleet-ha/main.go +++ b/server/cmd/fleet-ha/main.go @@ -14,6 +14,7 @@ import ( "github.com/alecthomas/kong" "github.com/google/uuid" + "github.com/block/proto-fleet/server/internal/ha" "github.com/block/proto-fleet/server/internal/ha/deployment" "github.com/block/proto-fleet/server/internal/updaterapi" ) @@ -36,9 +37,11 @@ type cli struct { Start startCmd `cmd:"" help:"start installed HA services"` Stop stopCmd `cmd:"" help:"stop installed HA services"` RequirePassive requirePassiveCmd `cmd:"" help:"verify that the local Fleet instance is passive"` + RequireActive requireActiveCmd `cmd:"" help:"verify that the local Fleet instance is active"` UpdatePreflight updatePreflightCmd `cmd:"" help:"prepare the current release for an application update"` AppStop appStopCmd `cmd:"" help:"stop the Fleet application services"` AppStart appStartCmd `cmd:"" help:"start the Fleet application services"` + WaitTakeover waitTakeoverCmd `cmd:"" help:"wait for the VIP to serve an application version"` } type preflightCmd struct { @@ -110,11 +113,12 @@ func (c *installCmd) Run(ctx context.Context) error { } type updateCmd struct { - Version string `arg:"" help:"target application version"` + Version string `arg:"" help:"target application version"` + Complete bool `help:"complete the update through failover from the active host"` } func (c *updateCmd) Run(ctx context.Context) error { - return runPassiveUpdate(ctx, c.Version, os.Stdout, deployment.ValidatePassiveUpdate, updaterapi.NewClient(defaultUpdaterSocket), deployment.Status) + return runPassiveUpdate(ctx, c.Version, c.Complete, os.Stdout, validateHAUpdate, updaterapi.NewClient(defaultUpdaterSocket), deployment.Status) } type startCmd struct { @@ -143,6 +147,14 @@ func (c *requirePassiveCmd) Run(ctx context.Context) error { return deployment.ValidatePassiveUpdate(ctx, c.NodeEnv, c.Version) } +type requireActiveCmd struct { + NodeEnv string `arg:"" type:"path" help:"node environment file"` +} + +func (c *requireActiveCmd) Run(ctx context.Context) error { + return deployment.RequireActive(ctx, c.NodeEnv) +} + type updatePreflightCmd struct{} func (*updatePreflightCmd) Run(ctx context.Context) error { @@ -153,19 +165,21 @@ func (*updatePreflightCmd) Run(ctx context.Context) error { return deployment.PrepareApplicationUpdate(ctx, root) } -type appStopCmd struct{} +type appStopCmd struct { + Role ha.RuntimeRole `arg:"" enum:"passive,active" help:"expected local HA role"` +} -func (*appStopCmd) Run(ctx context.Context) error { +func (c *appStopCmd) Run(ctx context.Context) error { root, err := deployment.ReleaseRoot() if err != nil { return err } - return deployment.StopApplication(ctx, root) + return deployment.StopApplication(ctx, root, c.Role) } type appStartCmd struct { Version string `arg:"" help:"application version to start"` - Mode string `arg:"" optional:"" default:"passive" enum:"passive,any" help:"required HA role after startup"` + Mode string `arg:"" enum:"passive,any" help:"required HA role after startup"` } func (c *appStartCmd) Run(ctx context.Context) error { @@ -176,6 +190,14 @@ func (c *appStartCmd) Run(ctx context.Context) error { return deployment.StartApplication(ctx, root, c.Version, c.Mode == "passive") } +type waitTakeoverCmd struct { + Version string `arg:"" help:"application version expected on the VIP"` +} + +func (c *waitTakeoverCmd) Run(ctx context.Context) error { + return deployment.WaitForVIPVersion(ctx, installedNodeEnv, c.Version) +} + func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGHUP, syscall.SIGTERM) defer stop() @@ -214,6 +236,7 @@ func runStatus(ctx context.Context, envPath string, output io.Writer, read statu type updaterClient interface { Status(ctx context.Context) (updaterapi.StatusResponse, error) Trigger(ctx context.Context, operationID, targetVersion string) (updaterapi.Operation, error) + TriggerComplete(ctx context.Context, operationID, targetVersion string) (updaterapi.Operation, error) } const ( @@ -223,17 +246,26 @@ const ( type updateTrigger func(context.Context, string, string) (updaterapi.Operation, error) -type updatePreflight func(context.Context, string, string) error +type updatePreflight func(context.Context, string, string, bool) error + +func validateHAUpdate(ctx context.Context, envPath, targetVersion string, complete bool) error { + if complete { + _, err := deployment.ValidateActiveUpdate(ctx, envPath, targetVersion) + return err + } + return deployment.ValidatePassiveUpdate(ctx, envPath, targetVersion) +} func runPassiveUpdate( ctx context.Context, targetVersion string, + complete bool, output io.Writer, preflight updatePreflight, client updaterClient, read statusReader, ) error { - if err := runUpdate(ctx, targetVersion, output, preflight, client); err != nil { + if err := runUpdate(ctx, targetVersion, complete, output, preflight, client); err != nil { return err } report, err := read(ctx, installedNodeEnv) @@ -259,15 +291,20 @@ func runPassiveUpdate( func runUpdate( ctx context.Context, targetVersion string, + complete bool, output io.Writer, preflight updatePreflight, client updaterClient, ) error { - if err := preflight(ctx, installedNodeEnv, targetVersion); err != nil { + if err := preflight(ctx, installedNodeEnv, targetVersion, complete); err != nil { return err } operationID := uuid.NewString() - operation, err := triggerUpdate(ctx, operationID, targetVersion, client.Trigger) + trigger := updateTrigger(client.Trigger) + if complete { + trigger = client.TriggerComplete + } + operation, err := triggerUpdate(ctx, operationID, targetVersion, trigger) if err != nil { return err } diff --git a/server/cmd/fleet-ha/main_test.go b/server/cmd/fleet-ha/main_test.go index b4c050aa7c..7d40e6f328 100644 --- a/server/cmd/fleet-ha/main_test.go +++ b/server/cmd/fleet-ha/main_test.go @@ -15,10 +15,20 @@ import ( type fakeUpdaterClient struct { triggered bool + complete bool triggerErr error operation updaterapi.Operation } +func (f *fakeUpdaterClient) TriggerComplete(_ context.Context, operationID, targetVersion string) (updaterapi.Operation, error) { + f.triggered = true + f.complete = true + if f.triggerErr != nil { + return updaterapi.Operation{}, f.triggerErr + } + return updaterapi.Operation{ID: operationID, TargetVersion: targetVersion, Phase: updaterapi.PhaseSucceeded}, nil +} + func (f *fakeUpdaterClient) Status(context.Context) (updaterapi.StatusResponse, error) { return updaterapi.StatusResponse{}, nil } @@ -79,7 +89,7 @@ func TestUpdateRequiresPassiveBeforeTriggering(t *testing.T) { client := &fakeUpdaterClient{} // Act - err := runUpdate(t.Context(), "v1.2.3", &bytes.Buffer{}, func(context.Context, string, string) error { + err := runUpdate(t.Context(), "v1.2.3", false, &bytes.Buffer{}, func(context.Context, string, string, bool) error { return errors.New("local Fleet is active") }, client) @@ -98,7 +108,7 @@ func TestUpdateReportsTerminalSuccess(t *testing.T) { var output bytes.Buffer // Act - err := runUpdate(t.Context(), "v1.2.3", &output, func(context.Context, string, string) error { return nil }, client) + err := runUpdate(t.Context(), "v1.2.3", false, &output, func(context.Context, string, string, bool) error { return nil }, client) // Assert require.NoError(t, err) @@ -120,7 +130,7 @@ func TestPassiveUpdateReportsDegradedFailoverReadiness(t *testing.T) { } // Act - err := runPassiveUpdate(t.Context(), "v1.2.3", &output, func(context.Context, string, string) error { return nil }, client, read) + err := runPassiveUpdate(t.Context(), "v1.2.3", false, &output, func(context.Context, string, string, bool) error { return nil }, client, read) // Assert require.ErrorContains(t, err, "failover readiness is degraded") @@ -138,7 +148,7 @@ func TestPassiveUpdateAllowsExpectedVersionMismatch(t *testing.T) { } // Act - err := runPassiveUpdate(t.Context(), "v1.2.3", &bytes.Buffer{}, func(context.Context, string, string) error { return nil }, client, read) + err := runPassiveUpdate(t.Context(), "v1.2.3", false, &bytes.Buffer{}, func(context.Context, string, string, bool) error { return nil }, client, read) // Assert require.NoError(t, err) @@ -149,7 +159,7 @@ func TestUpdateReturnsWhenUpdaterIsUnavailable(t *testing.T) { client := &fakeUpdaterClient{triggerErr: updaterapi.ErrUnavailable} // Act - err := runUpdate(t.Context(), "v1.2.3", &bytes.Buffer{}, func(context.Context, string, string) error { return nil }, client) + err := runUpdate(t.Context(), "v1.2.3", false, &bytes.Buffer{}, func(context.Context, string, string, bool) error { return nil }, client) // Assert require.ErrorIs(t, err, updaterapi.ErrUnavailable) @@ -164,10 +174,31 @@ func TestUpdateFailureIncludesRecoveryDetails(t *testing.T) { var output bytes.Buffer // Act - err := runUpdate(t.Context(), "v1.2.3", &output, func(context.Context, string, string) error { return nil }, client) + err := runUpdate(t.Context(), "v1.2.3", false, &output, func(context.Context, string, string, bool) error { return nil }, client) // Assert require.ErrorContains(t, err, "Recovery: fleet-ha app-start v1.2.3") require.ErrorContains(t, err, "Log: /var/log/proto-fleet-updater/update.log") require.Contains(t, output.String(), "Update operation") } + +func TestCompleteUpdateRequiresActiveAndUsesCompletionRequest(t *testing.T) { + // Arrange + client := &fakeUpdaterClient{} + activeChecked := false + + // Act + err := runUpdate( + t.Context(), "v1.2.3", true, &bytes.Buffer{}, + func(_ context.Context, _, _ string, complete bool) error { + activeChecked = complete + return nil + }, + client, + ) + + // Assert + require.NoError(t, err) + require.True(t, activeChecked) + require.True(t, client.complete) +} diff --git a/server/internal/domain/updates/executor_test.go b/server/internal/domain/updates/executor_test.go index 5f15c53fb6..9d693e5f55 100644 --- a/server/internal/domain/updates/executor_test.go +++ b/server/internal/domain/updates/executor_test.go @@ -133,6 +133,31 @@ func TestUnixExecutorClientTrigger(t *testing.T) { assert.NoError(t, observation.decodeErr) } +func TestUnixExecutorClientTriggerComplete(t *testing.T) { + // Arrange + operationID := "11111111-1111-4111-8111-111111111111" + observed := make(chan executorRequestObservation, 1) + client := startExecutorTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var request updaterapi.TriggerRequest + decodeErr := json.NewDecoder(r.Body).Decode(&request) + observed <- executorRequestObservation{trigger: request, decodeErr: decodeErr} + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(updaterapi.TriggerResponse{Operation: updaterapi.Operation{ + ID: operationID, TargetVersion: "v1.2.3", Phase: updaterapi.PhaseQueued, + }}) + })) + + // Act + operation, err := client.TriggerComplete(t.Context(), operationID, "v1.2.3") + + // Assert + require.NoError(t, err) + require.Equal(t, operationID, operation.ID) + observation := <-observed + require.NoError(t, observation.decodeErr) + require.True(t, observation.trigger.Complete) +} + func TestUnixExecutorClientHTTPFailures(t *testing.T) { t.Parallel() diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index 20de3d827f..52913b11f3 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/http" "os" "path/filepath" "strings" @@ -11,8 +12,11 @@ import ( "time" "github.com/block/proto-fleet/server/internal/ha" + "github.com/block/proto-fleet/server/internal/transportguard" ) +const vipTakeoverTimeout = 15 * time.Second + func requirePassiveStatus(ctx context.Context, envPath string) (StatusReport, error) { report, err := Status(ctx, envPath) if err != nil { @@ -58,6 +62,56 @@ var releaseImageRepositories = [...]string{ "proto-fleet-client", } +func RequireActive(ctx context.Context, envPath string) error { + _, err := requireActiveStatus(ctx, envPath) + return err +} + +func requireActiveStatus(ctx context.Context, envPath string) (StatusReport, error) { + report, err := Status(ctx, envPath) + if err != nil { + return StatusReport{}, err + } + if report.Runtime.Observation != ha.ObservationCurrent || report.Runtime.Role != ha.RoleActive || report.Runtime.Endpoint != ha.EndpointHealthy { + return StatusReport{}, fmt.Errorf("HA completion update requires a healthy active node; local role is %s", report.Runtime.Role) + } + if !rollingUpdateControlReady(report.Control) { + return StatusReport{}, errors.New("HA completion update requires rolling-update readiness") + } + return report, nil +} + +func RequireUpdatedPeer(ctx context.Context, envPath, targetVersion string) error { + config, err := loadNodeConfig(envPath) + if err != nil { + return err + } + peerAddress := config.DatabaseAIP + if config.NodeIP == config.DatabaseAIP { + peerAddress = config.DatabaseBIP + } + tlsConfig, err := ha.LoadServiceTLS(filepath.Join(config.SecretsDir, "service-ca.crt")) + if err != nil { + return err + } + status := probeFleetHost(ctx, tlsConfig, config.VirtualIP, peerAddress) + if !updatedPassivePeerReady(status, targetVersion) { + return fmt.Errorf("HA completion update requires the passive peer to run %s", targetVersion) + } + return nil +} + +func ValidateActiveUpdate(ctx context.Context, envPath, targetVersion string) (string, error) { + report, err := requireActiveStatus(ctx, envPath) + if err != nil { + return "", err + } + if err := RequireUpdatedPeer(ctx, envPath, targetVersion); err != nil { + return "", err + } + return report.Runtime.Version, nil +} + // PrepareApplicationUpdate builds only the Fleet API and client from a verified release. func PrepareApplicationUpdate(ctx context.Context, root string) error { deps := defaultInstallDependencies() @@ -139,10 +193,19 @@ func pruneReleaseImages(ctx context.Context) error { return nil } -// StopApplication stops only Fleet containers; the HA substrate keeps running. -func StopApplication(ctx context.Context, root string) error { - if _, err := requirePassiveStatus(ctx, filepath.Join(configRoot, "node.env")); err != nil { - return fmt.Errorf("refuse to stop HA application after passive role changed: %w", err) +// StopApplication rechecks the expected role, then stops only Fleet containers. +func StopApplication(ctx context.Context, root string, expectedRole ha.RuntimeRole) error { + var err error + switch expectedRole { + case ha.RolePassive: + _, err = requirePassiveStatus(ctx, filepath.Join(configRoot, "node.env")) + case ha.RoleActive: + _, err = requireActiveStatus(ctx, filepath.Join(configRoot, "node.env")) + case ha.RoleInitializing, ha.RoleDegraded: + return fmt.Errorf("unsupported HA role %q", expectedRole) + } + if err != nil { + return fmt.Errorf("refuse to stop HA application after %s role changed: %w", expectedRole, err) } // The crash-only design intentionally has no maintenance lease. If the role // changes after this final proof, normal update recovery restarts Fleet. @@ -152,6 +215,10 @@ func StopApplication(ctx context.Context, root string) error { return nil } +func updatedPassivePeerReady(status fleetHostStatus, targetVersion string) bool { + return status.reachable && status.passive && status.version == targetVersion +} + // StartApplication starts the target release and proves it serves its observed HA role. func StartApplication(ctx context.Context, root, targetVersion string, requirePassive bool) error { config, err := loadNodeConfig(filepath.Join(configRoot, "node.env")) @@ -226,16 +293,6 @@ func rollingUpdateApplicationReady(report StatusReport, public fleetHostStatus, rollingUpdateControlReady(report.Control), nil } -func updatedApplicationReady(report StatusReport, publicStatus fleetHostStatus, targetVersion string, requirePassive bool) (bool, error) { - if requirePassive { - return rollingUpdateApplicationReady(report, publicStatus, targetVersion) - } - if report.Control == nil || !report.Control.ControlReady { - return false, nil - } - return applicationReady(report.Runtime, publicStatus, targetVersion), nil -} - func rollingUpdateControlReady(control *ControlStatus) bool { if control == nil || !control.ControlReady { return false @@ -249,6 +306,16 @@ func ExpectedRollingVersionMismatch(control *ControlStatus) bool { len(control.ReasonCodes) == 1 && control.ReasonCodes[0] == ReasonFleetVersionMismatch } +func updatedApplicationReady(report StatusReport, publicStatus fleetHostStatus, targetVersion string, requirePassive bool) (bool, error) { + if requirePassive { + return rollingUpdateApplicationReady(report, publicStatus, targetVersion) + } + if report.Control == nil || !report.Control.ControlReady { + return false, nil + } + return applicationReady(report.Runtime, publicStatus, targetVersion), nil +} + func applicationReady(runtime ha.Status, public fleetHostStatus, targetVersion string) bool { publicRoleReady := runtime.Role == ha.RoleActive && public.active || runtime.Role == ha.RolePassive && public.passive return runtime.Version == targetVersion && @@ -257,3 +324,56 @@ func applicationReady(runtime ha.Status, public fleetHostStatus, targetVersion s public.reachable && public.version == targetVersion } + +func WaitForVIPVersion(ctx context.Context, envPath, targetVersion string) error { + config, err := loadNodeConfig(envPath) + if err != nil { + return err + } + tlsConfig, err := ha.LoadServiceTLS(filepath.Join(config.SecretsDir, "service-ca.crt")) + if err != nil { + return err + } + transport := &http.Transport{TLSClientConfig: tlsConfig, Proxy: nil} + client := &http.Client{Transport: transport, Timeout: 2 * time.Second, CheckRedirect: transportguard.RejectRedirect} + defer transport.CloseIdleConnections() + deadline, cancel := context.WithTimeout(ctx, vipTakeoverTimeout) + defer cancel() + endpoint := "https://" + config.VirtualIP + "/api-proxy/health" + for { + request, requestErr := http.NewRequestWithContext(deadline, http.MethodGet, endpoint, nil) + if requestErr != nil { + return fmt.Errorf("create VIP takeover probe: %w", requestErr) + } + response, requestErr := client.Do(request) + if requestErr == nil { + response.Body.Close() + version := response.Header.Get("X-Proto-Fleet-Version") + ready, versionErr := acceptVIPVersion(response.StatusCode, version, targetVersion) + if versionErr != nil { + return versionErr + } + if ready { + return nil + } + } + select { + case <-deadline.Done(): + return errors.New("updated peer did not serve the VIP within 15 seconds") + case <-time.After(500 * time.Millisecond): + } + } +} + +func acceptVIPVersion(status int, version, targetVersion string) (bool, error) { + if status != http.StatusOK { + return false, nil + } + if version == targetVersion { + return true, nil + } + if version != "" { + return false, fmt.Errorf("VIP is served by %s, expected %s", version, targetVersion) + } + return false, nil +} diff --git a/server/internal/ha/deployment/update_test.go b/server/internal/ha/deployment/update_test.go index b5f9176ad4..fae64757c8 100644 --- a/server/internal/ha/deployment/update_test.go +++ b/server/internal/ha/deployment/update_test.go @@ -1,6 +1,7 @@ package deployment import ( + "net/http" "testing" "github.com/stretchr/testify/require" @@ -68,6 +69,22 @@ func TestApplicationConvergenceRequiresExpectedRuntimeRole(t *testing.T) { } } +func TestRecoveryAcceptsHealthyActiveApplication(t *testing.T) { + // Arrange + report := StatusReport{ + Runtime: ha.Status{Version: "v1.1.0", Role: ha.RoleActive, Observation: ha.ObservationCurrent}, + Control: &ControlStatus{ControlReady: true}, + } + public := fleetHostStatus{reachable: true, active: true, version: "v1.1.0"} + + // Act + ready, err := updatedApplicationReady(report, public, "v1.1.0", false) + + // Assert + require.NoError(t, err) + require.True(t, ready) +} + func TestRollingUpdateControlAllowsOnlyExpectedVersionMismatch(t *testing.T) { for _, test := range []struct { name string @@ -95,3 +112,20 @@ func TestValidateInfrastructureGenerationRejectsChainedUpdate(t *testing.T) { // Assert require.ErrorContains(t, err, "chained HA application updates are not supported") } + +func TestAcceptVIPVersionRejectsWrongPeerRelease(t *testing.T) { + // Act + ready, err := acceptVIPVersion(http.StatusOK, "v1.0.0", "v1.1.0") + + // Assert + require.False(t, ready) + require.ErrorContains(t, err, "v1.0.0") +} + +func TestUpdatedPassivePeerReady(t *testing.T) { + // Act and assert + require.True(t, updatedPassivePeerReady(fleetHostStatus{reachable: true, passive: true, version: "v1.1.0"}, "v1.1.0")) + require.False(t, updatedPassivePeerReady(fleetHostStatus{reachable: true, version: "v1.1.0"}, "v1.1.0")) + require.False(t, updatedPassivePeerReady(fleetHostStatus{reachable: true, active: true, version: "v1.1.0"}, "v1.1.0")) + require.False(t, updatedPassivePeerReady(fleetHostStatus{reachable: true, passive: true, version: "v1.0.0"}, "v1.1.0")) +} diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 64d592bbfa..3303596821 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -690,6 +690,11 @@ func NewManager(cfg Config) (*Manager, error) { _ = logRoot.Close() return nil, err } + if err := m.recoverHAApplication(); err != nil { + _ = processLock.Close() + _ = logRoot.Close() + return nil, err + } if err := m.cleanupStaleArtifacts(); err != nil { _ = processLock.Close() _ = logRoot.Close() @@ -914,7 +919,7 @@ func (m *Manager) Trigger(targetVersion string) (updaterapi.Operation, error) { if m.cfg.NewID == nil { return updaterapi.Operation{}, fmt.Errorf("generate updater operation id: generator is not configured") } - return m.trigger(targetVersion, m.cfg.NewID(), false) + return m.trigger(targetVersion, m.cfg.NewID(), false, false) } // TriggerWithID accepts a caller-generated UUID so a client that loses the @@ -922,14 +927,22 @@ func (m *Manager) Trigger(targetVersion string) (updaterapi.Operation, error) { // guessing by target or time. Reusing the ID for the same target is // idempotent; reusing it for another target is rejected. func (m *Manager) TriggerWithID(targetVersion, operationID string) (updaterapi.Operation, error) { + return m.triggerWithID(targetVersion, operationID, false) +} + +func (m *Manager) TriggerCompleteWithID(targetVersion, operationID string) (updaterapi.Operation, error) { + return m.triggerWithID(targetVersion, operationID, true) +} + +func (m *Manager) triggerWithID(targetVersion, operationID string, complete bool) (updaterapi.Operation, error) { parsedID, err := uuid.Parse(operationID) if err != nil || parsedID.String() != operationID { return updaterapi.Operation{}, newTriggerError(errTriggerInvalid, "operation id must be a canonical UUID") } - return m.trigger(targetVersion, operationID, true) + return m.trigger(targetVersion, operationID, true, complete) } -func (m *Manager) trigger(targetVersion, operationID string, idempotent bool) (updaterapi.Operation, error) { +func (m *Manager) trigger(targetVersion, operationID string, idempotent, complete bool) (updaterapi.Operation, error) { if operationID == "" { return updaterapi.Operation{}, fmt.Errorf("generate updater operation id: empty value") } @@ -939,6 +952,9 @@ func (m *Manager) trigger(targetVersion, operationID string, idempotent bool) (u "target version must be a stable or RC release tag", ) } + if complete && m.cfg.DeploymentMode != DeploymentModeHA { + return updaterapi.Operation{}, newTriggerError(errTriggerPrecondition, "completion updates require HA deployment mode") + } // Check before host-state validation so a retry can recover the original // response even after that operation has completed and changed the // installed version. The write-locked check below closes the race with a @@ -1051,7 +1067,7 @@ func (m *Manager) trigger(targetVersion, operationID string, idempotent bool) (u go func() { defer m.operationWG.Done() defer m.finishOperation() - m.run(operationCtx, operationCopy.ID, targetVersion) + m.run(operationCtx, operationCopy.ID, targetVersion, complete) }() return operationCopy, nil } @@ -1063,7 +1079,7 @@ func (m *Manager) finishOperation() { m.cancelOperation = nil } -func (m *Manager) run(ctx context.Context, operationID, targetVersion string) { +func (m *Manager) run(ctx context.Context, operationID, targetVersion string, complete bool) { logName := operationLogFilename(operationID) logPath := filepath.Join(m.cfg.StateDir, "logs", logName) logFile, err := m.logRoot.OpenFile(logName, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) @@ -1256,9 +1272,15 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string) { return } + requiredRole := "passive" if m.cfg.DeploymentMode == DeploymentModeHA { - if err := m.runHACommand(ctx, m.cfg.ActivationTimeout, currentDeployment, commandOutput, "require-passive", haNodeEnvPath, targetVersion); err != nil { - m.fail(operationID, fmt.Errorf("local Fleet is not passive immediately before activation: %w", err), recovery) + requireArgs := []string{"require-passive", haNodeEnvPath, targetVersion} + if complete { + requiredRole = "active" + requireArgs = []string{"require-active", haNodeEnvPath} + } + if err := m.runHACommand(ctx, m.cfg.ActivationTimeout, currentDeployment, commandOutput, requireArgs...); err != nil { + m.fail(operationID, fmt.Errorf("local Fleet is not %s immediately before activation: %w", requiredRole, err), recovery) return } } @@ -1275,10 +1297,22 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string) { m.failActivation(operationID, targetVersion, fmt.Errorf("persist passive application recovery: %w", err), logFile, false) return } - if err := m.runHACommand(activationCtx, m.cfg.ActivationTimeout, currentDeployment, commandOutput, "app-stop"); err != nil { - m.failActivation(operationID, targetVersion, fmt.Errorf("stop passive HA application: %w", err), logFile, true) + if err := m.runHACommand(activationCtx, m.cfg.ActivationTimeout, currentDeployment, commandOutput, "app-stop", requiredRole); err != nil { + m.failActivation(operationID, targetVersion, fmt.Errorf("stop HA application: %w", err), logFile, true) return } + if complete { + if err := m.runHACommand(activationCtx, m.cfg.ActivationTimeout, currentDeployment, commandOutput, "wait-takeover", targetVersion); err != nil { + restartErr := m.runHACommand(activationCtx, m.cfg.ActivationTimeout, currentDeployment, commandOutput, "app-start", previousVersion, "any") + clearErr := m.clearActivationMarker() + if restartErr == nil && clearErr == nil { + m.fail(operationID, fmt.Errorf("updated peer did not take over; previous release restarted: %w", err), "") + return + } + m.fail(operationID, errors.Join(err, restartErr, clearErr), m.activationRecoveryCommand(currentDeployment, previousVersion)) + return + } + } } if err := activateDeployment(stageDeployment, currentDeployment, backupDeployment); err != nil { m.failActivation(operationID, targetVersion, err, logFile, m.cfg.DeploymentMode == DeploymentModeHA) @@ -1352,7 +1386,7 @@ func (m *Manager) runPreflight(ctx context.Context, deployment string, output io func (m *Manager) runActivation(ctx context.Context, deployment, targetVersion string, output io.Writer) error { if m.cfg.DeploymentMode == DeploymentModeHA { - return m.runHACommand(ctx, m.cfg.ActivationTimeout, deployment, output, "app-start", targetVersion) + return m.runHACommand(ctx, m.cfg.ActivationTimeout, deployment, output, "app-start", targetVersion, "passive") } return m.runCommand(ctx, m.cfg.ActivationTimeout, deployment, output, "/bin/bash", "./run-fleet.sh", "--non-interactive", "--skip-build") } @@ -1978,7 +2012,7 @@ func (m *Manager) loadState() error { op.Phase = updaterapi.PhaseFailed if restoredPrevious { op.Message = "Upgrade interrupted; previous deployment restored" - op.Error = "The updater restarted during the activation swap before Fleet was stopped. The previous deployment was restored safely." + op.Error = "The updater restarted during the activation swap. The previous deployment was restored." } else { op.Message = "Upgrade interrupted" op.Error = "The updater restarted before the operation completed; inspect the host log and recovery details before retrying." @@ -1994,7 +2028,9 @@ func (m *Manager) loadState() error { } else { op.Error += " The active deployment was missing during startup; the updater restored the validated previous deployment." } - op.RecoveryCommand = "" + if m.cfg.DeploymentMode != DeploymentModeHA { + op.RecoveryCommand = "" + } op.UpdatedAt = now if op.CompletedAt == nil { op.CompletedAt = &now diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index 0554a4ed09..d945726f23 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -242,8 +242,8 @@ func TestManagerHAUpdateTouchesOnlyThePassiveApplication(t *testing.T) { require.Len(t, commands, 5) assert.Equal(t, []string{"update-preflight"}, commands[0].Args) assert.Equal(t, []string{"require-passive", "/etc/proto-fleet/ha/node.env", "v1.1.0"}, commands[1].Args) - assert.Equal(t, []string{"app-stop"}, commands[2].Args) - assert.Equal(t, []string{"app-start", "v1.1.0"}, commands[3].Args) + assert.Equal(t, []string{"app-stop", "passive"}, commands[2].Args) + assert.Equal(t, []string{"app-start", "v1.1.0", "passive"}, commands[3].Args) for _, command := range commands[:4] { assert.Contains(t, command.Name, filepath.Join("ha", "fleet-ha")) assert.NotContains(t, strings.Join(command.Args, " "), "etcd") @@ -276,7 +276,7 @@ func TestManagerHAUpdateKeepsForwardRecoveryWhenStartupFails(t *testing.T) { assert.Contains(t, completed.Error, "new stack failed to start") assert.Equal(t, "v1.1.0", mustReadVersion(t, filepath.Join(installRoot, "deployment", "version.txt"))) commands := runner.Commands() - assert.Equal(t, []string{"app-start", "v1.1.0"}, commands[len(commands)-1].Args) + assert.Equal(t, []string{"app-start", "v1.1.0", "passive"}, commands[len(commands)-1].Args) } func TestManagerHAPreflightFailureLeavesCurrentApplicationUntouched(t *testing.T) { @@ -383,7 +383,7 @@ func TestManagerHARejectsUnqualifiedSourceRelease(t *testing.T) { } } -func TestManagerHAInterruptedAfterStopRetainsRestartCommand(t *testing.T) { +func TestManagerHAInterruptedAfterStopRestartsCurrentApplication(t *testing.T) { // Arrange installRoot := t.TempDir() stateDir := filepath.Join(t.TempDir(), "state") @@ -391,9 +391,91 @@ func TestManagerHAInterruptedAfterStopRetainsRestartCommand(t *testing.T) { require.NoError(t, os.Rename(filepath.Join(installRoot, "deployment"), filepath.Join(installRoot, "deployment.previous"))) writeInterruptedOperationState(t, stateDir, "v1.1.0") + // Act + runner := &haRecordingRunner{fail: make(map[string]error)} + manager, err := NewManager(Config{ + InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", DeploymentMode: DeploymentModeHA, Runner: runner, + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, manager.Close()) }) + + // Assert + operation := manager.Status().Operation + require.NotNil(t, operation) + require.Equal(t, updaterapi.PhaseFailed, operation.Phase) + assert.Empty(t, operation.RecoveryCommand) + assert.Contains(t, operation.Message, "HA application restarted") + commands := runner.Commands() + require.Len(t, commands, 1) + assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[0].Args) +} + +func TestManagerHACompletionWaitsForUpdatedPeerBeforeSwap(t *testing.T) { + // Arrange + installRoot := t.TempDir() + writeCurrentDeployment(t, installRoot, "v1.0.0") + bundle := releaseBundle(t, "v1.1.0") + server := releaseServer(t, "v1.1.0", "amd64", bundle, "") + runner := &haRecordingRunner{fail: make(map[string]error)} + manager := newTestManagerWithConfig(t, installRoot, server, runner, func(cfg *Config) { + cfg.DeploymentMode = DeploymentModeHA + }) + + // Act + _, err := manager.TriggerCompleteWithID("v1.1.0", "11111111-1111-4111-8111-111111111111") + require.NoError(t, err) + completed := waitForTerminal(t, manager) + + // Assert + require.Equal(t, updaterapi.PhaseSucceeded, completed.Phase, completed.Error) + commands := runner.Commands() + require.Len(t, commands, 5) + assert.Equal(t, []string{"update-preflight"}, commands[0].Args) + assert.Equal(t, []string{"require-active", "/etc/proto-fleet/ha/node.env"}, commands[1].Args) + assert.Equal(t, []string{"app-stop", "active"}, commands[2].Args) + assert.Equal(t, []string{"wait-takeover", "v1.1.0"}, commands[3].Args) + assert.Equal(t, []string{"app-start", "v1.1.0", "passive"}, commands[4].Args) +} + +func TestManagerHACompletionRestartsOldReleaseWhenTakeoverFails(t *testing.T) { + // Arrange + installRoot := t.TempDir() + writeCurrentDeployment(t, installRoot, "v1.0.0") + bundle := releaseBundle(t, "v1.1.0") + server := releaseServer(t, "v1.1.0", "amd64", bundle, "") + runner := &haRecordingRunner{fail: map[string]error{"wait-takeover": assert.AnError}} + manager := newTestManagerWithConfig(t, installRoot, server, runner, func(cfg *Config) { + cfg.DeploymentMode = DeploymentModeHA + }) + + // Act + _, err := manager.TriggerCompleteWithID("v1.1.0", "11111111-1111-4111-8111-111111111111") + require.NoError(t, err) + completed := waitForTerminal(t, manager) + + // Assert + require.Equal(t, updaterapi.PhaseFailed, completed.Phase) + assert.Contains(t, completed.Error, "previous release restarted") + assert.Empty(t, completed.RecoveryCommand) + assert.Equal(t, "v1.0.0", mustReadVersion(t, filepath.Join(installRoot, "deployment", "version.txt"))) + commands := runner.Commands() + assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[len(commands)-1].Args) +} + +func TestManagerHACompletionInterruptedAfterSwapStartsTargetRelease(t *testing.T) { + // Arrange + installRoot := t.TempDir() + stateDir := filepath.Join(t.TempDir(), "state") + writeCurrentDeployment(t, installRoot, "v1.1.0") + previousVersionPath := filepath.Join(installRoot, "deployment.previous", "version.txt") + require.NoError(t, os.MkdirAll(filepath.Dir(previousVersionPath), 0o750)) + require.NoError(t, os.WriteFile(previousVersionPath, []byte("version: v1.0.0\n"), 0o600)) + writeInterruptedOperationState(t, stateDir, "v1.1.0") + runner := &haRecordingRunner{fail: make(map[string]error)} + // Act manager, err := NewManager(Config{ - InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", DeploymentMode: DeploymentModeHA, + InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", DeploymentMode: DeploymentModeHA, Runner: runner, }) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close()) }) @@ -402,9 +484,10 @@ func TestManagerHAInterruptedAfterStopRetainsRestartCommand(t *testing.T) { operation := manager.Status().Operation require.NotNil(t, operation) require.Equal(t, updaterapi.PhaseFailed, operation.Phase) - assert.Contains(t, operation.RecoveryCommand, "app-start") - assert.Contains(t, operation.RecoveryCommand, "v1.0.0") - assert.True(t, strings.HasSuffix(operation.RecoveryCommand, " any")) + require.Empty(t, operation.RecoveryCommand) + commands := runner.Commands() + require.Len(t, commands, 1) + assert.Equal(t, []string{"app-start", "v1.1.0", "any"}, commands[0].Args) } func TestManagerTriggerWithIDDeduplicatesConcurrentAdmission(t *testing.T) { @@ -1727,6 +1810,35 @@ func TestRepairStartupRestoresUpdaterFromInstalledDeployment(t *testing.T) { assert.NoFileExists(t, installedUpdater+selfUpdateHandoffSuffix) } +func TestManagerRestartsHAApplicationAfterInterruptedSwap(t *testing.T) { + t.Parallel() + + // Arrange + installRoot := t.TempDir() + writeCurrentDeployment(t, installRoot, "v1.0.0") + require.NoError(t, os.Rename( + filepath.Join(installRoot, "deployment"), + filepath.Join(installRoot, "deployment.previous"), + )) + stateDir := filepath.Join(t.TempDir(), "state") + writeInterruptedOperationState(t, stateDir, "v1.1.0") + runner := &haRecordingRunner{} + + // Act + manager, err := NewManager(Config{ + InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", + DeploymentMode: DeploymentModeHA, Runner: runner, + }) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, manager.Close()) }) + + // Assert + commands := runner.Commands() + require.Len(t, commands, 1) + assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[0].Args) + assert.Empty(t, manager.Status().Operation.RecoveryCommand) +} + func TestManagerReconcilesTerminalFailedActivationBeforeCleaningArtifacts(t *testing.T) { t.Parallel() diff --git a/server/internal/updater/server.go b/server/internal/updater/server.go index d048bd32d4..c5f3473f13 100644 --- a/server/internal/updater/server.go +++ b/server/internal/updater/server.go @@ -250,7 +250,12 @@ func (s *Server) handleUpgrade(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusBadRequest, updaterapi.ErrorResponse{Error: "invalid request body"}) return } - operation, err := s.manager.TriggerWithID(request.TargetVersion, request.OperationID) + var operation updaterapi.Operation + if request.Complete { + operation, err = s.manager.TriggerCompleteWithID(request.TargetVersion, request.OperationID) + } else { + operation, err = s.manager.TriggerWithID(request.TargetVersion, request.OperationID) + } if err != nil { status := triggerErrorHTTPStatus(err) message := err.Error() diff --git a/server/internal/updaterapi/client.go b/server/internal/updaterapi/client.go index 44d2665e4d..970464b693 100644 --- a/server/internal/updaterapi/client.go +++ b/server/internal/updaterapi/client.go @@ -72,7 +72,15 @@ func (c *Client) Status(ctx context.Context) (StatusResponse, error) { } func (c *Client) Trigger(ctx context.Context, operationID, targetVersion string) (Operation, error) { - request := TriggerRequest{OperationID: operationID, TargetVersion: targetVersion} + return c.trigger(ctx, operationID, targetVersion, false) +} + +func (c *Client) TriggerComplete(ctx context.Context, operationID, targetVersion string) (Operation, error) { + return c.trigger(ctx, operationID, targetVersion, true) +} + +func (c *Client) trigger(ctx context.Context, operationID, targetVersion string, complete bool) (Operation, error) { + request := TriggerRequest{OperationID: operationID, TargetVersion: targetVersion, Complete: complete} var response TriggerResponse if err := c.do(ctx, http.MethodPost, "/v1/upgrade", request, &response); err != nil { return Operation{}, err diff --git a/server/internal/updaterapi/types.go b/server/internal/updaterapi/types.go index 56cb064e89..fa9764a8a6 100644 --- a/server/internal/updaterapi/types.go +++ b/server/internal/updaterapi/types.go @@ -41,6 +41,7 @@ type StatusResponse struct { type TriggerRequest struct { OperationID string `json:"operation_id"` TargetVersion string `json:"target_version"` + Complete bool `json:"complete,omitempty"` } type TriggerResponse struct { From 9d87e963e232f01ca0d357b5751169dd412e35be Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 14:37:40 -0700 Subject: [PATCH 02/40] Address complete update review feedback (#891) --- deployment-files/ha/README.md | 2 +- server/cmd/fleet-ha/main.go | 2 +- server/internal/ha/deployment/update.go | 11 +++++++--- server/internal/updater/manager.go | 5 ----- server/internal/updater/manager_test.go | 28 +++++++++++++++++++++++++ 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/deployment-files/ha/README.md b/deployment-files/ha/README.md index 09ec00c522..31d4c748ba 100644 --- a/deployment-files/ha/README.md +++ b/deployment-files/ha/README.md @@ -203,7 +203,7 @@ rather than upgraded through this workflow. The updater stages everything first, stops the local Fleet containers, and waits for the updated peer to serve the VIP with the target version. Only then does it swap and restart the local application as passive. If takeover does -not complete within 15 seconds, it restarts the old local release without +not complete within 30 seconds, it restarts the old local release without swapping. This is a bounded interruption, not a zero-downtime update. ## Qualification diff --git a/server/cmd/fleet-ha/main.go b/server/cmd/fleet-ha/main.go index 287ddbb46e..dbd48bed9b 100644 --- a/server/cmd/fleet-ha/main.go +++ b/server/cmd/fleet-ha/main.go @@ -179,7 +179,7 @@ func (c *appStopCmd) Run(ctx context.Context) error { type appStartCmd struct { Version string `arg:"" help:"application version to start"` - Mode string `arg:"" enum:"passive,any" help:"required HA role after startup"` + Mode string `arg:"" optional:"" default:"passive" enum:"passive,any" help:"required HA role after startup"` } func (c *appStartCmd) Run(ctx context.Context) error { diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index 52913b11f3..d886307198 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -15,7 +15,7 @@ import ( "github.com/block/proto-fleet/server/internal/transportguard" ) -const vipTakeoverTimeout = 15 * time.Second +const vipTakeoverTimeout = 30 * time.Second func requirePassiveStatus(ctx context.Context, envPath string) (StatusReport, error) { report, err := Status(ctx, envPath) @@ -73,7 +73,12 @@ func requireActiveStatus(ctx context.Context, envPath string) (StatusReport, err return StatusReport{}, err } if report.Runtime.Observation != ha.ObservationCurrent || report.Runtime.Role != ha.RoleActive || report.Runtime.Endpoint != ha.EndpointHealthy { - return StatusReport{}, fmt.Errorf("HA completion update requires a healthy active node; local role is %s", report.Runtime.Role) + return StatusReport{}, fmt.Errorf( + "HA completion update requires a healthy active node; local role is %s, observation is %s, and endpoint is %s", + report.Runtime.Role, + report.Runtime.Observation, + report.Runtime.Endpoint, + ) } if !rollingUpdateControlReady(report.Control) { return StatusReport{}, errors.New("HA completion update requires rolling-update readiness") @@ -359,7 +364,7 @@ func WaitForVIPVersion(ctx context.Context, envPath, targetVersion string) error } select { case <-deadline.Done(): - return errors.New("updated peer did not serve the VIP within 15 seconds") + return fmt.Errorf("updated peer did not serve the VIP within %s", vipTakeoverTimeout) case <-time.After(500 * time.Millisecond): } } diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 3303596821..bd1d67375e 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -690,11 +690,6 @@ func NewManager(cfg Config) (*Manager, error) { _ = logRoot.Close() return nil, err } - if err := m.recoverHAApplication(); err != nil { - _ = processLock.Close() - _ = logRoot.Close() - return nil, err - } if err := m.cleanupStaleArtifacts(); err != nil { _ = processLock.Close() _ = logRoot.Close() diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index d945726f23..0b570b0394 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -1839,6 +1839,34 @@ func TestManagerRestartsHAApplicationAfterInterruptedSwap(t *testing.T) { assert.Empty(t, manager.Status().Operation.RecoveryCommand) } +func TestManagerDoesNotReplayTerminalHARecovery(t *testing.T) { + // Arrange + installRoot := t.TempDir() + writeCurrentDeployment(t, installRoot, "v1.0.0") + stateDir := filepath.Join(t.TempDir(), "state") + require.NoError(t, os.MkdirAll(stateDir, 0o700)) + now := time.Date(2026, 8, 5, 8, 0, 0, 0, time.UTC) + operation := updaterapi.Operation{ + ID: "failed", TargetVersion: "v1.1.0", Phase: updaterapi.PhaseFailed, + RecoveryCommand: "stale", StartedAt: now, UpdatedAt: now, CompletedAt: &now, + } + data, err := json.Marshal(operation) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, stateFilename), data, 0o600)) + runner := &haRecordingRunner{} + + // Act + manager, err := NewManager(Config{ + InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", + DeploymentMode: DeploymentModeHA, Runner: runner, + }) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, manager.Close()) }) + + // Assert + assert.Empty(t, runner.Commands()) +} + func TestManagerReconcilesTerminalFailedActivationBeforeCleaningArtifacts(t *testing.T) { t.Parallel() From 9a30bb7f69d0fd0e56f91d700278de826437d770 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 16:00:02 -0700 Subject: [PATCH 03/40] Persist interrupted HA recovery intent (#891) --- server/internal/updater/manager.go | 4 ++- server/internal/updater/manager_test.go | 34 +++++++++++++++++++++++++ server/internal/updaterapi/types.go | 1 + 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index bd1d67375e..4a3e876b6b 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -754,7 +754,7 @@ func (m *Manager) restoreUpdaterFromInstalledDeployment() error { // RecoverApplication restarts an application left stopped by an interrupted HA update. func (m *Manager) RecoverApplication() error { - if m.cfg.DeploymentMode != DeploymentModeHA || m.operation == nil || m.operation.RecoveryCommand == "" { + if m.operation == nil || !m.operation.RecoveryPending || m.operation.RecoveryCommand == "" { return nil } deployment := filepath.Join(m.cfg.InstallRoot, "deployment") @@ -768,6 +768,7 @@ func (m *Manager) RecoverApplication() error { return fmt.Errorf("restart interrupted HA application: %w", err) } m.operation.RecoveryCommand = "" + m.operation.RecoveryPending = false m.operation.Message += "; HA application restarted" m.operation.UpdatedAt = m.cfg.Now().UTC() return m.persistLocked() @@ -2003,6 +2004,7 @@ func (m *Manager) loadState() error { } } if !wasTerminal { + op.RecoveryPending = m.cfg.DeploymentMode == DeploymentModeHA && op.RecoveryCommand != "" now := m.cfg.Now().UTC() op.Phase = updaterapi.PhaseFailed if restoredPrevious { diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index 0b570b0394..fdf2a35ec1 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -7,6 +7,7 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -1837,6 +1838,39 @@ func TestManagerRestartsHAApplicationAfterInterruptedSwap(t *testing.T) { require.Len(t, commands, 1) assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[0].Args) assert.Empty(t, manager.Status().Operation.RecoveryCommand) + assert.False(t, manager.Status().Operation.RecoveryPending) +} + +func TestManagerRetriesInterruptedHARecoveryAfterRestartFailure(t *testing.T) { + // Arrange + installRoot := t.TempDir() + writeCurrentDeployment(t, installRoot, "v1.0.0") + require.NoError(t, os.Rename( + filepath.Join(installRoot, "deployment"), + filepath.Join(installRoot, "deployment.previous"), + )) + stateDir := filepath.Join(t.TempDir(), "state") + writeInterruptedOperationState(t, stateDir, "v1.1.0") + + // Act + _, err := NewManager(Config{ + InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", + DeploymentMode: DeploymentModeHA, + Runner: &haRecordingRunner{fail: map[string]error{"app-start": errors.New("restart failed")}}, + }) + require.ErrorContains(t, err, "restart failed") + manager, err := NewManager(Config{ + InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", + DeploymentMode: DeploymentModeHA, Runner: &haRecordingRunner{}, + }) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, manager.Close()) }) + + // Assert + operation := manager.Status().Operation + require.NotNil(t, operation) + assert.False(t, operation.RecoveryPending) + assert.Empty(t, operation.RecoveryCommand) } func TestManagerDoesNotReplayTerminalHARecovery(t *testing.T) { diff --git a/server/internal/updaterapi/types.go b/server/internal/updaterapi/types.go index fa9764a8a6..8eefadb6c8 100644 --- a/server/internal/updaterapi/types.go +++ b/server/internal/updaterapi/types.go @@ -31,6 +31,7 @@ type Operation struct { CompletedAt *time.Time `json:"completed_at,omitempty"` Error string `json:"error,omitempty"` RecoveryCommand string `json:"recovery_command,omitempty"` + RecoveryPending bool `json:"recovery_pending,omitempty"` LogPath string `json:"log_path,omitempty"` } From 5a8d71fedd11dfd1a83bbca68bbfca339c8ba2f4 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 17:16:55 -0700 Subject: [PATCH 04/40] Keep HA updater available after recovery failure (#891) --- server/internal/ha/deployment/update.go | 4 +++- server/internal/updater/manager.go | 7 ++++++- server/internal/updater/manager_test.go | 14 +++++--------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index d886307198..403e2aa933 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -15,7 +15,9 @@ import ( "github.com/block/proto-fleet/server/internal/transportguard" ) -const vipTakeoverTimeout = 30 * time.Second +// Covers one 10s lease lifetime, the coordinator's two-lease acquire budget, +// and a small keepalived scheduling margin. +const vipTakeoverTimeout = 35 * time.Second func requirePassiveStatus(ctx context.Context, envPath string) (StatusReport, error) { report, err := Status(ctx, envPath) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 4a3e876b6b..e3b2eefbdf 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -765,7 +765,12 @@ func (m *Manager) RecoverApplication() error { ctx, cancel := context.WithTimeout(context.Background(), m.cfg.ActivationTimeout) defer cancel() if err := m.runHACommand(ctx, m.cfg.ActivationTimeout, deployment, io.Discard, "app-start", version, "any"); err != nil { - return fmt.Errorf("restart interrupted HA application: %w", err) + recoveryErr := fmt.Errorf("restart interrupted HA application: %w", err) + m.operation.Phase = updaterapi.PhaseFailed + m.operation.Message = "HA application recovery failed" + m.operation.Error = recoveryErr.Error() + m.operation.UpdatedAt = m.cfg.Now().UTC() + return m.persistLocked() } m.operation.RecoveryCommand = "" m.operation.RecoveryPending = false diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index fdf2a35ec1..8df57bf0b6 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -1841,7 +1841,7 @@ func TestManagerRestartsHAApplicationAfterInterruptedSwap(t *testing.T) { assert.False(t, manager.Status().Operation.RecoveryPending) } -func TestManagerRetriesInterruptedHARecoveryAfterRestartFailure(t *testing.T) { +func TestManagerServesRecoveryStateAfterRestartFailure(t *testing.T) { // Arrange installRoot := t.TempDir() writeCurrentDeployment(t, installRoot, "v1.0.0") @@ -1853,24 +1853,20 @@ func TestManagerRetriesInterruptedHARecoveryAfterRestartFailure(t *testing.T) { writeInterruptedOperationState(t, stateDir, "v1.1.0") // Act - _, err := NewManager(Config{ + manager, err := NewManager(Config{ InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", DeploymentMode: DeploymentModeHA, Runner: &haRecordingRunner{fail: map[string]error{"app-start": errors.New("restart failed")}}, }) - require.ErrorContains(t, err, "restart failed") - manager, err := NewManager(Config{ - InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", - DeploymentMode: DeploymentModeHA, Runner: &haRecordingRunner{}, - }) require.NoError(t, err) t.Cleanup(func() { assert.NoError(t, manager.Close()) }) // Assert operation := manager.Status().Operation require.NotNil(t, operation) - assert.False(t, operation.RecoveryPending) - assert.Empty(t, operation.RecoveryCommand) + assert.True(t, operation.RecoveryPending) + assert.NotEmpty(t, operation.RecoveryCommand) + assert.Contains(t, operation.Error, "restart failed") } func TestManagerDoesNotReplayTerminalHARecovery(t *testing.T) { From f33eb501911ef71b694f6f891d61d261999d0464 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 17:32:13 -0700 Subject: [PATCH 05/40] Recover HA updates without recreating live services (#891) --- server/cmd/fleet-ha/main.go | 13 +++++++++++++ server/internal/ha/deployment/update.go | 9 +++++++++ server/internal/updater/manager.go | 2 +- server/internal/updater/manager_test.go | 8 ++++---- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/server/cmd/fleet-ha/main.go b/server/cmd/fleet-ha/main.go index dbd48bed9b..59b7253d9d 100644 --- a/server/cmd/fleet-ha/main.go +++ b/server/cmd/fleet-ha/main.go @@ -41,6 +41,7 @@ type cli struct { UpdatePreflight updatePreflightCmd `cmd:"" help:"prepare the current release for an application update"` AppStop appStopCmd `cmd:"" help:"stop the Fleet application services"` AppStart appStartCmd `cmd:"" help:"start the Fleet application services"` + AppRecover appRecoverCmd `cmd:"" help:"recover an interrupted Fleet application update"` WaitTakeover waitTakeoverCmd `cmd:"" help:"wait for the VIP to serve an application version"` } @@ -190,6 +191,18 @@ func (c *appStartCmd) Run(ctx context.Context) error { return deployment.StartApplication(ctx, root, c.Version, c.Mode == "passive") } +type appRecoverCmd struct { + Version string `arg:"" help:"application version to recover"` +} + +func (c *appRecoverCmd) Run(ctx context.Context) error { + root, err := deployment.ReleaseRoot() + if err != nil { + return err + } + return deployment.RecoverApplication(ctx, root, c.Version) +} + type waitTakeoverCmd struct { Version string `arg:"" help:"application version expected on the VIP"` } diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index 403e2aa933..1ea63c3151 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -228,6 +228,15 @@ func updatedPassivePeerReady(status fleetHostStatus, targetVersion string) bool // StartApplication starts the target release and proves it serves its observed HA role. func StartApplication(ctx context.Context, root, targetVersion string, requirePassive bool) error { + return startApplication(ctx, root, targetVersion, requirePassive, false) +} + +// RecoverApplication resumes interrupted activation without recreating running containers. +func RecoverApplication(ctx context.Context, root, targetVersion string) error { + return startApplication(ctx, root, targetVersion, false, true) +} + +func startApplication(ctx context.Context, root, targetVersion string, requirePassive, recovering bool) error { config, err := loadNodeConfig(filepath.Join(configRoot, "node.env")) if err != nil { return err diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index e3b2eefbdf..1dd7978d0d 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -764,7 +764,7 @@ func (m *Manager) RecoverApplication() error { } ctx, cancel := context.WithTimeout(context.Background(), m.cfg.ActivationTimeout) defer cancel() - if err := m.runHACommand(ctx, m.cfg.ActivationTimeout, deployment, io.Discard, "app-start", version, "any"); err != nil { + if err := m.runHACommand(ctx, m.cfg.ActivationTimeout, deployment, io.Discard, "app-recover", version); err != nil { recoveryErr := fmt.Errorf("restart interrupted HA application: %w", err) m.operation.Phase = updaterapi.PhaseFailed m.operation.Message = "HA application recovery failed" diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index 8df57bf0b6..c2b40eac13 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -408,7 +408,7 @@ func TestManagerHAInterruptedAfterStopRestartsCurrentApplication(t *testing.T) { assert.Contains(t, operation.Message, "HA application restarted") commands := runner.Commands() require.Len(t, commands, 1) - assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[0].Args) + assert.Equal(t, []string{"app-recover", "v1.0.0"}, commands[0].Args) } func TestManagerHACompletionWaitsForUpdatedPeerBeforeSwap(t *testing.T) { @@ -488,7 +488,7 @@ func TestManagerHACompletionInterruptedAfterSwapStartsTargetRelease(t *testing.T require.Empty(t, operation.RecoveryCommand) commands := runner.Commands() require.Len(t, commands, 1) - assert.Equal(t, []string{"app-start", "v1.1.0", "any"}, commands[0].Args) + assert.Equal(t, []string{"app-recover", "v1.1.0"}, commands[0].Args) } func TestManagerTriggerWithIDDeduplicatesConcurrentAdmission(t *testing.T) { @@ -1836,7 +1836,7 @@ func TestManagerRestartsHAApplicationAfterInterruptedSwap(t *testing.T) { // Assert commands := runner.Commands() require.Len(t, commands, 1) - assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[0].Args) + assert.Equal(t, []string{"app-recover", "v1.0.0"}, commands[0].Args) assert.Empty(t, manager.Status().Operation.RecoveryCommand) assert.False(t, manager.Status().Operation.RecoveryPending) } @@ -1856,7 +1856,7 @@ func TestManagerServesRecoveryStateAfterRestartFailure(t *testing.T) { manager, err := NewManager(Config{ InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", DeploymentMode: DeploymentModeHA, - Runner: &haRecordingRunner{fail: map[string]error{"app-start": errors.New("restart failed")}}, + Runner: &haRecordingRunner{fail: map[string]error{"app-recover": errors.New("restart failed")}}, }) require.NoError(t, err) t.Cleanup(func() { assert.NoError(t, manager.Close()) }) From 771ab6b8da398fa69a364ef62f1e39229397d7a9 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 17:49:08 -0700 Subject: [PATCH 06/40] Converge interrupted HA updates safely --- server/internal/ha/deployment/update.go | 8 ++++---- server/internal/updater/manager.go | 21 +++++++++++++++++---- server/internal/updater/manager_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index 1ea63c3151..dea86e8857 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -228,15 +228,15 @@ func updatedPassivePeerReady(status fleetHostStatus, targetVersion string) bool // StartApplication starts the target release and proves it serves its observed HA role. func StartApplication(ctx context.Context, root, targetVersion string, requirePassive bool) error { - return startApplication(ctx, root, targetVersion, requirePassive, false) + return startApplication(ctx, root, targetVersion, requirePassive) } -// RecoverApplication resumes interrupted activation without recreating running containers. +// RecoverApplication converges both Fleet services after interrupted activation. func RecoverApplication(ctx context.Context, root, targetVersion string) error { - return startApplication(ctx, root, targetVersion, false, true) + return startApplication(ctx, root, targetVersion, false) } -func startApplication(ctx context.Context, root, targetVersion string, requirePassive, recovering bool) error { +func startApplication(ctx context.Context, root, targetVersion string, requirePassive bool) error { config, err := loadNodeConfig(filepath.Join(configRoot, "node.env")) if err != nil { return err diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 1dd7978d0d..4cb8c9755a 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -1304,13 +1304,12 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string, co } if complete { if err := m.runHACommand(activationCtx, m.cfg.ActivationTimeout, currentDeployment, commandOutput, "wait-takeover", targetVersion); err != nil { - restartErr := m.runHACommand(activationCtx, m.cfg.ActivationTimeout, currentDeployment, commandOutput, "app-start", previousVersion, "any") - clearErr := m.clearActivationMarker() - if restartErr == nil && clearErr == nil { + restartErr := m.restartHAApplication(ctx, currentDeployment, previousVersion, commandOutput) + if restartErr == nil { m.fail(operationID, fmt.Errorf("updated peer did not take over; previous release restarted: %w", err), "") return } - m.fail(operationID, errors.Join(err, restartErr, clearErr), m.activationRecoveryCommand(currentDeployment, previousVersion)) + m.fail(operationID, errors.Join(err, restartErr), m.activationRecoveryCommand(currentDeployment, previousVersion)) return } } @@ -1378,6 +1377,20 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string, co _, _ = fmt.Fprintf(logFile, "[%s] upgrade completed\n", m.cfg.Now().UTC().Format(time.RFC3339)) } +func (m *Manager) restartHAApplication( + parent context.Context, + deployment string, + version string, + output io.Writer, +) error { + ctx, cancel := context.WithTimeout(context.WithoutCancel(parent), m.cfg.ActivationTimeout) + defer cancel() + if err := m.runHACommand(ctx, m.cfg.ActivationTimeout, deployment, output, "app-start", version, "any"); err != nil { + return err + } + return m.clearActivationMarker() +} + func (m *Manager) runPreflight(ctx context.Context, deployment string, output io.Writer) error { if m.cfg.DeploymentMode == DeploymentModeHA { return m.runHACommand(ctx, m.cfg.PreflightTimeout, deployment, output, "update-preflight") diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index c2b40eac13..2f493a1e37 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -463,6 +463,30 @@ func TestManagerHACompletionRestartsOldReleaseWhenTakeoverFails(t *testing.T) { assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[len(commands)-1].Args) } +func TestManagerHACompletionRestartsOldReleaseWhenStopFails(t *testing.T) { + // Arrange + installRoot := t.TempDir() + writeCurrentDeployment(t, installRoot, "v1.0.0") + bundle := releaseBundle(t, "v1.1.0") + server := releaseServer(t, "v1.1.0", "amd64", bundle, "") + runner := &haRecordingRunner{fail: map[string]error{"app-stop": assert.AnError}} + manager := newTestManagerWithConfig(t, installRoot, server, runner, func(cfg *Config) { + cfg.DeploymentMode = DeploymentModeHA + }) + + // Act + _, err := manager.TriggerCompleteWithID("v1.1.0", "11111111-1111-4111-8111-111111111111") + require.NoError(t, err) + completed := waitForTerminal(t, manager) + + // Assert + require.Equal(t, updaterapi.PhaseFailed, completed.Phase) + assert.Contains(t, completed.Error, "previous release restarted") + assert.Empty(t, completed.RecoveryCommand) + commands := runner.Commands() + assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[len(commands)-1].Args) +} + func TestManagerHACompletionInterruptedAfterSwapStartsTargetRelease(t *testing.T) { // Arrange installRoot := t.TempDir() From a1cb3bec1d0ec6d8b989bc1d7ab28f255d17fbc0 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 18:17:52 -0700 Subject: [PATCH 07/40] Retry interrupted HA recovery on startup --- server/internal/updater/manager.go | 19 +++++++++++++- server/internal/updater/manager_test.go | 33 +++++++++++-------------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 4cb8c9755a..cf5baf6ad6 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -770,7 +770,10 @@ func (m *Manager) RecoverApplication() error { m.operation.Message = "HA application recovery failed" m.operation.Error = recoveryErr.Error() m.operation.UpdatedAt = m.cfg.Now().UTC() - return m.persistLocked() + if persistErr := m.persistLocked(); persistErr != nil { + return errors.Join(recoveryErr, fmt.Errorf("persist HA application recovery failure: %w", persistErr)) + } + return recoveryErr } m.operation.RecoveryCommand = "" m.operation.RecoveryPending = false @@ -972,6 +975,14 @@ func (m *Manager) trigger(targetVersion, operationID string, idempotent, complet } return existing, nil } + if m.operation != nil && m.operation.RecoveryPending { + recoveryOperationID := m.operation.ID + m.mu.RUnlock() + return updaterapi.Operation{}, newTriggerError( + errTriggerBusy, + fmt.Sprintf("HA application recovery for operation %s is pending", recoveryOperationID), + ) + } m.mu.RUnlock() marker, err := m.readActivationMarker() if err != nil { @@ -1001,6 +1012,12 @@ func (m *Manager) trigger(targetVersion, operationID string, idempotent, complet } return *m.operation, nil } + if m.operation != nil && m.operation.RecoveryPending { + return updaterapi.Operation{}, newTriggerError( + errTriggerBusy, + fmt.Sprintf("HA application recovery for operation %s is pending", m.operation.ID), + ) + } if m.closing { return updaterapi.Operation{}, errTriggerClosing } diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index 2f493a1e37..e376391fef 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -1296,31 +1296,28 @@ func TestManagerRejectsAnotherUpgradeWhileActivationRecoveryIsPending(t *testing now := time.Date(2026, 8, 5, 8, 0, 0, 0, time.UTC) manager.mu.Lock() manager.operation = &updaterapi.Operation{ - ID: "pending-recovery", - TargetVersion: "v1.1.0", - Phase: updaterapi.PhaseFailed, - Message: "Activation layout requires manual recovery", - StartedAt: now, - UpdatedAt: now, - CompletedAt: &now, + ID: "pending-recovery", + TargetVersion: "v1.1.0", + Phase: updaterapi.PhaseFailed, + Message: "Activation layout requires manual recovery", + RecoveryCommand: "app-recover", + RecoveryPending: true, + StartedAt: now, + UpdatedAt: now, + CompletedAt: &now, } require.NoError(t, manager.persistLocked()) manager.mu.Unlock() - require.NoError(t, manager.writeActivationMarker(activationMarker{ - OperationID: "pending-recovery", - TargetVersion: "v1.1.0", - })) stateBefore := mustReadFile(t, filepath.Join(stateDir, stateFilename)) _, err = manager.Trigger("v1.2.0") - require.ErrorContains(t, err, "activation recovery for operation pending-recovery is pending") + require.ErrorContains(t, err, "HA application recovery for operation pending-recovery is pending") require.ErrorIs(t, err, errTriggerBusy) operation := manager.Status().Operation require.NotNil(t, operation) assert.Equal(t, "pending-recovery", operation.ID) assert.Equal(t, updaterapi.PhaseFailed, operation.Phase) assert.Equal(t, stateBefore, mustReadFile(t, filepath.Join(stateDir, stateFilename))) - assert.FileExists(t, filepath.Join(stateDir, activationMarkerFilename)) } func TestManagerProcessLockPreventsASecondDaemonFromMutatingState(t *testing.T) { @@ -1865,7 +1862,7 @@ func TestManagerRestartsHAApplicationAfterInterruptedSwap(t *testing.T) { assert.False(t, manager.Status().Operation.RecoveryPending) } -func TestManagerServesRecoveryStateAfterRestartFailure(t *testing.T) { +func TestManagerFailsStartupWhenHARecoveryFails(t *testing.T) { // Arrange installRoot := t.TempDir() writeCurrentDeployment(t, installRoot, "v1.0.0") @@ -1882,12 +1879,12 @@ func TestManagerServesRecoveryStateAfterRestartFailure(t *testing.T) { DeploymentMode: DeploymentModeHA, Runner: &haRecordingRunner{fail: map[string]error{"app-recover": errors.New("restart failed")}}, }) - require.NoError(t, err) - t.Cleanup(func() { assert.NoError(t, manager.Close()) }) + require.ErrorContains(t, err, "restart interrupted HA application") + require.Nil(t, manager) // Assert - operation := manager.Status().Operation - require.NotNil(t, operation) + var operation updaterapi.Operation + require.NoError(t, json.Unmarshal([]byte(mustReadFile(t, filepath.Join(stateDir, stateFilename))), &operation)) assert.True(t, operation.RecoveryPending) assert.NotEmpty(t, operation.RecoveryCommand) assert.Contains(t, operation.Error, "restart failed") From 7dfbbf928a9677c6d5ff3bfb1f483dcc63086457 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 19:44:57 -0700 Subject: [PATCH 08/40] Simplify HA update recovery --- server/cmd/fleet-ha/main.go | 13 ------------- server/internal/ha/deployment/update.go | 10 ++-------- server/internal/updater/manager.go | 2 +- server/internal/updater/manager_test.go | 10 +++++----- 4 files changed, 8 insertions(+), 27 deletions(-) diff --git a/server/cmd/fleet-ha/main.go b/server/cmd/fleet-ha/main.go index 59b7253d9d..dbd48bed9b 100644 --- a/server/cmd/fleet-ha/main.go +++ b/server/cmd/fleet-ha/main.go @@ -41,7 +41,6 @@ type cli struct { UpdatePreflight updatePreflightCmd `cmd:"" help:"prepare the current release for an application update"` AppStop appStopCmd `cmd:"" help:"stop the Fleet application services"` AppStart appStartCmd `cmd:"" help:"start the Fleet application services"` - AppRecover appRecoverCmd `cmd:"" help:"recover an interrupted Fleet application update"` WaitTakeover waitTakeoverCmd `cmd:"" help:"wait for the VIP to serve an application version"` } @@ -191,18 +190,6 @@ func (c *appStartCmd) Run(ctx context.Context) error { return deployment.StartApplication(ctx, root, c.Version, c.Mode == "passive") } -type appRecoverCmd struct { - Version string `arg:"" help:"application version to recover"` -} - -func (c *appRecoverCmd) Run(ctx context.Context) error { - root, err := deployment.ReleaseRoot() - if err != nil { - return err - } - return deployment.RecoverApplication(ctx, root, c.Version) -} - type waitTakeoverCmd struct { Version string `arg:"" help:"application version expected on the VIP"` } diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index dea86e8857..89de12e846 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -15,9 +15,8 @@ import ( "github.com/block/proto-fleet/server/internal/transportguard" ) -// Covers one 10s lease lifetime, the coordinator's two-lease acquire budget, -// and a small keepalived scheduling margin. -const vipTakeoverTimeout = 35 * time.Second +// Bounds the planned interruption promised by the HA update flow. +const vipTakeoverTimeout = 15 * time.Second func requirePassiveStatus(ctx context.Context, envPath string) (StatusReport, error) { report, err := Status(ctx, envPath) @@ -231,11 +230,6 @@ func StartApplication(ctx context.Context, root, targetVersion string, requirePa return startApplication(ctx, root, targetVersion, requirePassive) } -// RecoverApplication converges both Fleet services after interrupted activation. -func RecoverApplication(ctx context.Context, root, targetVersion string) error { - return startApplication(ctx, root, targetVersion, false) -} - func startApplication(ctx context.Context, root, targetVersion string, requirePassive bool) error { config, err := loadNodeConfig(filepath.Join(configRoot, "node.env")) if err != nil { diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index cf5baf6ad6..5446bf5de0 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -764,7 +764,7 @@ func (m *Manager) RecoverApplication() error { } ctx, cancel := context.WithTimeout(context.Background(), m.cfg.ActivationTimeout) defer cancel() - if err := m.runHACommand(ctx, m.cfg.ActivationTimeout, deployment, io.Discard, "app-recover", version); err != nil { + if err := m.runHACommand(ctx, m.cfg.ActivationTimeout, deployment, io.Discard, "app-start", version, "any"); err != nil { recoveryErr := fmt.Errorf("restart interrupted HA application: %w", err) m.operation.Phase = updaterapi.PhaseFailed m.operation.Message = "HA application recovery failed" diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index e376391fef..9983357057 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -408,7 +408,7 @@ func TestManagerHAInterruptedAfterStopRestartsCurrentApplication(t *testing.T) { assert.Contains(t, operation.Message, "HA application restarted") commands := runner.Commands() require.Len(t, commands, 1) - assert.Equal(t, []string{"app-recover", "v1.0.0"}, commands[0].Args) + assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[0].Args) } func TestManagerHACompletionWaitsForUpdatedPeerBeforeSwap(t *testing.T) { @@ -512,7 +512,7 @@ func TestManagerHACompletionInterruptedAfterSwapStartsTargetRelease(t *testing.T require.Empty(t, operation.RecoveryCommand) commands := runner.Commands() require.Len(t, commands, 1) - assert.Equal(t, []string{"app-recover", "v1.1.0"}, commands[0].Args) + assert.Equal(t, []string{"app-start", "v1.1.0", "any"}, commands[0].Args) } func TestManagerTriggerWithIDDeduplicatesConcurrentAdmission(t *testing.T) { @@ -1300,7 +1300,7 @@ func TestManagerRejectsAnotherUpgradeWhileActivationRecoveryIsPending(t *testing TargetVersion: "v1.1.0", Phase: updaterapi.PhaseFailed, Message: "Activation layout requires manual recovery", - RecoveryCommand: "app-recover", + RecoveryCommand: "app-start", RecoveryPending: true, StartedAt: now, UpdatedAt: now, @@ -1857,7 +1857,7 @@ func TestManagerRestartsHAApplicationAfterInterruptedSwap(t *testing.T) { // Assert commands := runner.Commands() require.Len(t, commands, 1) - assert.Equal(t, []string{"app-recover", "v1.0.0"}, commands[0].Args) + assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[0].Args) assert.Empty(t, manager.Status().Operation.RecoveryCommand) assert.False(t, manager.Status().Operation.RecoveryPending) } @@ -1877,7 +1877,7 @@ func TestManagerFailsStartupWhenHARecoveryFails(t *testing.T) { manager, err := NewManager(Config{ InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", DeploymentMode: DeploymentModeHA, - Runner: &haRecordingRunner{fail: map[string]error{"app-recover": errors.New("restart failed")}}, + Runner: &haRecordingRunner{fail: map[string]error{"app-start": errors.New("restart failed")}}, }) require.ErrorContains(t, err, "restart interrupted HA application") require.Nil(t, manager) From 75c67baa9096fe8b6aaac226499a0df79f779115 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 20:02:00 -0700 Subject: [PATCH 09/40] Separate HA takeover deadline from SLO --- deployment-files/ha/README.md | 5 +++-- server/internal/ha/deployment/update.go | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/deployment-files/ha/README.md b/deployment-files/ha/README.md index 31d4c748ba..514d0fb8e2 100644 --- a/deployment-files/ha/README.md +++ b/deployment-files/ha/README.md @@ -203,8 +203,9 @@ rather than upgraded through this workflow. The updater stages everything first, stops the local Fleet containers, and waits for the updated peer to serve the VIP with the target version. Only then does it swap and restart the local application as passive. If takeover does -not complete within 30 seconds, it restarts the old local release without -swapping. This is a bounded interruption, not a zero-downtime update. +not complete within 35 seconds, it restarts the old local release without +swapping. Qualification still requires a healthy takeover in less than 15 +seconds. This is a bounded interruption, not a zero-downtime update. ## Qualification diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index 89de12e846..0353f7fe85 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -15,8 +15,8 @@ import ( "github.com/block/proto-fleet/server/internal/transportguard" ) -// Bounds the planned interruption promised by the HA update flow. -const vipTakeoverTimeout = 15 * time.Second +// Leaves room for lease expiry and acquisition before falling back to the old release. +const vipTakeoverTimeout = 35 * time.Second func requirePassiveStatus(ctx context.Context, envPath string) (StatusReport, error) { report, err := Status(ctx, envPath) From 05f057e6118583c54d9dba79c5188cb23455eb2f Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 20:29:05 -0700 Subject: [PATCH 10/40] Repair update layout before HA startup --- server/internal/updater/manager.go | 20 +++++++++++++++++ server/internal/updater/manager_test.go | 30 +++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 5446bf5de0..fcbf42bd60 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -569,6 +569,19 @@ func wrapIfError(message string, err error) error { } func NewManager(cfg Config) (*Manager, error) { + return newManager(cfg, true) +} + +// RepairStartup restores a crash-interrupted deployment layout without starting Fleet. +func RepairStartup(cfg Config) error { + manager, err := newManager(cfg, false) + if err != nil { + return err + } + return manager.Close() +} + +func newManager(cfg Config, recoverApplication bool) (*Manager, error) { if !filepath.IsAbs(cfg.InstallRoot) { return nil, fmt.Errorf("install root must be absolute") } @@ -695,6 +708,13 @@ func NewManager(cfg Config) (*Manager, error) { _ = logRoot.Close() return nil, err } + if recoverApplication { + if err := m.recoverHAApplication(); err != nil { + _ = processLock.Close() + _ = logRoot.Close() + return nil, err + } + } protectedLogName := "" if m.operation != nil { protectedLogName = operationLogFilename(m.operation.ID) diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index 9983357057..b1ebb411f2 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -1862,6 +1862,36 @@ func TestManagerRestartsHAApplicationAfterInterruptedSwap(t *testing.T) { assert.False(t, manager.Status().Operation.RecoveryPending) } +func TestRepairStartupRestoresLayoutWithoutStartingHAApplication(t *testing.T) { + t.Parallel() + + // Arrange + installRoot := t.TempDir() + writeCurrentDeployment(t, installRoot, "v1.0.0") + require.NoError(t, os.Rename( + filepath.Join(installRoot, "deployment"), + filepath.Join(installRoot, "deployment.previous"), + )) + stateDir := filepath.Join(t.TempDir(), "state") + writeInterruptedOperationState(t, stateDir, "v1.1.0") + runner := &haRecordingRunner{} + + // Act + err := RepairStartup(Config{ + InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", + DeploymentMode: DeploymentModeHA, Runner: runner, + }) + + // Assert + require.NoError(t, err) + assert.Empty(t, runner.Commands()) + assert.Equal(t, "v1.0.0", mustReadVersion(t, filepath.Join(installRoot, "deployment", "version.txt"))) + assert.NoDirExists(t, filepath.Join(installRoot, "deployment.previous")) + var operation updaterapi.Operation + require.NoError(t, json.Unmarshal([]byte(mustReadFile(t, filepath.Join(stateDir, stateFilename))), &operation)) + assert.True(t, operation.RecoveryPending) +} + func TestManagerFailsStartupWhenHARecoveryFails(t *testing.T) { // Arrange installRoot := t.TempDir() From bdef2d19fce7453924efb2d7e104be055776c5d4 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 20:52:41 -0700 Subject: [PATCH 11/40] Reconcile updater handoff before startup repair --- server/internal/updater/manager.go | 3 +++ server/internal/updater/manager_test.go | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index fcbf42bd60..e9e846f08d 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -574,6 +574,9 @@ func NewManager(cfg Config) (*Manager, error) { // RepairStartup restores a crash-interrupted deployment layout without starting Fleet. func RepairStartup(cfg Config) error { + if _, err := PrepareSelfUpdateStartup(cfg.SelfUpdatePath, ""); err != nil { + return err + } manager, err := newManager(cfg, false) if err != nil { return err diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index b1ebb411f2..40ce58f2cc 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -1892,6 +1892,22 @@ func TestRepairStartupRestoresLayoutWithoutStartingHAApplication(t *testing.T) { assert.True(t, operation.RecoveryPending) } +func TestRepairStartupRestoresInterruptedSelfUpdateBeforeState(t *testing.T) { + // Arrange + destination := installSelfUpdateForHandoffTest(t) + stateDir := filepath.Join(t.TempDir(), "state") + + // Act + err := RepairStartup(Config{ + InstallRoot: t.TempDir(), StateDir: stateDir, SelfUpdatePath: destination, + }) + + // Assert + require.ErrorIs(t, err, ErrInterruptedSelfUpdateRestored) + assert.Equal(t, "old updater", mustReadFile(t, destination)) + assert.NoDirExists(t, stateDir) +} + func TestManagerFailsStartupWhenHARecoveryFails(t *testing.T) { // Arrange installRoot := t.TempDir() From 0fdbc8488214d332044802d06b7660b05657420b Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 21:22:18 -0700 Subject: [PATCH 12/40] Persist pending HA application recovery --- server/internal/updater/manager.go | 1 + server/internal/updater/manager_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index e9e846f08d..d6a68282c2 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -1957,6 +1957,7 @@ func (m *Manager) fail(id string, err error, recovery string) { m.operation.Message = "Upgrade failed" m.operation.Error = err.Error() m.operation.RecoveryCommand = recovery + m.operation.RecoveryPending = m.cfg.DeploymentMode == DeploymentModeHA && recovery != "" m.operation.UpdatedAt = now m.operation.CompletedAt = &now if persistErr := m.persistLocked(); persistErr != nil { diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index 40ce58f2cc..78f0219c19 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -274,6 +274,7 @@ func TestManagerHAUpdateKeepsForwardRecoveryWhenStartupFails(t *testing.T) { assert.Contains(t, completed.RecoveryCommand, "app-start") assert.Contains(t, completed.RecoveryCommand, "v1.1.0") assert.True(t, strings.HasSuffix(completed.RecoveryCommand, " any")) + assert.True(t, completed.RecoveryPending) assert.Contains(t, completed.Error, "new stack failed to start") assert.Equal(t, "v1.1.0", mustReadVersion(t, filepath.Join(installRoot, "deployment", "version.txt"))) commands := runner.Commands() From 11d1df895c7f952c855a37fc83d5c3984755d492 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 21:48:41 -0700 Subject: [PATCH 13/40] Avoid duplicating HA migration contract --- deployment-files/ha/README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/deployment-files/ha/README.md b/deployment-files/ha/README.md index 514d0fb8e2..b65ba9956c 100644 --- a/deployment-files/ha/README.md +++ b/deployment-files/ha/README.md @@ -180,11 +180,6 @@ only `fleet-api` and `fleet-client`; etcd, Patroni, PostgreSQL, and keepalived remain running. The command returns only after the target version is healthy and passive. -The old active and new passive share the database during this rolling window. -Every migration in the target release must be expand-only and remain compatible -with the previous release; destructive contract migrations belong in a later -release after both HA hosts have advanced. - After the peer is confirmed on the target release, complete the update from the old active host: From 9a096dd42f9a30389d0cbb179daaaa5140ea0f07 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 22:58:48 -0700 Subject: [PATCH 14/40] Preserve simplified HA stop contract --- server/cmd/fleet-ha/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/cmd/fleet-ha/main.go b/server/cmd/fleet-ha/main.go index dbd48bed9b..be1e290788 100644 --- a/server/cmd/fleet-ha/main.go +++ b/server/cmd/fleet-ha/main.go @@ -14,7 +14,6 @@ import ( "github.com/alecthomas/kong" "github.com/google/uuid" - "github.com/block/proto-fleet/server/internal/ha" "github.com/block/proto-fleet/server/internal/ha/deployment" "github.com/block/proto-fleet/server/internal/updaterapi" ) From c9089f040fc3a4bd5e71a108d402713c63a8b520 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 23:20:49 -0700 Subject: [PATCH 15/40] Recheck HA peer before active shutdown --- server/cmd/fleet-ha/main.go | 5 ++++- server/internal/ha/deployment/update.go | 5 ----- server/internal/updater/manager.go | 2 +- server/internal/updater/manager_test.go | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/server/cmd/fleet-ha/main.go b/server/cmd/fleet-ha/main.go index be1e290788..b4c86cff48 100644 --- a/server/cmd/fleet-ha/main.go +++ b/server/cmd/fleet-ha/main.go @@ -14,6 +14,7 @@ import ( "github.com/alecthomas/kong" "github.com/google/uuid" + "github.com/block/proto-fleet/server/internal/ha" "github.com/block/proto-fleet/server/internal/ha/deployment" "github.com/block/proto-fleet/server/internal/updaterapi" ) @@ -148,10 +149,12 @@ func (c *requirePassiveCmd) Run(ctx context.Context) error { type requireActiveCmd struct { NodeEnv string `arg:"" type:"path" help:"node environment file"` + Version string `arg:"" help:"target application version"` } func (c *requireActiveCmd) Run(ctx context.Context) error { - return deployment.RequireActive(ctx, c.NodeEnv) + _, err := deployment.ValidateActiveUpdate(ctx, c.NodeEnv, c.Version) + return err } type updatePreflightCmd struct{} diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index 0353f7fe85..35145144c4 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -63,11 +63,6 @@ var releaseImageRepositories = [...]string{ "proto-fleet-client", } -func RequireActive(ctx context.Context, envPath string) error { - _, err := requireActiveStatus(ctx, envPath) - return err -} - func requireActiveStatus(ctx context.Context, envPath string) (StatusReport, error) { report, err := Status(ctx, envPath) if err != nil { diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index d6a68282c2..8df6700618 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -1318,7 +1318,7 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string, co requireArgs := []string{"require-passive", haNodeEnvPath, targetVersion} if complete { requiredRole = "active" - requireArgs = []string{"require-active", haNodeEnvPath} + requireArgs = []string{"require-active", haNodeEnvPath, targetVersion} } if err := m.runHACommand(ctx, m.cfg.ActivationTimeout, currentDeployment, commandOutput, requireArgs...); err != nil { m.fail(operationID, fmt.Errorf("local Fleet is not %s immediately before activation: %w", requiredRole, err), recovery) diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index 78f0219c19..dd9b09dd69 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -433,7 +433,7 @@ func TestManagerHACompletionWaitsForUpdatedPeerBeforeSwap(t *testing.T) { commands := runner.Commands() require.Len(t, commands, 5) assert.Equal(t, []string{"update-preflight"}, commands[0].Args) - assert.Equal(t, []string{"require-active", "/etc/proto-fleet/ha/node.env"}, commands[1].Args) + assert.Equal(t, []string{"require-active", "/etc/proto-fleet/ha/node.env", "v1.1.0"}, commands[1].Args) assert.Equal(t, []string{"app-stop", "active"}, commands[2].Args) assert.Equal(t, []string{"wait-takeover", "v1.1.0"}, commands[3].Args) assert.Equal(t, []string{"app-start", "v1.1.0", "passive"}, commands[4].Args) From 94ac4e1c287b1420b0a655b32083e67d71101bc6 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Fri, 7 Aug 2026 23:58:37 -0700 Subject: [PATCH 16/40] Bound HA completion outage timing --- server/internal/ha/deployment/update.go | 4 +-- server/internal/ha/update_timing.go | 8 +++++ server/internal/updater/manager.go | 13 +++++-- server/internal/updater/manager_test.go | 47 ++++++++++++++++++++----- 4 files changed, 60 insertions(+), 12 deletions(-) create mode 100644 server/internal/ha/update_timing.go diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index 35145144c4..55ac4f37cf 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -342,7 +342,7 @@ func WaitForVIPVersion(ctx context.Context, envPath, targetVersion string) error transport := &http.Transport{TLSClientConfig: tlsConfig, Proxy: nil} client := &http.Client{Transport: transport, Timeout: 2 * time.Second, CheckRedirect: transportguard.RejectRedirect} defer transport.CloseIdleConnections() - deadline, cancel := context.WithTimeout(ctx, vipTakeoverTimeout) + deadline, cancel := context.WithTimeout(ctx, ha.UpdateTakeoverTimeout) defer cancel() endpoint := "https://" + config.VirtualIP + "/api-proxy/health" for { @@ -364,7 +364,7 @@ func WaitForVIPVersion(ctx context.Context, envPath, targetVersion string) error } select { case <-deadline.Done(): - return fmt.Errorf("updated peer did not serve the VIP within %s", vipTakeoverTimeout) + return fmt.Errorf("updated peer did not serve the VIP within %s", ha.UpdateTakeoverTimeout) case <-time.After(500 * time.Millisecond): } } diff --git a/server/internal/ha/update_timing.go b/server/internal/ha/update_timing.go new file mode 100644 index 0000000000..f3fab8e21e --- /dev/null +++ b/server/internal/ha/update_timing.go @@ -0,0 +1,8 @@ +package ha + +import "time" + +const ( + UpdateActiveStopTimeout = 5 * time.Second + UpdateTakeoverTimeout = 35 * time.Second +) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 8df6700618..67d01f6956 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -29,6 +29,7 @@ import ( "github.com/google/uuid" "golang.org/x/mod/semver" + "github.com/block/proto-fleet/server/internal/ha" "github.com/block/proto-fleet/server/internal/updaterapi" ) @@ -1338,12 +1339,20 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string, co m.failActivation(operationID, targetVersion, fmt.Errorf("persist passive application recovery: %w", err), logFile, false) return } - if err := m.runHACommand(activationCtx, m.cfg.ActivationTimeout, currentDeployment, commandOutput, "app-stop", requiredRole); err != nil { + haActivationCtx := activationCtx + stopTimeout := m.cfg.ActivationTimeout + cancelOutage := func() {} + if complete { + haActivationCtx, cancelOutage = context.WithTimeout(activationCtx, ha.UpdateTakeoverTimeout) + stopTimeout = ha.UpdateActiveStopTimeout + } + defer cancelOutage() + if err := m.runHACommand(haActivationCtx, stopTimeout, currentDeployment, commandOutput, "app-stop", requiredRole); err != nil { m.failActivation(operationID, targetVersion, fmt.Errorf("stop HA application: %w", err), logFile, true) return } if complete { - if err := m.runHACommand(activationCtx, m.cfg.ActivationTimeout, currentDeployment, commandOutput, "wait-takeover", targetVersion); err != nil { + if err := m.runHACommand(haActivationCtx, ha.UpdateTakeoverTimeout, currentDeployment, commandOutput, "wait-takeover", targetVersion); err != nil { restartErr := m.restartHAApplication(ctx, currentDeployment, previousVersion, commandOutput) if restartErr == nil { m.fail(operationID, fmt.Errorf("updated peer did not take over; previous release restarted: %w", err), "") diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index dd9b09dd69..83931ac754 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -52,27 +52,34 @@ type recordingRunner struct { } type haRecordingRunner struct { - mu sync.Mutex - commands []recordedCommand - fail map[string]error + mu sync.Mutex + commands []recordedCommand + fail map[string]error + blockStop bool } -func (r *haRecordingRunner) Run(_ context.Context, dir string, output io.Writer, name string, args ...string) error { +func (r *haRecordingRunner) Run(ctx context.Context, dir string, output io.Writer, name string, args ...string) error { r.mu.Lock() - defer r.mu.Unlock() r.commands = append(r.commands, recordedCommand{Dir: dir, Name: name, Args: append([]string(nil), args...)}) + blockStop := r.blockStop && len(args) > 0 && args[0] == "app-stop" if len(args) == 1 && args[0] == "--version" { + r.mu.Unlock() if _, err := fmt.Fprintln(output, "v1.1.0"); err != nil { return fmt.Errorf("write candidate version: %w", err) } return nil } + var err error if len(args) > 0 { - err := r.fail[args[0]] + err = r.fail[args[0]] delete(r.fail, args[0]) - return err } - return nil + r.mu.Unlock() + if blockStop { + <-ctx.Done() + return fmt.Errorf("blocked application stop: %w", ctx.Err()) + } + return err } func (r *haRecordingRunner) Commands() []recordedCommand { @@ -488,6 +495,30 @@ func TestManagerHACompletionRestartsOldReleaseWhenStopFails(t *testing.T) { assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[len(commands)-1].Args) } +func TestManagerHACompletionRestartsOldReleaseWhenStopBlocks(t *testing.T) { + // Arrange + installRoot := t.TempDir() + writeCurrentDeployment(t, installRoot, "v1.0.0") + bundle := releaseBundle(t, "v1.1.0") + server := releaseServer(t, "v1.1.0", "amd64", bundle, "") + runner := &haRecordingRunner{fail: make(map[string]error), blockStop: true} + manager := newTestManagerWithConfig(t, installRoot, server, runner, func(cfg *Config) { + cfg.DeploymentMode = DeploymentModeHA + cfg.ActivationTimeout = 250 * time.Millisecond + }) + + // Act + _, err := manager.TriggerCompleteWithID("v1.1.0", "11111111-1111-4111-8111-111111111111") + require.NoError(t, err) + completed := waitForTerminal(t, manager) + + // Assert + require.Equal(t, updaterapi.PhaseFailed, completed.Phase) + assert.Contains(t, completed.Error, "previous release restarted") + commands := runner.Commands() + assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[len(commands)-1].Args) +} + func TestManagerHACompletionInterruptedAfterSwapStartsTargetRelease(t *testing.T) { // Arrange installRoot := t.TempDir() From 76f8c5eb7e7fc62174fcc976628a2f30aee04898 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 00:23:24 -0700 Subject: [PATCH 17/40] Harden HA completion retries --- .../internal/domain/updates/executor_test.go | 20 ++++++++++++++++++- server/internal/ha/deployment/update.go | 16 ++++++++++++--- server/internal/updater/manager.go | 9 +++++---- server/internal/updater/manager_test.go | 4 ++++ server/internal/updaterapi/client.go | 5 +++-- server/internal/updaterapi/types.go | 1 + 6 files changed, 45 insertions(+), 10 deletions(-) diff --git a/server/internal/domain/updates/executor_test.go b/server/internal/domain/updates/executor_test.go index 9d693e5f55..2af50a7d41 100644 --- a/server/internal/domain/updates/executor_test.go +++ b/server/internal/domain/updates/executor_test.go @@ -143,7 +143,7 @@ func TestUnixExecutorClientTriggerComplete(t *testing.T) { observed <- executorRequestObservation{trigger: request, decodeErr: decodeErr} w.WriteHeader(http.StatusAccepted) _ = json.NewEncoder(w).Encode(updaterapi.TriggerResponse{Operation: updaterapi.Operation{ - ID: operationID, TargetVersion: "v1.2.3", Phase: updaterapi.PhaseQueued, + ID: operationID, TargetVersion: "v1.2.3", Complete: true, Phase: updaterapi.PhaseQueued, }}) })) @@ -234,6 +234,24 @@ func TestUnixExecutorClientRejectsMismatchedTriggerIdentity(t *testing.T) { assert.ErrorAs(t, err, &protocolErr) } +func TestUnixExecutorClientRejectsMismatchedCompletionMode(t *testing.T) { + // Arrange + client := startExecutorTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(updaterapi.TriggerResponse{Operation: updaterapi.Operation{ + ID: "11111111-1111-4111-8111-111111111111", TargetVersion: "v1.2.3", + }}) + })) + + // Act + _, err := client.TriggerComplete(t.Context(), "11111111-1111-4111-8111-111111111111", "v1.2.3") + + // Assert + require.Error(t, err) + var protocolErr *updaterapi.ProtocolError + assert.ErrorAs(t, err, &protocolErr) +} + func TestUnixExecutorClientUnavailableSocket(t *testing.T) { t.Parallel() diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index 55ac4f37cf..af226737b9 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -210,7 +210,9 @@ func StopApplication(ctx context.Context, root string, expectedRole ha.RuntimeRo } // The crash-only design intentionally has no maintenance lease. If the role // changes after this final proof, normal update recovery restarts Fleet. - if err := RunCompose(ctx, fleetComposeArgsAt(root, "stop", "fleet-api", "fleet-client")); err != nil { + // Leave two seconds inside the updater's five-second active-stop deadline + // for Compose and Docker to confirm both containers stopped. + if err := RunCompose(ctx, fleetComposeArgsAt(root, "stop", "--timeout", "3", "fleet-api", "fleet-client")); err != nil { return fmt.Errorf("stop HA application: %w", err) } return nil @@ -256,8 +258,16 @@ func startApplication(ctx context.Context, root, targetVersion string, requirePa return fmt.Errorf("verify local HA application is stopped: %w", statusErr) } args := fleetComposeArgsAt(root, "up", "-d", "--no-deps", "--no-build", "--pull", "never", "fleet-api", "fleet-client") - if err := RunCompose(ctx, args); err != nil { - return fmt.Errorf("start HA application: %w", err) + for { + err := RunCompose(ctx, args) + if err == nil { + break + } + select { + case <-ctx.Done(): + return fmt.Errorf("start HA application after Compose failure %v: %w", err, ctx.Err()) + case <-time.After(2 * time.Second): + } } for { report, err := Status(ctx, filepath.Join(configRoot, "node.env")) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 67d01f6956..7e24326c16 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -991,10 +991,10 @@ func (m *Manager) trigger(targetVersion, operationID string, idempotent, complet if idempotent && m.operation != nil && m.operation.ID == operationID { existing := *m.operation m.mu.RUnlock() - if existing.TargetVersion != targetVersion { + if existing.TargetVersion != targetVersion || existing.Complete != complete { return updaterapi.Operation{}, newTriggerError( errTriggerInvalid, - "operation id is already associated with another target", + "operation id is already associated with another update", ) } return existing, nil @@ -1028,10 +1028,10 @@ func (m *Manager) trigger(targetVersion, operationID string, idempotent, complet m.mu.Lock() defer m.mu.Unlock() if idempotent && m.operation != nil && m.operation.ID == operationID { - if m.operation.TargetVersion != targetVersion { + if m.operation.TargetVersion != targetVersion || m.operation.Complete != complete { return updaterapi.Operation{}, newTriggerError( errTriggerInvalid, - "operation id is already associated with another target", + "operation id is already associated with another update", ) } return *m.operation, nil @@ -1090,6 +1090,7 @@ func (m *Manager) trigger(targetVersion, operationID string, idempotent, complet op := &updaterapi.Operation{ ID: operationID, TargetVersion: targetVersion, + Complete: complete, Phase: updaterapi.PhaseQueued, Message: "Upgrade queued", StartedAt: now, diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index 83931ac754..d479a8b3f4 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -437,6 +437,7 @@ func TestManagerHACompletionWaitsForUpdatedPeerBeforeSwap(t *testing.T) { // Assert require.Equal(t, updaterapi.PhaseSucceeded, completed.Phase, completed.Error) + require.True(t, completed.Complete) commands := runner.Commands() require.Len(t, commands, 5) assert.Equal(t, []string{"update-preflight"}, commands[0].Args) @@ -444,6 +445,9 @@ func TestManagerHACompletionWaitsForUpdatedPeerBeforeSwap(t *testing.T) { assert.Equal(t, []string{"app-stop", "active"}, commands[2].Args) assert.Equal(t, []string{"wait-takeover", "v1.1.0"}, commands[3].Args) assert.Equal(t, []string{"app-start", "v1.1.0", "passive"}, commands[4].Args) + + _, err = manager.TriggerWithID("v1.1.0", "11111111-1111-4111-8111-111111111111") + require.ErrorContains(t, err, "operation id is already associated with another update") } func TestManagerHACompletionRestartsOldReleaseWhenTakeoverFails(t *testing.T) { diff --git a/server/internal/updaterapi/client.go b/server/internal/updaterapi/client.go index 970464b693..0239218617 100644 --- a/server/internal/updaterapi/client.go +++ b/server/internal/updaterapi/client.go @@ -85,11 +85,12 @@ func (c *Client) trigger(ctx context.Context, operationID, targetVersion string, if err := c.do(ctx, http.MethodPost, "/v1/upgrade", request, &response); err != nil { return Operation{}, err } - if response.Operation.ID != operationID || response.Operation.TargetVersion != targetVersion { + if response.Operation.ID != operationID || response.Operation.TargetVersion != targetVersion || response.Operation.Complete != complete { return Operation{}, &ProtocolError{Cause: fmt.Errorf( - "operation identity mismatch: got id %q target %q", + "operation identity mismatch: got id %q target %q complete %t", response.Operation.ID, response.Operation.TargetVersion, + response.Operation.Complete, )} } return response.Operation, nil diff --git a/server/internal/updaterapi/types.go b/server/internal/updaterapi/types.go index 8eefadb6c8..d9870f7fc3 100644 --- a/server/internal/updaterapi/types.go +++ b/server/internal/updaterapi/types.go @@ -24,6 +24,7 @@ func (p Phase) Terminal() bool { type Operation struct { ID string `json:"id"` TargetVersion string `json:"target_version"` + Complete bool `json:"complete,omitempty"` Phase Phase `json:"phase"` Message string `json:"message,omitempty"` StartedAt time.Time `json:"started_at"` From cca456c2fce2fec65a37b13f359517c715919c2a Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 00:29:21 -0700 Subject: [PATCH 18/40] Stop replaying completed HA failures --- server/internal/updater/manager.go | 2 +- server/internal/updater/manager_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 7e24326c16..ea8d3f3c7c 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -1967,7 +1967,7 @@ func (m *Manager) fail(id string, err error, recovery string) { m.operation.Message = "Upgrade failed" m.operation.Error = err.Error() m.operation.RecoveryCommand = recovery - m.operation.RecoveryPending = m.cfg.DeploymentMode == DeploymentModeHA && recovery != "" + m.operation.RecoveryPending = false m.operation.UpdatedAt = now m.operation.CompletedAt = &now if persistErr := m.persistLocked(); persistErr != nil { diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index d479a8b3f4..a65032a1cb 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -281,7 +281,7 @@ func TestManagerHAUpdateKeepsForwardRecoveryWhenStartupFails(t *testing.T) { assert.Contains(t, completed.RecoveryCommand, "app-start") assert.Contains(t, completed.RecoveryCommand, "v1.1.0") assert.True(t, strings.HasSuffix(completed.RecoveryCommand, " any")) - assert.True(t, completed.RecoveryPending) + assert.False(t, completed.RecoveryPending) assert.Contains(t, completed.Error, "new stack failed to start") assert.Equal(t, "v1.1.0", mustReadVersion(t, filepath.Join(installRoot, "deployment", "version.txt"))) commands := runner.Commands() From f010649dc268e75cbb3e7511f9ac92e3117146a3 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 00:52:35 -0700 Subject: [PATCH 19/40] Keep HA active shutdown within its hard deadline --- server/internal/ha/deployment/update.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index af226737b9..5494bab33d 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -210,9 +210,9 @@ func StopApplication(ctx context.Context, root string, expectedRole ha.RuntimeRo } // The crash-only design intentionally has no maintenance lease. If the role // changes after this final proof, normal update recovery restarts Fleet. - // Leave two seconds inside the updater's five-second active-stop deadline - // for Compose and Docker to confirm both containers stopped. - if err := RunCompose(ctx, fleetComposeArgsAt(root, "stop", "--timeout", "3", "fleet-api", "fleet-client")); err != nil { + // The HA workload is crash-only. Avoid per-service graceful-stop timers so + // the updater's five-second deadline remains a hard total shutdown bound. + if err := RunCompose(ctx, fleetComposeArgsAt(root, "kill", "fleet-api", "fleet-client")); err != nil { return fmt.Errorf("stop HA application: %w", err) } return nil From 9de38be18f58c964b1f46181554698d09fc67d7d Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 01:35:31 -0700 Subject: [PATCH 20/40] Remove redundant HA update helpers --- server/internal/ha/deployment/update.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index 5494bab33d..d20db3d32b 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -15,9 +15,6 @@ import ( "github.com/block/proto-fleet/server/internal/transportguard" ) -// Leaves room for lease expiry and acquisition before falling back to the old release. -const vipTakeoverTimeout = 35 * time.Second - func requirePassiveStatus(ctx context.Context, envPath string) (StatusReport, error) { report, err := Status(ctx, envPath) if err != nil { @@ -224,10 +221,6 @@ func updatedPassivePeerReady(status fleetHostStatus, targetVersion string) bool // StartApplication starts the target release and proves it serves its observed HA role. func StartApplication(ctx context.Context, root, targetVersion string, requirePassive bool) error { - return startApplication(ctx, root, targetVersion, requirePassive) -} - -func startApplication(ctx context.Context, root, targetVersion string, requirePassive bool) error { config, err := loadNodeConfig(filepath.Join(configRoot, "node.env")) if err != nil { return err From fc2a1251db61b77dde16a3dd31705107e8dbe3c0 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 02:02:53 -0700 Subject: [PATCH 21/40] Require final HA update readiness --- server/cmd/fleet-ha/main.go | 4 ++-- server/internal/ha/deployment/update.go | 17 +++++++++++------ server/internal/ha/deployment/update_test.go | 20 ++++++++++++++++++-- server/internal/updater/manager.go | 10 +++++++--- server/internal/updater/manager_test.go | 2 +- 5 files changed, 39 insertions(+), 14 deletions(-) diff --git a/server/cmd/fleet-ha/main.go b/server/cmd/fleet-ha/main.go index b4c86cff48..d228e4962c 100644 --- a/server/cmd/fleet-ha/main.go +++ b/server/cmd/fleet-ha/main.go @@ -181,7 +181,7 @@ func (c *appStopCmd) Run(ctx context.Context) error { type appStartCmd struct { Version string `arg:"" help:"application version to start"` - Mode string `arg:"" optional:"" default:"passive" enum:"passive,any" help:"required HA role after startup"` + Mode string `arg:"" optional:"" default:"passive" enum:"passive,complete,any" help:"required HA readiness after startup"` } func (c *appStartCmd) Run(ctx context.Context) error { @@ -189,7 +189,7 @@ func (c *appStartCmd) Run(ctx context.Context) error { if err != nil { return err } - return deployment.StartApplication(ctx, root, c.Version, c.Mode == "passive") + return deployment.StartApplication(ctx, root, c.Version, c.Mode != "any", c.Mode == "complete") } type waitTakeoverCmd struct { diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index d20db3d32b..ddd1c3dc65 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -220,7 +220,7 @@ func updatedPassivePeerReady(status fleetHostStatus, targetVersion string) bool } // StartApplication starts the target release and proves it serves its observed HA role. -func StartApplication(ctx context.Context, root, targetVersion string, requirePassive bool) error { +func StartApplication(ctx context.Context, root, targetVersion string, requirePassive, requireFailoverReady bool) error { config, err := loadNodeConfig(filepath.Join(configRoot, "node.env")) if err != nil { return err @@ -236,6 +236,7 @@ func StartApplication(ctx context.Context, root, targetVersion string, requirePa probeFleetHost(ctx, tlsConfig, config.VirtualIP, config.NodeIP), targetVersion, requirePassive, + requireFailoverReady, ) if readinessErr != nil { return readinessErr @@ -266,7 +267,7 @@ func StartApplication(ctx context.Context, root, targetVersion string, requirePa report, err := Status(ctx, filepath.Join(configRoot, "node.env")) if err == nil { publicStatus := probeFleetHost(ctx, tlsConfig, config.VirtualIP, config.NodeIP) - ready, readinessErr := updatedApplicationReady(report, publicStatus, targetVersion, requirePassive) + ready, readinessErr := updatedApplicationReady(report, publicStatus, targetVersion, requirePassive, requireFailoverReady) if readinessErr != nil { return readinessErr } @@ -292,13 +293,17 @@ func applicationMayConverge(runtime ha.Status, targetVersion string, requirePass return runtime.Role == ha.RoleActive || runtime.Role == ha.RolePassive } -func rollingUpdateApplicationReady(report StatusReport, public fleetHostStatus, targetVersion string) (bool, error) { +func rollingUpdateApplicationReady(report StatusReport, public fleetHostStatus, targetVersion string, requireFailoverReady bool) (bool, error) { if report.Runtime.Observation == ha.ObservationCurrent && report.Runtime.Role == ha.RoleActive { return false, errors.New("updated node became active; inspect the peer before retrying") } + controlReady := rollingUpdateControlReady(report.Control) + if requireFailoverReady { + controlReady = report.Control != nil && report.Control.FailoverReady + } return report.Runtime.Role == ha.RolePassive && applicationReady(report.Runtime, public, targetVersion) && - rollingUpdateControlReady(report.Control), nil + controlReady, nil } func rollingUpdateControlReady(control *ControlStatus) bool { @@ -314,9 +319,9 @@ func ExpectedRollingVersionMismatch(control *ControlStatus) bool { len(control.ReasonCodes) == 1 && control.ReasonCodes[0] == ReasonFleetVersionMismatch } -func updatedApplicationReady(report StatusReport, publicStatus fleetHostStatus, targetVersion string, requirePassive bool) (bool, error) { +func updatedApplicationReady(report StatusReport, publicStatus fleetHostStatus, targetVersion string, requirePassive, requireFailoverReady bool) (bool, error) { if requirePassive { - return rollingUpdateApplicationReady(report, publicStatus, targetVersion) + return rollingUpdateApplicationReady(report, publicStatus, targetVersion, requireFailoverReady) } if report.Control == nil || !report.Control.ControlReady { return false, nil diff --git a/server/internal/ha/deployment/update_test.go b/server/internal/ha/deployment/update_test.go index fae64757c8..8554f84eb2 100644 --- a/server/internal/ha/deployment/update_test.go +++ b/server/internal/ha/deployment/update_test.go @@ -40,7 +40,7 @@ func TestRollingUpdateApplicationRejectsActiveTakeover(t *testing.T) { public := fleetHostStatus{reachable: true, active: true, version: "v1.1.0"} // Act - ready, err := rollingUpdateApplicationReady(report, public, "v1.1.0") + ready, err := rollingUpdateApplicationReady(report, public, "v1.1.0", false) // Assert require.False(t, ready) @@ -78,13 +78,29 @@ func TestRecoveryAcceptsHealthyActiveApplication(t *testing.T) { public := fleetHostStatus{reachable: true, active: true, version: "v1.1.0"} // Act - ready, err := updatedApplicationReady(report, public, "v1.1.0", false) + ready, err := updatedApplicationReady(report, public, "v1.1.0", false, false) // Assert require.NoError(t, err) require.True(t, ready) } +func TestCompletedUpdateRequiresFullFailoverReadiness(t *testing.T) { + // Arrange + report := StatusReport{ + Runtime: ha.Status{Version: "v1.1.0", Role: ha.RolePassive, Observation: ha.ObservationCurrent}, + Control: &ControlStatus{ControlReady: true, ReasonCodes: []ControlReasonCode{ReasonFleetVersionMismatch}}, + } + public := fleetHostStatus{reachable: true, passive: true, version: "v1.1.0"} + + // Act + ready, err := rollingUpdateApplicationReady(report, public, "v1.1.0", true) + + // Assert + require.NoError(t, err) + require.False(t, ready) +} + func TestRollingUpdateControlAllowsOnlyExpectedVersionMismatch(t *testing.T) { for _, test := range []struct { name string diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index ea8d3f3c7c..353020779c 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -1378,7 +1378,7 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string, co return } - if err := m.runActivation(activationCtx, currentDeployment, targetVersion, commandOutput); err != nil { + if err := m.runActivation(activationCtx, currentDeployment, targetVersion, complete, commandOutput); err != nil { activationErr := fmt.Errorf("new stack failed to start: %w", err) // Migrations may already have run, so keep the new deployment active for // forward recovery instead of starting an older binary against its schema. @@ -1448,9 +1448,13 @@ func (m *Manager) runPreflight(ctx context.Context, deployment string, output io return m.runCommand(ctx, m.cfg.PreflightTimeout, deployment, output, "/bin/bash", "./run-fleet.sh", "--non-interactive", "--preflight-only") } -func (m *Manager) runActivation(ctx context.Context, deployment, targetVersion string, output io.Writer) error { +func (m *Manager) runActivation(ctx context.Context, deployment, targetVersion string, complete bool, output io.Writer) error { if m.cfg.DeploymentMode == DeploymentModeHA { - return m.runHACommand(ctx, m.cfg.ActivationTimeout, deployment, output, "app-start", targetVersion, "passive") + mode := "passive" + if complete { + mode = "complete" + } + return m.runHACommand(ctx, m.cfg.ActivationTimeout, deployment, output, "app-start", targetVersion, mode) } return m.runCommand(ctx, m.cfg.ActivationTimeout, deployment, output, "/bin/bash", "./run-fleet.sh", "--non-interactive", "--skip-build") } diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index a65032a1cb..a3117c7a3f 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -444,7 +444,7 @@ func TestManagerHACompletionWaitsForUpdatedPeerBeforeSwap(t *testing.T) { assert.Equal(t, []string{"require-active", "/etc/proto-fleet/ha/node.env", "v1.1.0"}, commands[1].Args) assert.Equal(t, []string{"app-stop", "active"}, commands[2].Args) assert.Equal(t, []string{"wait-takeover", "v1.1.0"}, commands[3].Args) - assert.Equal(t, []string{"app-start", "v1.1.0", "passive"}, commands[4].Args) + assert.Equal(t, []string{"app-start", "v1.1.0", "complete"}, commands[4].Args) _, err = manager.TriggerWithID("v1.1.0", "11111111-1111-4111-8111-111111111111") require.ErrorContains(t, err, "operation id is already associated with another update") From 2b0bddecbabfc14e5ed5610c1811c6925ace3d66 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 02:32:48 -0700 Subject: [PATCH 22/40] Start takeover timeout after shutdown --- server/internal/updater/manager.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 353020779c..e970bfc306 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -1340,20 +1340,19 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string, co m.failActivation(operationID, targetVersion, fmt.Errorf("persist passive application recovery: %w", err), logFile, false) return } - haActivationCtx := activationCtx stopTimeout := m.cfg.ActivationTimeout - cancelOutage := func() {} if complete { - haActivationCtx, cancelOutage = context.WithTimeout(activationCtx, ha.UpdateTakeoverTimeout) stopTimeout = ha.UpdateActiveStopTimeout } - defer cancelOutage() - if err := m.runHACommand(haActivationCtx, stopTimeout, currentDeployment, commandOutput, "app-stop", requiredRole); err != nil { + if err := m.runHACommand(activationCtx, stopTimeout, currentDeployment, commandOutput, "app-stop", requiredRole); err != nil { m.failActivation(operationID, targetVersion, fmt.Errorf("stop HA application: %w", err), logFile, true) return } if complete { - if err := m.runHACommand(haActivationCtx, ha.UpdateTakeoverTimeout, currentDeployment, commandOutput, "wait-takeover", targetVersion); err != nil { + takeoverCtx, cancelTakeover := context.WithTimeout(activationCtx, ha.UpdateTakeoverTimeout) + err := m.runHACommand(takeoverCtx, ha.UpdateTakeoverTimeout, currentDeployment, commandOutput, "wait-takeover", targetVersion) + cancelTakeover() + if err != nil { restartErr := m.restartHAApplication(ctx, currentDeployment, previousVersion, commandOutput) if restartErr == nil { m.fail(operationID, fmt.Errorf("updated peer did not take over; previous release restarted: %w", err), "") From 921e664d0c5e1cc6d959c4596c52330c1c6e7ab7 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 02:41:17 -0700 Subject: [PATCH 23/40] Require complete update readiness --- server/cmd/fleet-ha/main.go | 2 +- server/cmd/fleet-ha/main_test.go | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/server/cmd/fleet-ha/main.go b/server/cmd/fleet-ha/main.go index d228e4962c..d89f101268 100644 --- a/server/cmd/fleet-ha/main.go +++ b/server/cmd/fleet-ha/main.go @@ -277,7 +277,7 @@ func runPassiveUpdate( if report.Control != nil && report.Control.FailoverReady { return nil } - if deployment.ExpectedRollingVersionMismatch(report.Control) { + if !complete && deployment.ExpectedRollingVersionMismatch(report.Control) { _, err = fmt.Fprintln(output, "Update succeeded; failover readiness will recover after the peer is updated.") if err != nil { return fmt.Errorf("write update outcome: %w", err) diff --git a/server/cmd/fleet-ha/main_test.go b/server/cmd/fleet-ha/main_test.go index 7d40e6f328..babc900e36 100644 --- a/server/cmd/fleet-ha/main_test.go +++ b/server/cmd/fleet-ha/main_test.go @@ -154,6 +154,23 @@ func TestPassiveUpdateAllowsExpectedVersionMismatch(t *testing.T) { require.NoError(t, err) } +func TestCompleteUpdateRejectsExpectedVersionMismatch(t *testing.T) { + // Arrange + client := &fakeUpdaterClient{} + read := func(context.Context, string, bool) (deployment.StatusReport, error) { + return deployment.StatusReport{Control: &deployment.ControlStatus{ + ControlReady: true, + ReasonCodes: []deployment.ControlReasonCode{deployment.ReasonFleetVersionMismatch}, + }}, nil + } + + // Act + err := runPassiveUpdate(t.Context(), []string{"v1.2.3", "--complete"}, &bytes.Buffer{}, func(context.Context, string, string, bool) error { return nil }, client, read) + + // Assert + require.ErrorContains(t, err, "failover readiness is degraded") +} + func TestUpdateReturnsWhenUpdaterIsUnavailable(t *testing.T) { // Arrange client := &fakeUpdaterClient{triggerErr: updaterapi.ErrUnavailable} From 1dc51060be389728d610c3b2128ae1c0a3eec973 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 02:53:55 -0700 Subject: [PATCH 24/40] Lock updater startup repair --- server/internal/updater/manager.go | 14 +++++++++++++- server/internal/updater/manager_test.go | 20 +++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index e970bfc306..3fd086cbf7 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -575,9 +575,21 @@ func NewManager(cfg Config) (*Manager, error) { // RepairStartup restores a crash-interrupted deployment layout without starting Fleet. func RepairStartup(cfg Config) error { - if _, err := PrepareSelfUpdateStartup(cfg.SelfUpdatePath, ""); err != nil { + canonicalStateDir, err := ensureUpdaterStateDirectory(cfg.StateDir) + if err != nil { + return err + } + processLock, err := acquireProcessLock(canonicalStateDir) + if err != nil { return err } + if _, err := PrepareSelfUpdateStartup(cfg.SelfUpdatePath, ""); err != nil { + return errors.Join(err, processLock.Close()) + } + if err := processLock.Close(); err != nil { + return fmt.Errorf("release updater repair lock: %w", err) + } + cfg.StateDir = canonicalStateDir manager, err := newManager(cfg, false) if err != nil { return err diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index a3117c7a3f..2b5b8bfb8f 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -1941,7 +1941,25 @@ func TestRepairStartupRestoresInterruptedSelfUpdateBeforeState(t *testing.T) { // Assert require.ErrorIs(t, err, ErrInterruptedSelfUpdateRestored) assert.Equal(t, "old updater", mustReadFile(t, destination)) - assert.NoDirExists(t, stateDir) + assert.DirExists(t, stateDir) +} + +func TestRepairStartupDoesNotTouchSelfUpdateWhileManagerRuns(t *testing.T) { + // Arrange + installRoot := t.TempDir() + writeCurrentDeployment(t, installRoot, "v1.0.0") + destination := installSelfUpdateForHandoffTest(t) + stateDir := filepath.Join(t.TempDir(), "state") + manager, err := NewManager(Config{InstallRoot: installRoot, StateDir: stateDir}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, manager.Close()) }) + + // Act + err = RepairStartup(Config{InstallRoot: installRoot, StateDir: stateDir, SelfUpdatePath: destination}) + + // Assert + require.ErrorContains(t, err, "another updater process is already running") + assert.Equal(t, "new updater", mustReadFile(t, destination)) } func TestManagerFailsStartupWhenHARecoveryFails(t *testing.T) { From a59fe19a359a7078b55c08b757b1d019875df7dc Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 03:23:29 -0700 Subject: [PATCH 25/40] Preserve active update recovery behavior --- server/internal/updater/manager.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 3fd086cbf7..6bd40fe79e 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -1357,6 +1357,18 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string, co stopTimeout = ha.UpdateActiveStopTimeout } if err := m.runHACommand(activationCtx, stopTimeout, currentDeployment, commandOutput, "app-stop", requiredRole); err != nil { + if complete { + restartErr := m.restartHAApplication(ctx, currentDeployment, previousVersion, commandOutput) + if restartErr == nil { + m.fail(operationID, fmt.Errorf("stop HA application failed; previous release restarted: %w", err), "") + return + } + m.fail(operationID, errors.Join( + fmt.Errorf("stop HA application: %w", err), + fmt.Errorf("restart previous release: %w", restartErr), + ), recovery) + return + } m.failActivation(operationID, targetVersion, fmt.Errorf("stop HA application: %w", err), logFile, true) return } From 8c4e3b13c80e840823e934f51f3dd884bc804bab Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 03:53:59 -0700 Subject: [PATCH 26/40] Add deterministic HA update qualification barrier --- server/internal/updater/manager.go | 28 +++++++++++++++++++++++++ server/internal/updater/manager_test.go | 26 +++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 6bd40fe79e..20e8f5c65f 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -62,6 +62,7 @@ const ( processLockFilename = "updater.lock" activationMarkerFilename = "activation-swap.json" activationMarkerTempName = ".activation-swap.json.tmp" + qualificationBarrierName = "qualification-pause-before-ha-stop" preflightProofFilename = ".update-preflight-complete" operationArtifactPrefix = ".proto-fleet-upgrade-" selfUpdateBackupSuffix = ".previous" @@ -1352,6 +1353,12 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string, co m.failActivation(operationID, targetVersion, fmt.Errorf("persist passive application recovery: %w", err), logFile, false) return } + if complete { + if err := m.waitForQualificationBarrier(activationCtx); err != nil { + m.fail(operationID, errors.Join(err, m.clearActivationMarker()), "") + return + } + } stopTimeout := m.cfg.ActivationTimeout if complete { stopTimeout = ha.UpdateActiveStopTimeout @@ -1464,6 +1471,27 @@ func (m *Manager) restartHAApplication( return m.clearActivationMarker() } +// A root-created barrier lets exact release qualification stop the peer after +// final preflight without racing the old application's stop. Normal hosts never +// create this file and take the fast path. +func (m *Manager) waitForQualificationBarrier(ctx context.Context) error { + path := filepath.Join(m.cfg.StateDir, qualificationBarrierName) + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + if _, err := os.Lstat(path); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return fmt.Errorf("inspect HA update qualification barrier: %w", err) + } + select { + case <-ctx.Done(): + return fmt.Errorf("wait for HA update qualification barrier: %w", ctx.Err()) + case <-ticker.C: + } + } +} + func (m *Manager) runPreflight(ctx context.Context, deployment string, output io.Writer) error { if m.cfg.DeploymentMode == DeploymentModeHA { return m.runHACommand(ctx, m.cfg.PreflightTimeout, deployment, output, "update-preflight") diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index 2b5b8bfb8f..97bd6cbcf5 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -523,6 +523,32 @@ func TestManagerHACompletionRestartsOldReleaseWhenStopBlocks(t *testing.T) { assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[len(commands)-1].Args) } +func TestHAQualificationBarrierPausesBeforeStop(t *testing.T) { + // Arrange + stateDir := t.TempDir() + barrier := filepath.Join(stateDir, qualificationBarrierName) + require.NoError(t, os.WriteFile(barrier, nil, 0o600)) + manager := &Manager{cfg: Config{StateDir: stateDir}} + done := make(chan error, 1) + go func() { done <- manager.waitForQualificationBarrier(t.Context()) }() + + // Act + select { + case err := <-done: + t.Fatalf("barrier returned before release: %v", err) + case <-time.After(50 * time.Millisecond): + } + require.NoError(t, os.Remove(barrier)) + + // Assert + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("barrier did not release") + } +} + func TestManagerHACompletionInterruptedAfterSwapStartsTargetRelease(t *testing.T) { // Arrange installRoot := t.TempDir() From a924d2ca271502c1b5a18cd205dc3f025ed988c2 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 04:17:49 -0700 Subject: [PATCH 27/40] Reuse VIP probe connections during takeover --- server/internal/ha/deployment/update.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index ddd1c3dc65..0a6b4de93e 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "net/http" "os" "path/filepath" @@ -360,14 +361,17 @@ func WaitForVIPVersion(ctx context.Context, envPath, targetVersion string) error } response, requestErr := client.Do(request) if requestErr == nil { - response.Body.Close() + _, drainErr := io.Copy(io.Discard, response.Body) + _ = response.Body.Close() version := response.Header.Get("X-Proto-Fleet-Version") - ready, versionErr := acceptVIPVersion(response.StatusCode, version, targetVersion) - if versionErr != nil { - return versionErr - } - if ready { - return nil + if drainErr == nil { + ready, versionErr := acceptVIPVersion(response.StatusCode, version, targetVersion) + if versionErr != nil { + return versionErr + } + if ready { + return nil + } } } select { From 6b7415dc9a81b33e055ecd2b488069c520c87201 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 04:36:55 -0700 Subject: [PATCH 28/40] Keep HA role probes outside stop deadline --- server/internal/ha/deployment/update.go | 8 +++++--- server/internal/updater/manager.go | 6 +----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index 0a6b4de93e..d116b8df14 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -208,9 +208,11 @@ func StopApplication(ctx context.Context, root string, expectedRole ha.RuntimeRo } // The crash-only design intentionally has no maintenance lease. If the role // changes after this final proof, normal update recovery restarts Fleet. - // The HA workload is crash-only. Avoid per-service graceful-stop timers so - // the updater's five-second deadline remains a hard total shutdown bound. - if err := RunCompose(ctx, fleetComposeArgsAt(root, "kill", "fleet-api", "fleet-client")); err != nil { + // Role validation runs while Fleet still serves. Bound only the crash-only + // kill so slow control probes cannot consume the interruption budget. + stopCtx, cancel := context.WithTimeout(ctx, ha.UpdateActiveStopTimeout) + defer cancel() + if err := RunCompose(stopCtx, fleetComposeArgsAt(root, "kill", "fleet-api", "fleet-client")); err != nil { return fmt.Errorf("stop HA application: %w", err) } return nil diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 20e8f5c65f..5ead662761 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -1359,11 +1359,7 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string, co return } } - stopTimeout := m.cfg.ActivationTimeout - if complete { - stopTimeout = ha.UpdateActiveStopTimeout - } - if err := m.runHACommand(activationCtx, stopTimeout, currentDeployment, commandOutput, "app-stop", requiredRole); err != nil { + if err := m.runHACommand(activationCtx, m.cfg.ActivationTimeout, currentDeployment, commandOutput, "app-stop", requiredRole); err != nil { if complete { restartErr := m.restartHAApplication(ctx, currentDeployment, previousVersion, commandOutput) if restartErr == nil { From 3faadf1481e0b560cf18cd8622d08d268d6e3925 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 04:56:38 -0700 Subject: [PATCH 29/40] Integrate updater handoff recovery with startup repair --- server/internal/updater/manager.go | 25 +++++++++++-------------- server/internal/updater/manager_test.go | 16 ---------------- 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 5ead662761..b1e7bd50f9 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -571,10 +571,10 @@ func wrapIfError(message string, err error) error { } func NewManager(cfg Config) (*Manager, error) { - return newManager(cfg, true) + return newManager(cfg) } -// RepairStartup restores a crash-interrupted deployment layout without starting Fleet. +// RepairStartup restores crash-interrupted updater and deployment state before HA starts. func RepairStartup(cfg Config) error { canonicalStateDir, err := ensureUpdaterStateDirectory(cfg.StateDir) if err != nil { @@ -584,21 +584,25 @@ func RepairStartup(cfg Config) error { if err != nil { return err } - if _, err := PrepareSelfUpdateStartup(cfg.SelfUpdatePath, ""); err != nil { - return errors.Join(err, processLock.Close()) + _, prepareErr := PrepareSelfUpdateStartup(cfg.SelfUpdatePath, "") + if prepareErr != nil && !errors.Is(prepareErr, ErrInterruptedSelfUpdateRestored) { + return errors.Join(prepareErr, processLock.Close()) } if err := processLock.Close(); err != nil { return fmt.Errorf("release updater repair lock: %w", err) } cfg.StateDir = canonicalStateDir - manager, err := newManager(cfg, false) + manager, err := newManager(cfg) if err != nil { return err } - return manager.Close() + if prepareErr != nil { + err = manager.restoreUpdaterFromInstalledDeployment() + } + return errors.Join(err, manager.Close()) } -func newManager(cfg Config, recoverApplication bool) (*Manager, error) { +func newManager(cfg Config) (*Manager, error) { if !filepath.IsAbs(cfg.InstallRoot) { return nil, fmt.Errorf("install root must be absolute") } @@ -725,13 +729,6 @@ func newManager(cfg Config, recoverApplication bool) (*Manager, error) { _ = logRoot.Close() return nil, err } - if recoverApplication { - if err := m.recoverHAApplication(); err != nil { - _ = processLock.Close() - _ = logRoot.Close() - return nil, err - } - } protectedLogName := "" if m.operation != nil { protectedLogName = operationLogFilename(m.operation.ID) diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index 97bd6cbcf5..cd275f32be 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -1954,22 +1954,6 @@ func TestRepairStartupRestoresLayoutWithoutStartingHAApplication(t *testing.T) { assert.True(t, operation.RecoveryPending) } -func TestRepairStartupRestoresInterruptedSelfUpdateBeforeState(t *testing.T) { - // Arrange - destination := installSelfUpdateForHandoffTest(t) - stateDir := filepath.Join(t.TempDir(), "state") - - // Act - err := RepairStartup(Config{ - InstallRoot: t.TempDir(), StateDir: stateDir, SelfUpdatePath: destination, - }) - - // Assert - require.ErrorIs(t, err, ErrInterruptedSelfUpdateRestored) - assert.Equal(t, "old updater", mustReadFile(t, destination)) - assert.DirExists(t, stateDir) -} - func TestRepairStartupDoesNotTouchSelfUpdateWhileManagerRuns(t *testing.T) { // Arrange installRoot := t.TempDir() From 4ff24c008cdd0b8c7b7f632625ee60b7d9bb7b1f Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 05:09:05 -0700 Subject: [PATCH 30/40] Make HA update recovery retry-safe --- server/internal/ha/deployment/update_test.go | 21 ++++++++++++++++++++ server/internal/updater/manager.go | 5 +++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/server/internal/ha/deployment/update_test.go b/server/internal/ha/deployment/update_test.go index 8554f84eb2..b0e9b23d86 100644 --- a/server/internal/ha/deployment/update_test.go +++ b/server/internal/ha/deployment/update_test.go @@ -85,6 +85,27 @@ func TestRecoveryAcceptsHealthyActiveApplication(t *testing.T) { require.True(t, ready) } +func TestApplicationConvergenceRequiresExpectedVersionAndRole(t *testing.T) { + for _, test := range []struct { + name string + runtime ha.Status + requirePassive bool + want bool + }{ + {name: "active recovery", runtime: ha.Status{Version: "v1.1.0", Role: ha.RoleActive, Observation: ha.ObservationCurrent}, want: true}, + {name: "active passive-update", runtime: ha.Status{Version: "v1.1.0", Role: ha.RoleActive, Observation: ha.ObservationCurrent}, requirePassive: true}, + {name: "wrong version", runtime: ha.Status{Version: "v1.0.0", Role: ha.RolePassive, Observation: ha.ObservationCurrent}}, + } { + t.Run(test.name, func(t *testing.T) { + // Act + got := applicationMayConverge(test.runtime, "v1.1.0", test.requirePassive) + + // Assert + require.Equal(t, test.want, got) + }) + } +} + func TestCompletedUpdateRequiresFullFailoverReadiness(t *testing.T) { // Arrange report := StatusReport{ diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index b1e7bd50f9..7b01135720 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -585,7 +585,8 @@ func RepairStartup(cfg Config) error { return err } _, prepareErr := PrepareSelfUpdateStartup(cfg.SelfUpdatePath, "") - if prepareErr != nil && !errors.Is(prepareErr, ErrInterruptedSelfUpdateRestored) { + restoreUpdater := errors.Is(prepareErr, ErrInterruptedSelfUpdateRestored) + if prepareErr != nil && !restoreUpdater && !errors.Is(prepareErr, errRetriedSelfUpdateRestored) { return errors.Join(prepareErr, processLock.Close()) } if err := processLock.Close(); err != nil { @@ -596,7 +597,7 @@ func RepairStartup(cfg Config) error { if err != nil { return err } - if prepareErr != nil { + if restoreUpdater { err = manager.restoreUpdaterFromInstalledDeployment() } return errors.Join(err, manager.Close()) From fa4b3c61e69b233adda504e4590803a2196ea97b Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 05:17:08 -0700 Subject: [PATCH 31/40] Align HA recovery tests with startup repair --- server/internal/updater/manager_test.go | 82 +++---------------------- 1 file changed, 9 insertions(+), 73 deletions(-) diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index cd275f32be..1ca3e1f1ce 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -402,15 +402,14 @@ func TestManagerHAInterruptedAfterStopRestartsCurrentApplication(t *testing.T) { // Act runner := &haRecordingRunner{fail: make(map[string]error)} - manager, err := NewManager(Config{ + err := RepairStartup(Config{ InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", DeploymentMode: DeploymentModeHA, Runner: runner, }) require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, manager.Close()) }) // Assert - operation := manager.Status().Operation - require.NotNil(t, operation) + var operation updaterapi.Operation + require.NoError(t, json.Unmarshal([]byte(mustReadFile(t, filepath.Join(stateDir, stateFilename))), &operation)) require.Equal(t, updaterapi.PhaseFailed, operation.Phase) assert.Empty(t, operation.RecoveryCommand) assert.Contains(t, operation.Message, "HA application restarted") @@ -561,15 +560,14 @@ func TestManagerHACompletionInterruptedAfterSwapStartsTargetRelease(t *testing.T runner := &haRecordingRunner{fail: make(map[string]error)} // Act - manager, err := NewManager(Config{ + err := RepairStartup(Config{ InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", DeploymentMode: DeploymentModeHA, Runner: runner, }) require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, manager.Close()) }) // Assert - operation := manager.Status().Operation - require.NotNil(t, operation) + var operation updaterapi.Operation + require.NoError(t, json.Unmarshal([]byte(mustReadFile(t, filepath.Join(stateDir, stateFilename))), &operation)) require.Equal(t, updaterapi.PhaseFailed, operation.Phase) require.Empty(t, operation.RecoveryCommand) commands := runner.Commands() @@ -1894,66 +1892,6 @@ func TestRepairStartupRestoresUpdaterFromInstalledDeployment(t *testing.T) { assert.NoFileExists(t, installedUpdater+selfUpdateHandoffSuffix) } -func TestManagerRestartsHAApplicationAfterInterruptedSwap(t *testing.T) { - t.Parallel() - - // Arrange - installRoot := t.TempDir() - writeCurrentDeployment(t, installRoot, "v1.0.0") - require.NoError(t, os.Rename( - filepath.Join(installRoot, "deployment"), - filepath.Join(installRoot, "deployment.previous"), - )) - stateDir := filepath.Join(t.TempDir(), "state") - writeInterruptedOperationState(t, stateDir, "v1.1.0") - runner := &haRecordingRunner{} - - // Act - manager, err := NewManager(Config{ - InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", - DeploymentMode: DeploymentModeHA, Runner: runner, - }) - require.NoError(t, err) - t.Cleanup(func() { assert.NoError(t, manager.Close()) }) - - // Assert - commands := runner.Commands() - require.Len(t, commands, 1) - assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[0].Args) - assert.Empty(t, manager.Status().Operation.RecoveryCommand) - assert.False(t, manager.Status().Operation.RecoveryPending) -} - -func TestRepairStartupRestoresLayoutWithoutStartingHAApplication(t *testing.T) { - t.Parallel() - - // Arrange - installRoot := t.TempDir() - writeCurrentDeployment(t, installRoot, "v1.0.0") - require.NoError(t, os.Rename( - filepath.Join(installRoot, "deployment"), - filepath.Join(installRoot, "deployment.previous"), - )) - stateDir := filepath.Join(t.TempDir(), "state") - writeInterruptedOperationState(t, stateDir, "v1.1.0") - runner := &haRecordingRunner{} - - // Act - err := RepairStartup(Config{ - InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", - DeploymentMode: DeploymentModeHA, Runner: runner, - }) - - // Assert - require.NoError(t, err) - assert.Empty(t, runner.Commands()) - assert.Equal(t, "v1.0.0", mustReadVersion(t, filepath.Join(installRoot, "deployment", "version.txt"))) - assert.NoDirExists(t, filepath.Join(installRoot, "deployment.previous")) - var operation updaterapi.Operation - require.NoError(t, json.Unmarshal([]byte(mustReadFile(t, filepath.Join(stateDir, stateFilename))), &operation)) - assert.True(t, operation.RecoveryPending) -} - func TestRepairStartupDoesNotTouchSelfUpdateWhileManagerRuns(t *testing.T) { // Arrange installRoot := t.TempDir() @@ -1972,7 +1910,7 @@ func TestRepairStartupDoesNotTouchSelfUpdateWhileManagerRuns(t *testing.T) { assert.Equal(t, "new updater", mustReadFile(t, destination)) } -func TestManagerFailsStartupWhenHARecoveryFails(t *testing.T) { +func TestRepairStartupFailsWhenHARecoveryFails(t *testing.T) { // Arrange installRoot := t.TempDir() writeCurrentDeployment(t, installRoot, "v1.0.0") @@ -1984,13 +1922,12 @@ func TestManagerFailsStartupWhenHARecoveryFails(t *testing.T) { writeInterruptedOperationState(t, stateDir, "v1.1.0") // Act - manager, err := NewManager(Config{ + err := RepairStartup(Config{ InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", DeploymentMode: DeploymentModeHA, Runner: &haRecordingRunner{fail: map[string]error{"app-start": errors.New("restart failed")}}, }) require.ErrorContains(t, err, "restart interrupted HA application") - require.Nil(t, manager) // Assert var operation updaterapi.Operation @@ -2017,12 +1954,11 @@ func TestManagerDoesNotReplayTerminalHARecovery(t *testing.T) { runner := &haRecordingRunner{} // Act - manager, err := NewManager(Config{ + err = RepairStartup(Config{ InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", DeploymentMode: DeploymentModeHA, Runner: runner, }) require.NoError(t, err) - t.Cleanup(func() { assert.NoError(t, manager.Close()) }) // Assert assert.Empty(t, runner.Commands()) From 324bae98aaf143927449e0cf0242cdc8fc8d41f2 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 05:22:38 -0700 Subject: [PATCH 32/40] Keep startup repair ownership singular --- server/internal/updater/manager.go | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 7b01135720..edd1b0a3cd 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -584,7 +584,7 @@ func RepairStartup(cfg Config) error { if err != nil { return err } - _, prepareErr := PrepareSelfUpdateStartup(cfg.SelfUpdatePath, "") + prepareErr := prepareSelfUpdateRepair(cfg.SelfUpdatePath) restoreUpdater := errors.Is(prepareErr, ErrInterruptedSelfUpdateRestored) if prepareErr != nil && !restoreUpdater && !errors.Is(prepareErr, errRetriedSelfUpdateRestored) { return errors.Join(prepareErr, processLock.Close()) @@ -742,23 +742,6 @@ func newManager(cfg Config) (*Manager, error) { return m, nil } -// RepairStartup restores crash-interrupted updater and deployment state before HA starts. -func RepairStartup(cfg Config) error { - prepareErr := prepareSelfUpdateRepair(cfg.SelfUpdatePath) - restoreUpdater := errors.Is(prepareErr, ErrInterruptedSelfUpdateRestored) - if prepareErr != nil && !restoreUpdater && !errors.Is(prepareErr, errRetriedSelfUpdateRestored) { - return prepareErr - } - manager, err := NewManager(cfg) - if err != nil { - return err - } - if restoreUpdater { - err = manager.restoreUpdaterFromInstalledDeployment() - } - return errors.Join(err, manager.Close()) -} - func (m *Manager) restoreUpdaterFromInstalledDeployment() error { targetVersion, err := readInstalledVersion(filepath.Join(m.cfg.InstallRoot, "deployment", "version.txt")) if err != nil { From 503db0f463b6cef6cc4ca93ebb05e60e2d62bb28 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 05:37:22 -0700 Subject: [PATCH 33/40] Test recovery after HA substrate startup --- server/internal/updater/manager_test.go | 42 ++++++++++++++++++------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index 1ca3e1f1ce..754c1f8d72 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -399,12 +399,18 @@ func TestManagerHAInterruptedAfterStopRestartsCurrentApplication(t *testing.T) { writeCurrentDeployment(t, installRoot, "v1.0.0") require.NoError(t, os.Rename(filepath.Join(installRoot, "deployment"), filepath.Join(installRoot, "deployment.previous"))) writeInterruptedOperationState(t, stateDir, "v1.1.0") - - // Act runner := &haRecordingRunner{fail: make(map[string]error)} - err := RepairStartup(Config{ + cfg := Config{ InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", DeploymentMode: DeploymentModeHA, Runner: runner, - }) + } + + // Act + err := RepairStartup(cfg) + require.NoError(t, err) + manager, err := NewManager(cfg) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, manager.Close()) }) + err = manager.RecoverApplication() require.NoError(t, err) // Assert @@ -558,11 +564,17 @@ func TestManagerHACompletionInterruptedAfterSwapStartsTargetRelease(t *testing.T require.NoError(t, os.WriteFile(previousVersionPath, []byte("version: v1.0.0\n"), 0o600)) writeInterruptedOperationState(t, stateDir, "v1.1.0") runner := &haRecordingRunner{fail: make(map[string]error)} + cfg := Config{ + InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", DeploymentMode: DeploymentModeHA, Runner: runner, + } // Act - err := RepairStartup(Config{ - InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", DeploymentMode: DeploymentModeHA, Runner: runner, - }) + err := RepairStartup(cfg) + require.NoError(t, err) + manager, err := NewManager(cfg) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, manager.Close()) }) + err = manager.RecoverApplication() require.NoError(t, err) // Assert @@ -1910,7 +1922,7 @@ func TestRepairStartupDoesNotTouchSelfUpdateWhileManagerRuns(t *testing.T) { assert.Equal(t, "new updater", mustReadFile(t, destination)) } -func TestRepairStartupFailsWhenHARecoveryFails(t *testing.T) { +func TestRecoverApplicationKeepsPendingRecoveryAfterFailure(t *testing.T) { // Arrange installRoot := t.TempDir() writeCurrentDeployment(t, installRoot, "v1.0.0") @@ -1920,13 +1932,19 @@ func TestRepairStartupFailsWhenHARecoveryFails(t *testing.T) { )) stateDir := filepath.Join(t.TempDir(), "state") writeInterruptedOperationState(t, stateDir, "v1.1.0") - - // Act - err := RepairStartup(Config{ + cfg := Config{ InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", DeploymentMode: DeploymentModeHA, Runner: &haRecordingRunner{fail: map[string]error{"app-start": errors.New("restart failed")}}, - }) + } + + // Act + err := RepairStartup(cfg) + require.NoError(t, err) + manager, err := NewManager(cfg) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, manager.Close()) }) + err = manager.RecoverApplication() require.ErrorContains(t, err, "restart interrupted HA application") // Assert From 5aa3c841226641295523ed2c0ce171658eaff115 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 05:47:12 -0700 Subject: [PATCH 34/40] Retry failed HA rollback after restart --- server/internal/updater/manager.go | 14 +++++++-- server/internal/updater/manager_test.go | 38 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index edd1b0a3cd..79aad2d5af 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -1347,7 +1347,7 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string, co m.fail(operationID, fmt.Errorf("stop HA application failed; previous release restarted: %w", err), "") return } - m.fail(operationID, errors.Join( + m.failPendingRecovery(operationID, errors.Join( fmt.Errorf("stop HA application: %w", err), fmt.Errorf("restart previous release: %w", restartErr), ), recovery) @@ -1366,7 +1366,7 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string, co m.fail(operationID, fmt.Errorf("updated peer did not take over; previous release restarted: %w", err), "") return } - m.fail(operationID, errors.Join(err, restartErr), m.activationRecoveryCommand(currentDeployment, previousVersion)) + m.failPendingRecovery(operationID, errors.Join(err, restartErr), m.activationRecoveryCommand(currentDeployment, previousVersion)) return } } @@ -1988,6 +1988,14 @@ func (m *Manager) setRecoveryCommand(id, recovery string) error { } func (m *Manager) fail(id string, err error, recovery string) { + m.finishFailure(id, err, recovery, false) +} + +func (m *Manager) failPendingRecovery(id string, err error, recovery string) { + m.finishFailure(id, err, recovery, true) +} + +func (m *Manager) finishFailure(id string, err error, recovery string, recoveryPending bool) { m.mu.Lock() defer m.mu.Unlock() if m.operation == nil || m.operation.ID != id { @@ -1999,7 +2007,7 @@ func (m *Manager) fail(id string, err error, recovery string) { m.operation.Message = "Upgrade failed" m.operation.Error = err.Error() m.operation.RecoveryCommand = recovery - m.operation.RecoveryPending = false + m.operation.RecoveryPending = recoveryPending m.operation.UpdatedAt = now m.operation.CompletedAt = &now if persistErr := m.persistLocked(); persistErr != nil { diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index 754c1f8d72..ab6653299d 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -504,6 +504,44 @@ func TestManagerHACompletionRestartsOldReleaseWhenStopFails(t *testing.T) { assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[len(commands)-1].Args) } +func TestManagerHACompletionRetriesFailedRollbackAfterRestart(t *testing.T) { + for _, failedCommand := range []string{"app-stop", "wait-takeover"} { + t.Run(failedCommand, func(t *testing.T) { + // Arrange + installRoot := t.TempDir() + writeCurrentDeployment(t, installRoot, "v1.0.0") + bundle := releaseBundle(t, "v1.1.0") + server := releaseServer(t, "v1.1.0", "amd64", bundle, "") + runner := &haRecordingRunner{fail: map[string]error{ + failedCommand: assert.AnError, + "app-start": errors.New("restart failed"), + }} + manager := newTestManagerWithConfig(t, installRoot, server, runner, func(cfg *Config) { + cfg.DeploymentMode = DeploymentModeHA + }) + + // Act + _, err := manager.TriggerCompleteWithID("v1.1.0", "11111111-1111-4111-8111-111111111111") + require.NoError(t, err) + failed := waitForTerminal(t, manager) + require.NoError(t, manager.Close()) + restarted, err := NewManager(manager.cfg) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, restarted.Close()) }) + err = restarted.RecoverApplication() + + // Assert + require.NoError(t, err) + assert.True(t, failed.RecoveryPending) + assert.NotEmpty(t, failed.RecoveryCommand) + recovered := restarted.Status().Operation + require.NotNil(t, recovered) + assert.False(t, recovered.RecoveryPending) + assert.Empty(t, recovered.RecoveryCommand) + }) + } +} + func TestManagerHACompletionRestartsOldReleaseWhenStopBlocks(t *testing.T) { // Arrange installRoot := t.TempDir() From 2d6328ad135ab234d0dc167e02dfe015ec585fe6 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 06:09:16 -0700 Subject: [PATCH 35/40] Retry failed activation recovery on startup --- server/internal/updater/manager.go | 6 ++++++ server/internal/updater/manager_test.go | 22 +++++++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 79aad2d5af..1ff7cbed1c 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -1606,6 +1606,7 @@ func (m *Manager) failActivation( logOutput io.Writer, restartHA bool, ) { + recoveryPending := false layout := &updaterapi.Operation{ TargetVersion: targetVersion, Phase: updaterapi.PhaseActivating, @@ -1632,10 +1633,15 @@ func (m *Manager) failActivation( } if err != nil { activationErr = errors.Join(activationErr, fmt.Errorf("restart HA application after failed activation: %w", err)) + recoveryPending = layout.RecoveryCommand != "" } else { layout.RecoveryCommand = "" } } + if recoveryPending { + m.failPendingRecovery(operationID, activationErr, layout.RecoveryCommand) + return + } m.fail(operationID, activationErr, layout.RecoveryCommand) } diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index ab6653299d..405ac42aae 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -1280,13 +1280,13 @@ func TestManagerActivationFailurePersistsProofAwareForwardRecovery(t *testing.T) } } -func TestManagerHAFailedActivationRestartsRestoredDeployment(t *testing.T) { +func TestManagerHAFailedActivationRetriesRestartAfterDaemonRecovery(t *testing.T) { t.Parallel() installRoot := t.TempDir() writeCurrentDeployment(t, installRoot, "v1.0.0") stateDir := filepath.Join(t.TempDir(), "state") - runner := &haRecordingRunner{fail: make(map[string]error)} + runner := &haRecordingRunner{fail: map[string]error{"app-start": errors.New("restart failed")}} manager, err := NewManager(Config{ InstallRoot: installRoot, StateDir: stateDir, @@ -1323,16 +1323,28 @@ func TestManagerHAFailedActivationRestartsRestoredDeployment(t *testing.T) { require.NotNil(t, operation) assert.Equal(t, updaterapi.PhaseFailed, operation.Phase) assert.Contains(t, operation.Error, assert.AnError.Error()) - assert.Empty(t, operation.RecoveryCommand) + assert.Contains(t, operation.Error, "restart failed") + assert.True(t, operation.RecoveryPending) + assert.NotEmpty(t, operation.RecoveryCommand) assert.Equal(t, "v1.0.0", mustReadVersion(t, filepath.Join(installRoot, "deployment", "version.txt"))) assert.NoDirExists(t, filepath.Join(installRoot, "deployment.previous")) var persisted updaterapi.Operation require.NoError(t, json.Unmarshal([]byte(mustReadFile(t, filepath.Join(stateDir, stateFilename))), &persisted)) assert.Equal(t, updaterapi.PhaseFailed, persisted.Phase) - assert.Empty(t, persisted.RecoveryCommand) + assert.True(t, persisted.RecoveryPending) + require.NoError(t, manager.Close()) + restarted, err := NewManager(manager.cfg) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, restarted.Close()) }) + require.NoError(t, restarted.RecoverApplication()) + recovered := restarted.Status().Operation + require.NotNil(t, recovered) + assert.False(t, recovered.RecoveryPending) + assert.Empty(t, recovered.RecoveryCommand) commands := runner.Commands() - require.Len(t, commands, 1) + require.Len(t, commands, 2) assert.Equal(t, []string{"app-start", "v1.0.0"}, commands[0].Args) + assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[1].Args) } func TestActivationMarkerWriteIsAtomicAndExclusive(t *testing.T) { From 4e00379fce364355659ebc4a79a66d0ba16715e7 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 06:49:28 -0700 Subject: [PATCH 36/40] Keep HA application stopped during takeover --- server/internal/ha/deployment/update.go | 6 +++--- server/internal/updater/manager.go | 7 ++++--- server/internal/updater/manager_test.go | 20 ++++++++++++++++---- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/server/internal/ha/deployment/update.go b/server/internal/ha/deployment/update.go index d116b8df14..dad2dbe5f3 100644 --- a/server/internal/ha/deployment/update.go +++ b/server/internal/ha/deployment/update.go @@ -208,11 +208,11 @@ func StopApplication(ctx context.Context, root string, expectedRole ha.RuntimeRo } // The crash-only design intentionally has no maintenance lease. If the role // changes after this final proof, normal update recovery restarts Fleet. - // Role validation runs while Fleet still serves. Bound only the crash-only - // kill so slow control probes cannot consume the interruption budget. + // Role validation runs while Fleet still serves. Give Compose one second to + // stop cleanly, then let the outer deadline bound forced termination. stopCtx, cancel := context.WithTimeout(ctx, ha.UpdateActiveStopTimeout) defer cancel() - if err := RunCompose(stopCtx, fleetComposeArgsAt(root, "kill", "fleet-api", "fleet-client")); err != nil { + if err := RunCompose(stopCtx, fleetComposeArgsAt(root, "stop", "--timeout", "1", "fleet-api", "fleet-client")); err != nil { return fmt.Errorf("stop HA application: %w", err) } return nil diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 1ff7cbed1c..301962ccb1 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -1606,7 +1606,6 @@ func (m *Manager) failActivation( logOutput io.Writer, restartHA bool, ) { - recoveryPending := false layout := &updaterapi.Operation{ TargetVersion: targetVersion, Phase: updaterapi.PhaseActivating, @@ -1633,12 +1632,11 @@ func (m *Manager) failActivation( } if err != nil { activationErr = errors.Join(activationErr, fmt.Errorf("restart HA application after failed activation: %w", err)) - recoveryPending = layout.RecoveryCommand != "" } else { layout.RecoveryCommand = "" } } - if recoveryPending { + if restartHA && layout.RecoveryCommand != "" { m.failPendingRecovery(operationID, activationErr, layout.RecoveryCommand) return } @@ -2148,6 +2146,9 @@ func (m *Manager) loadState() error { op.CompletedAt = &now } } + if marker != nil && m.cfg.DeploymentMode == DeploymentModeHA && op.RecoveryCommand != "" { + op.RecoveryPending = true + } m.operation = &op return m.persistReconciledState() } diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index 405ac42aae..d65afb0ee3 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -2070,10 +2070,13 @@ func TestManagerReconcilesTerminalFailedActivationBeforeCleaningArtifacts(t *tes require.NoError(t, err) require.NoError(t, os.WriteFile(filepath.Join(stateDir, activationMarkerFilename), marker, 0o600)) + runner := &haRecordingRunner{} manager, err := NewManager(Config{ - InstallRoot: installRoot, - StateDir: stateDir, - GOARCH: "amd64", + InstallRoot: installRoot, + StateDir: stateDir, + GOARCH: "amd64", + Runner: runner, + DeploymentMode: DeploymentModeHA, }) require.NoError(t, err) t.Cleanup(func() { assert.NoError(t, manager.Close()) }) @@ -2085,10 +2088,19 @@ func TestManagerReconcilesTerminalFailedActivationBeforeCleaningArtifacts(t *tes assert.Contains(t, operation.Error, "restored the validated previous deployment") assert.Contains(t, operation.Message, "Previous deployment restored") assert.Equal(t, completed, *operation.CompletedAt) - assert.Empty(t, operation.RecoveryCommand) + assert.NotEmpty(t, operation.RecoveryCommand) + assert.True(t, operation.RecoveryPending) assert.Equal(t, "v1.0.0", mustReadVersion(t, filepath.Join(installRoot, "deployment", "version.txt"))) assert.NoDirExists(t, filepath.Join(installRoot, "deployment.previous")) assert.NoDirExists(t, stageRoot) + require.NoError(t, manager.RecoverApplication()) + recovered := manager.Status().Operation + require.NotNil(t, recovered) + assert.Empty(t, recovered.RecoveryCommand) + assert.False(t, recovered.RecoveryPending) + commands := runner.Commands() + require.Len(t, commands, 1) + assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[0].Args) } func TestManagerDoesNotRestorePreviousWithoutAPendingSwapMarker(t *testing.T) { From 2a09d04ee9d5cb215d428693c7b4cd590beb13b1 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 06:56:20 -0700 Subject: [PATCH 37/40] Persist HA recovery before clearing marker --- server/internal/updater/manager.go | 13 +++++++------ server/internal/updater/manager_test.go | 24 +++++++++++++++++++----- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index 301962ccb1..fabe6268ab 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -2111,11 +2111,6 @@ func (m *Manager) loadState() error { } return err } - if marker != nil { - if err := m.clearActivationMarker(); err != nil { - return err - } - } if !wasTerminal { op.RecoveryPending = m.cfg.DeploymentMode == DeploymentModeHA && op.RecoveryCommand != "" now := m.cfg.Now().UTC() @@ -2150,7 +2145,13 @@ func (m *Manager) loadState() error { op.RecoveryPending = true } m.operation = &op - return m.persistReconciledState() + if err := m.persistReconciledState(); err != nil { + return err + } + if marker != nil { + return m.clearActivationMarker() + } + return nil } // persistReconciledState normally preserves the ordering of reconciliation, diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index d65afb0ee3..1d980abe1f 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -2032,7 +2032,7 @@ func TestManagerDoesNotReplayTerminalHARecovery(t *testing.T) { assert.Empty(t, runner.Commands()) } -func TestManagerReconcilesTerminalFailedActivationBeforeCleaningArtifacts(t *testing.T) { +func TestManagerPersistsTerminalHARecoveryBeforeClearingActivationMarker(t *testing.T) { t.Parallel() installRoot := t.TempDir() @@ -2071,13 +2071,28 @@ func TestManagerReconcilesTerminalFailedActivationBeforeCleaningArtifacts(t *tes require.NoError(t, os.WriteFile(filepath.Join(stateDir, activationMarkerFilename), marker, 0o600)) runner := &haRecordingRunner{} - manager, err := NewManager(Config{ + failPersist := true + config := Config{ InstallRoot: installRoot, StateDir: stateDir, GOARCH: "amd64", Runner: runner, DeploymentMode: DeploymentModeHA, - }) + beforePersistState: func(operation updaterapi.Operation) error { + if operation.RecoveryPending && failPersist { + failPersist = false + return assert.AnError + } + return nil + }, + } + manager, err := NewManager(config) + require.ErrorContains(t, err, assert.AnError.Error()) + require.Nil(t, manager) + assert.FileExists(t, filepath.Join(stateDir, activationMarkerFilename)) + + config.beforePersistState = nil + manager, err = NewManager(config) require.NoError(t, err) t.Cleanup(func() { assert.NoError(t, manager.Close()) }) @@ -2085,13 +2100,12 @@ func TestManagerReconcilesTerminalFailedActivationBeforeCleaningArtifacts(t *tes require.NotNil(t, operation) assert.Equal(t, updaterapi.PhaseFailed, operation.Phase) assert.Contains(t, operation.Error, "input/output error") - assert.Contains(t, operation.Error, "restored the validated previous deployment") - assert.Contains(t, operation.Message, "Previous deployment restored") assert.Equal(t, completed, *operation.CompletedAt) assert.NotEmpty(t, operation.RecoveryCommand) assert.True(t, operation.RecoveryPending) assert.Equal(t, "v1.0.0", mustReadVersion(t, filepath.Join(installRoot, "deployment", "version.txt"))) assert.NoDirExists(t, filepath.Join(installRoot, "deployment.previous")) + assert.NoFileExists(t, filepath.Join(stateDir, activationMarkerFilename)) assert.NoDirExists(t, stageRoot) require.NoError(t, manager.RecoverApplication()) recovered := manager.Status().Operation From 68c371de6ddb29c1dbd15c4639dabd8f955a542d Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Sat, 8 Aug 2026 07:22:42 -0700 Subject: [PATCH 38/40] Make HA update qualification crash windows deterministic --- server/internal/updater/manager.go | 77 ++++++++++++++++--------- server/internal/updater/manager_test.go | 27 ++++++++- 2 files changed, 74 insertions(+), 30 deletions(-) diff --git a/server/internal/updater/manager.go b/server/internal/updater/manager.go index fabe6268ab..2691f0beb5 100644 --- a/server/internal/updater/manager.go +++ b/server/internal/updater/manager.go @@ -49,25 +49,27 @@ const ( // Activation includes migrations and multiple readiness windows after the // old stack is stopped. Timing out requires forward manual recovery, making // this a minimum liveness bound rather than spare retry time. - defaultActivationTimeout = 45 * time.Minute - defaultCleanupTimeout = 2 * time.Minute - defaultCandidateTimeout = 10 * time.Second - maxCommandLogBytes = int64(64 << 20) - maxCandidateVersionBytes = int64(4096) - maxReleaseStateBytes = int64(8 << 20) - maxRetainedOperationLogs = 8 - maxRetainedLogBytes = int64(256 << 20) - canonicalDownloadBaseURL = "https://github.com/block/proto-fleet/releases/download" - canonicalReleaseAPIBase = "https://api.github.com/repos/block/proto-fleet/releases/tags" - processLockFilename = "updater.lock" - activationMarkerFilename = "activation-swap.json" - activationMarkerTempName = ".activation-swap.json.tmp" - qualificationBarrierName = "qualification-pause-before-ha-stop" - preflightProofFilename = ".update-preflight-complete" - operationArtifactPrefix = ".proto-fleet-upgrade-" - selfUpdateBackupSuffix = ".previous" - stateTempPrefix = ".state-" - haNodeEnvPath = "/etc/proto-fleet/ha/node.env" + defaultActivationTimeout = 45 * time.Minute + defaultCleanupTimeout = 2 * time.Minute + defaultCandidateTimeout = 10 * time.Second + maxCommandLogBytes = int64(64 << 20) + maxCandidateVersionBytes = int64(4096) + maxReleaseStateBytes = int64(8 << 20) + maxRetainedOperationLogs = 8 + maxRetainedLogBytes = int64(256 << 20) + canonicalDownloadBaseURL = "https://github.com/block/proto-fleet/releases/download" + canonicalReleaseAPIBase = "https://api.github.com/repos/block/proto-fleet/releases/tags" + processLockFilename = "updater.lock" + activationMarkerFilename = "activation-swap.json" + activationMarkerTempName = ".activation-swap.json.tmp" + qualificationBeforeStopBarrierName = "qualification-pause-before-ha-stop" + qualificationAfterStopBarrierName = "qualification-pause-after-ha-stop" + qualificationBetweenRenamesBarrierName = "qualification-pause-between-deployment-renames" + preflightProofFilename = ".update-preflight-complete" + operationArtifactPrefix = ".proto-fleet-upgrade-" + selfUpdateBackupSuffix = ".previous" + stateTempPrefix = ".state-" + haNodeEnvPath = "/etc/proto-fleet/ha/node.env" ) var ( @@ -1335,7 +1337,7 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string, co return } if complete { - if err := m.waitForQualificationBarrier(activationCtx); err != nil { + if err := m.waitForQualificationBarrier(activationCtx, qualificationBeforeStopBarrierName); err != nil { m.fail(operationID, errors.Join(err, m.clearActivationMarker()), "") return } @@ -1357,6 +1359,15 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string, co return } if complete { + if err := m.waitForQualificationBarrier(activationCtx, qualificationAfterStopBarrierName); err != nil { + restartErr := m.restartHAApplication(ctx, currentDeployment, previousVersion, commandOutput) + if restartErr == nil { + m.fail(operationID, fmt.Errorf("post-stop qualification pause failed; previous release restarted: %w", err), "") + return + } + m.failPendingRecovery(operationID, errors.Join(err, restartErr), recovery) + return + } takeoverCtx, cancelTakeover := context.WithTimeout(activationCtx, ha.UpdateTakeoverTimeout) err := m.runHACommand(takeoverCtx, ha.UpdateTakeoverTimeout, currentDeployment, commandOutput, "wait-takeover", targetVersion) cancelTakeover() @@ -1371,7 +1382,13 @@ func (m *Manager) run(ctx context.Context, operationID, targetVersion string, co } } } - if err := activateDeployment(stageDeployment, currentDeployment, backupDeployment); err != nil { + betweenRenames := func() error { return nil } + if m.cfg.DeploymentMode == DeploymentModeHA { + betweenRenames = func() error { + return m.waitForQualificationBarrier(activationCtx, qualificationBetweenRenamesBarrierName) + } + } + if err := activateDeployment(stageDeployment, currentDeployment, backupDeployment, betweenRenames); err != nil { m.failActivation(operationID, targetVersion, err, logFile, m.cfg.DeploymentMode == DeploymentModeHA) return } @@ -1448,11 +1465,10 @@ func (m *Manager) restartHAApplication( return m.clearActivationMarker() } -// A root-created barrier lets exact release qualification stop the peer after -// final preflight without racing the old application's stop. Normal hosts never -// create this file and take the fast path. -func (m *Manager) waitForQualificationBarrier(ctx context.Context) error { - path := filepath.Join(m.cfg.StateDir, qualificationBarrierName) +// Root-created barriers let exact release qualification pause at otherwise +// unobservable crash windows. Normal hosts never create them and take the fast path. +func (m *Manager) waitForQualificationBarrier(ctx context.Context, name string) error { + path := filepath.Join(m.cfg.StateDir, name) ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { @@ -1648,7 +1664,7 @@ func (m *Manager) failActivation( // portable filesystem operation, so every completed metadata step is fsynced // and every pre-command failure attempts a checked restoration. Startup // reconciliation covers a process or power loss between the renames. -func activateDeployment(staged, current, previous string) error { +func activateDeployment(staged, current, previous string, betweenRenames func() error) error { installRoot := filepath.Dir(current) stageRoot := filepath.Dir(staged) if err := os.RemoveAll(previous); err != nil { @@ -1667,6 +1683,13 @@ func activateDeployment(staged, current, previous string) error { restoreErr, ) } + if err := betweenRenames(); err != nil { + restoreErr := restorePreviousDeployment(current, previous, installRoot) + return errors.Join( + fmt.Errorf("pause between deployment renames: %w", err), + restoreErr, + ) + } if err := os.Rename(staged, current); err != nil { restoreErr := restorePreviousDeployment(current, previous, installRoot) return errors.Join( diff --git a/server/internal/updater/manager_test.go b/server/internal/updater/manager_test.go index 1d980abe1f..88d6f5db41 100644 --- a/server/internal/updater/manager_test.go +++ b/server/internal/updater/manager_test.go @@ -566,14 +566,14 @@ func TestManagerHACompletionRestartsOldReleaseWhenStopBlocks(t *testing.T) { assert.Equal(t, []string{"app-start", "v1.0.0", "any"}, commands[len(commands)-1].Args) } -func TestHAQualificationBarrierPausesBeforeStop(t *testing.T) { +func TestHAQualificationBarrierPausesUntilRemoved(t *testing.T) { // Arrange stateDir := t.TempDir() - barrier := filepath.Join(stateDir, qualificationBarrierName) + barrier := filepath.Join(stateDir, qualificationAfterStopBarrierName) require.NoError(t, os.WriteFile(barrier, nil, 0o600)) manager := &Manager{cfg: Config{StateDir: stateDir}} done := make(chan error, 1) - go func() { done <- manager.waitForQualificationBarrier(t.Context()) }() + go func() { done <- manager.waitForQualificationBarrier(t.Context(), qualificationAfterStopBarrierName) }() // Act select { @@ -592,6 +592,27 @@ func TestHAQualificationBarrierPausesBeforeStop(t *testing.T) { } } +func TestActivateDeploymentRestoresCurrentWhenQualificationPauseFails(t *testing.T) { + // Arrange + installRoot := t.TempDir() + current := filepath.Join(installRoot, "deployment") + staged := filepath.Join(installRoot, "staging", "deployment") + previous := filepath.Join(installRoot, "deployment.previous") + require.NoError(t, os.MkdirAll(current, 0o750)) + require.NoError(t, os.MkdirAll(staged, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(current, "version.txt"), []byte("v1"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(staged, "version.txt"), []byte("v2"), 0o600)) + + // Act + err := activateDeployment(staged, current, previous, func() error { return assert.AnError }) + + // Assert + require.ErrorIs(t, err, assert.AnError) + version, readErr := os.ReadFile(filepath.Join(current, "version.txt")) + require.NoError(t, readErr) + assert.Equal(t, "v1", string(version)) +} + func TestManagerHACompletionInterruptedAfterSwapStartsTargetRelease(t *testing.T) { // Arrange installRoot := t.TempDir() From 82cb32ed60dd64bd1dd544fa3ba2b4e24b2f851d Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Mon, 10 Aug 2026 16:07:03 -0700 Subject: [PATCH 39/40] Align HA completion with local status API --- server/cmd/fleet-ha/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/cmd/fleet-ha/main_test.go b/server/cmd/fleet-ha/main_test.go index babc900e36..785cb0a3af 100644 --- a/server/cmd/fleet-ha/main_test.go +++ b/server/cmd/fleet-ha/main_test.go @@ -157,7 +157,7 @@ func TestPassiveUpdateAllowsExpectedVersionMismatch(t *testing.T) { func TestCompleteUpdateRejectsExpectedVersionMismatch(t *testing.T) { // Arrange client := &fakeUpdaterClient{} - read := func(context.Context, string, bool) (deployment.StatusReport, error) { + read := func(context.Context, string) (deployment.StatusReport, error) { return deployment.StatusReport{Control: &deployment.ControlStatus{ ControlReady: true, ReasonCodes: []deployment.ControlReasonCode{deployment.ReasonFleetVersionMismatch}, From b1f29ca4eaebb85c241b032fbb69c426367e6680 Mon Sep 17 00:00:00 2001 From: Ankit Goswami Date: Tue, 11 Aug 2026 12:52:20 -0700 Subject: [PATCH 40/40] Adapt HA completion test to Kong CLI --- server/cmd/fleet-ha/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/cmd/fleet-ha/main_test.go b/server/cmd/fleet-ha/main_test.go index 785cb0a3af..7d58314cd4 100644 --- a/server/cmd/fleet-ha/main_test.go +++ b/server/cmd/fleet-ha/main_test.go @@ -165,7 +165,7 @@ func TestCompleteUpdateRejectsExpectedVersionMismatch(t *testing.T) { } // Act - err := runPassiveUpdate(t.Context(), []string{"v1.2.3", "--complete"}, &bytes.Buffer{}, func(context.Context, string, string, bool) error { return nil }, client, read) + err := runPassiveUpdate(t.Context(), "v1.2.3", true, &bytes.Buffer{}, func(context.Context, string, string, bool) error { return nil }, client, read) // Assert require.ErrorContains(t, err, "failover readiness is degraded")