From 14a80be76920d67f9d0d016e2ad93e3fbc0c6e43 Mon Sep 17 00:00:00 2001 From: Marvin Charles Date: Thu, 6 Aug 2026 12:31:17 +0300 Subject: [PATCH 01/13] feat(updates): add one-click upgrade experience --- .../settings/components/Updates.test.tsx | 203 +++++++++- .../features/settings/components/Updates.tsx | 167 +++++++- .../components/UpgradeOperationModal.test.tsx | 252 ++++++++++++ .../components/UpgradeOperationModal.tsx | 268 +++++++++++++ .../updates/api/useUpgradeOperation.test.tsx | 247 ++++++++++++ .../updates/api/useUpgradeOperation.ts | 371 ++++++++++++++++++ .../features/updates/copyInstallCommand.ts | 4 +- ...6-07-29-one-click-upgrade-executor-plan.md | 115 ++++++ 8 files changed, 1618 insertions(+), 9 deletions(-) create mode 100644 client/src/protoFleet/features/settings/components/UpgradeOperationModal.test.tsx create mode 100644 client/src/protoFleet/features/settings/components/UpgradeOperationModal.tsx create mode 100644 client/src/protoFleet/features/updates/api/useUpgradeOperation.test.tsx create mode 100644 client/src/protoFleet/features/updates/api/useUpgradeOperation.ts create mode 100644 docs/plans/archive/2026-07-29-one-click-upgrade-executor-plan.md diff --git a/client/src/protoFleet/features/settings/components/Updates.test.tsx b/client/src/protoFleet/features/settings/components/Updates.test.tsx index 2e5d31565a..d84627f967 100644 --- a/client/src/protoFleet/features/settings/components/Updates.test.tsx +++ b/client/src/protoFleet/features/settings/components/Updates.test.tsx @@ -1,4 +1,4 @@ -import { act, fireEvent, render, waitFor } from "@testing-library/react"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { create } from "@bufbuild/protobuf"; import { Code, ConnectError } from "@connectrpc/connect"; @@ -9,13 +9,17 @@ import type { GetUpdateStatusResponse, ReleaseInfo, SetReleaseChannelResponse, + UpgradeOperation, } from "@/protoFleet/api/generated/instance/v1/updates_pb"; import { GetUpdateStatusResponseSchema, ReleaseChannel, ReleaseInfoSchema, SetReleaseChannelResponseSchema, + UpgradeOperationSchema, + UpgradePhase, } from "@/protoFleet/api/generated/instance/v1/updates_pb"; +import { useUpgradeOperation } from "@/protoFleet/features/updates/api/useUpgradeOperation"; import { useHasPermission } from "@/protoFleet/store"; import { pushToast } from "@/shared/features/toaster"; import { copyToClipboard } from "@/shared/utils/utility"; @@ -29,6 +33,22 @@ const permissionsMock = vi.hoisted(() => ({ const authErrorsMock = vi.hoisted(() => ({ handleAuthErrors: vi.fn(), })); +interface UpgradeHookMockState { + acknowledgeOperation: ReturnType; + connectionLost: boolean; + manualFallbackReady: boolean; + operation?: UpgradeOperation; + reconciling: boolean; + reloadFleet: ReturnType; + triggerError: string | null; + triggering: boolean; + trackedTargetVersion?: string; + triggerUpgrade: ReturnType; + useManualFallback: ReturnType; +} +const upgradeHookMock = vi.hoisted(() => ({ + current: {} as UpgradeHookMockState, +})); vi.mock("react-router-dom", () => ({ Navigate: ({ to }: { to: string }) =>
, @@ -64,6 +84,12 @@ vi.mock("@/protoFleet/api/clients", () => ({ }, })); +vi.mock("@/protoFleet/features/updates/api/useUpgradeOperation", () => ({ + isUpgradeActive: (operation?: { phase: number }) => + Boolean(operation && operation.phase !== 0 && operation.phase !== 7 && operation.phase !== 8), + useUpgradeOperation: vi.fn(() => upgradeHookMock.current), +})); + vi.mock("@/shared/utils/utility", () => ({ copyToClipboard: vi.fn(), })); @@ -102,7 +128,17 @@ const buildStatus = (overrides?: MessageOverrides): Get ...overrides, }); +const buildOperation = (phase: UpgradePhase, overrides?: MessageOverrides): UpgradeOperation => + create(UpgradeOperationSchema, { + id: "operation-1", + targetVersion: "v1.3.0", + phase, + message: "Preparing upgrade", + ...overrides, + }); + const mockUseHasPermission = vi.mocked(useHasPermission); +const mockUseUpgradeOperation = vi.mocked(useUpgradeOperation); const mockGetUpdateStatus = vi.mocked(instanceUpdateClient.getUpdateStatus); const mockSetReleaseChannel = vi.mocked(instanceUpdateClient.setReleaseChannel); const mockCopyToClipboard = vi.mocked(copyToClipboard); @@ -125,6 +161,20 @@ const createDeferred = () => { beforeEach(() => { vi.clearAllMocks(); localStorage.clear(); + sessionStorage.clear(); + upgradeHookMock.current = { + acknowledgeOperation: vi.fn(), + connectionLost: false, + manualFallbackReady: false, + operation: undefined, + reconciling: false, + reloadFleet: vi.fn(), + triggerError: null, + triggering: false, + trackedTargetVersion: undefined, + triggerUpgrade: vi.fn().mockResolvedValue(undefined), + useManualFallback: vi.fn(), + }; permissionsMock.current = ["instance:update", "fleet:read"]; permissionsMock.isAuthenticated = true; permissionsMock.sessionExpiry = new Date(1_000); @@ -150,6 +200,135 @@ describe("Updates", () => { expect(link).toHaveAttribute("rel", "noopener noreferrer"); expect(getByText(INSTALL_COMMAND)).toBeInTheDocument(); expect(getByRole("button", { name: "Copy install command" })).toBeInTheDocument(); + expect(getByRole("button", { name: "Copy install command" })).toBeEnabled(); + expect(screen.queryByRole("button", { name: "Upgrade to v1.3.0" })).not.toBeInTheDocument(); + }); + + it("confirms the exact offered version before starting a one-click upgrade", async () => { + mockGetUpdateStatus.mockResolvedValue(buildStatus({ oneClickAvailable: true })); + + const page = render(); + fireEvent.click(await page.findByRole("button", { name: "Upgrade to v1.3.0" })); + + expect(page.getByTestId("upgrade-operation-modal")).toHaveTextContent( + "Fleet will validate and build this exact release", + ); + expect(upgradeHookMock.current.triggerUpgrade).not.toHaveBeenCalled(); + + fireEvent.click(page.getByRole("button", { name: "Confirm upgrade to v1.3.0" })); + await waitFor(() => expect(upgradeHookMock.current.triggerUpgrade).toHaveBeenCalledWith("v1.3.0")); + }); + + it("keeps an active operation ahead of a newer release offer", async () => { + upgradeHookMock.current.operation = buildOperation(UpgradePhase.PREFLIGHT, { + targetVersion: "v1.3.0", + message: "Validating v1.3.0", + }); + mockGetUpdateStatus.mockResolvedValue( + buildStatus({ + oneClickAvailable: true, + installCommand: "install v1.4.0", + latestEligible: buildReleaseInfo({ version: "v1.4.0" }), + }), + ); + + const page = render(); + + expect((await page.findAllByText("Validating v1.3.0")).length).toBeGreaterThan(0); + expect(page.queryByRole("button", { name: "Upgrade to v1.4.0" })).not.toBeInTheDocument(); + expect(page.getByRole("button", { name: "Copy install command" })).toBeDisabled(); + expect(page.getByRole("checkbox", { name: RC_CHECKBOX_NAME })).toBeDisabled(); + + fireEvent.click(page.getByRole("button", { name: "Close dialog" })); + fireEvent.click(page.getByRole("button", { name: "View upgrade details" })); + expect(page.getAllByText("Validating v1.3.0").length).toBeGreaterThan(0); + }); + + it("keeps manual recovery usable after a failed operation and can acknowledge its durable record", async () => { + upgradeHookMock.current.operation = buildOperation(UpgradePhase.FAILED, { + error: "new stack failed to start", + hostLogPath: "/var/lib/proto-fleet-updater/logs/operation-1.log", + recoveryCommand: "cd /opt/proto-fleet/deployment && ./run-fleet.sh --skip-build", + }); + mockGetUpdateStatus.mockResolvedValue(buildStatus({ oneClickAvailable: true })); + + const page = render(); + + expect(await page.findByText("new stack failed to start")).toBeInTheDocument(); + expect(page.getByRole("button", { name: "Copy install command" })).toBeEnabled(); + fireEvent.click(page.getByRole("button", { name: "Dismiss failure" })); + expect(upgradeHookMock.current.acknowledgeOperation).toHaveBeenCalledTimes(1); + }); + + it("offers a reload after the watched operation succeeds", async () => { + upgradeHookMock.current.operation = buildOperation(UpgradePhase.SUCCEEDED, { + message: "Upgrade complete", + }); + mockGetUpdateStatus.mockResolvedValue( + buildStatus({ + currentVersion: "v1.3.0", + updateAvailable: false, + installCommand: "", + latestEligible: undefined, + oneClickAvailable: true, + }), + ); + + const page = render(); + fireEvent.click(await page.findByRole("button", { name: "Reload Fleet" })); + + expect(upgradeHookMock.current.reloadFleet).toHaveBeenCalledTimes(1); + }); + + it("locks competing controls while reconciling an ambiguous trigger", async () => { + upgradeHookMock.current.reconciling = true; + upgradeHookMock.current.triggerError = "Fleet did not confirm the request"; + mockGetUpdateStatus.mockResolvedValue(buildStatus({ oneClickAvailable: true })); + + const page = render(); + + expect(await page.findByText(/checking whether the upgrade started/i)).toBeInTheDocument(); + expect(page.getByRole("button", { name: "Copy install command" })).toBeDisabled(); + expect(page.getByRole("checkbox", { name: RC_CHECKBOX_NAME })).toBeDisabled(); + }); + + it("keeps an ambiguous request's target when a newer release is now eligible", async () => { + upgradeHookMock.current.reconciling = true; + upgradeHookMock.current.trackedTargetVersion = "v1.3.0"; + upgradeHookMock.current.triggerError = "Fleet did not confirm the v1.3.0 request"; + mockGetUpdateStatus.mockResolvedValue( + buildStatus({ + installCommand: "install v1.4.0", + latestEligible: buildReleaseInfo({ version: "v1.4.0" }), + oneClickAvailable: true, + }), + ); + + const page = render(); + + expect(await page.findByText("v1.4.0")).toBeInTheDocument(); + expect(page.getByTestId("upgrade-operation-modal")).toHaveTextContent("Upgrade Fleet to v1.3.0"); + }); + + it("refreshes the eligible release after reconciliation finds no operation", async () => { + upgradeHookMock.current.reconciling = true; + upgradeHookMock.current.triggerError = "Fleet did not confirm the request"; + mockGetUpdateStatus.mockResolvedValueOnce(buildStatus({ oneClickAvailable: true })).mockResolvedValueOnce( + buildStatus({ + installCommand: "install v1.4.0", + latestEligible: buildReleaseInfo({ version: "v1.4.0" }), + oneClickAvailable: true, + }), + ); + + const page = render(); + expect(await page.findByText("v1.3.0")).toBeInTheDocument(); + + upgradeHookMock.current = { ...upgradeHookMock.current, reconciling: false }; + page.rerender(); + + await waitFor(() => expect(mockGetUpdateStatus).toHaveBeenCalledTimes(2)); + expect(await page.findByText("v1.4.0")).toBeInTheDocument(); }); it("omits the release notes link when the server provides no URL", async () => { @@ -692,6 +871,28 @@ describe("Updates", () => { expect(page.getByTestId("navigate")).toHaveAttribute("data-to", "/settings/network"); }); + it("invalidates stale client permission when upgrade polling is denied", async () => { + mockGetUpdateStatus.mockResolvedValue(buildStatus()); + + const page = render(); + await page.findByText("v1.2.0"); + const lastHookCall = mockUseUpgradeOperation.mock.calls[mockUseUpgradeOperation.mock.calls.length - 1]; + const onPollError = lastHookCall?.[0].onPollError; + + act(() => { + onPollError?.(new ConnectError("permission revoked", Code.PermissionDenied)); + }); + + expect(mockPushToast).toHaveBeenCalledWith({ + message: PERMISSION_REVOKED_MESSAGE, + status: "error", + }); + expect(permissionsMock.setPermissions).toHaveBeenCalledWith(["fleet:read"]); + + page.rerender(); + expect(page.getByTestId("navigate")).toHaveAttribute("data-to", "/settings/network"); + }); + it("toasts an error and leaves the checkbox unchecked when saving the channel fails", async () => { mockGetUpdateStatus .mockResolvedValueOnce(buildStatus({ channel: ReleaseChannel.STABLE })) diff --git a/client/src/protoFleet/features/settings/components/Updates.tsx b/client/src/protoFleet/features/settings/components/Updates.tsx index 7119e49a05..844ede6fd4 100644 --- a/client/src/protoFleet/features/settings/components/Updates.tsx +++ b/client/src/protoFleet/features/settings/components/Updates.tsx @@ -1,16 +1,19 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { Navigate } from "react-router-dom"; import { instanceUpdateClient } from "@/protoFleet/api/clients"; -import { ReleaseChannel } from "@/protoFleet/api/generated/instance/v1/updates_pb"; +import { ReleaseChannel, UpgradePhase } from "@/protoFleet/api/generated/instance/v1/updates_pb"; import type { GetUpdateStatusResponse } from "@/protoFleet/api/generated/instance/v1/updates_pb"; import { getErrorMessage } from "@/protoFleet/api/getErrorMessage"; import { isAuthOrPermissionError, isPermissionDeniedError } from "@/protoFleet/api/requestErrors"; import { getSettingsLandingPath } from "@/protoFleet/config/navItems"; import SettingsEmptyState from "@/protoFleet/features/settings/components/SettingsEmptyState"; import SettingsPageHeader from "@/protoFleet/features/settings/components/SettingsPageHeader"; +import UpgradeOperationModal from "@/protoFleet/features/settings/components/UpgradeOperationModal"; +import { isUpgradeActive, useUpgradeOperation } from "@/protoFleet/features/updates/api/useUpgradeOperation"; import { copyInstallCommand } from "@/protoFleet/features/updates/copyInstallCommand"; import { useAuthErrors, useFleetStore, useHasPermission, usePermissions, useSetPermissions } from "@/protoFleet/store"; import { Copy } from "@/shared/assets/icons"; +import Button, { variants } from "@/shared/components/Button"; import Checkbox from "@/shared/components/Checkbox"; import Header from "@/shared/components/Header"; import Row from "@/shared/components/Row"; @@ -21,7 +24,8 @@ const SkeletonLoader = ; const INSTANCE_UPDATE_PERMISSION = "instance:update"; const RELEASE_CHANNEL_SAVE_TIMEOUT_MS = 30_000; const PERMISSION_REVOKED_MESSAGE = "You no longer have permission to update this instance"; -const UPDATES_PAGE_DESCRIPTION = "View the server version and choose which releases this instance installs."; +const UPDATES_PAGE_DESCRIPTION = + "View the server version, choose which releases this instance installs, and apply eligible updates."; interface AuthSessionSnapshot { isAuthenticated: boolean; @@ -88,8 +92,12 @@ const Updates = () => { const [status, setStatus] = useState(null); const [loadError, setLoadError] = useState(null); const [isChannelChangePending, setIsChannelChangePending] = useState(false); + const [upgradeModalOpen, setUpgradeModalOpen] = useState(false); const latestStatusRequest = useRef(0); const isMounted = useRef(false); + const lastAutoOpenedOperation = useRef(null); + const previousReconciling = useRef(false); + const previousTriggerError = useRef(null); // The channel the server has persisted; the checkbox is controlled by it, // so a failed save never moves the control. const [channel, setChannel] = useState(ReleaseChannel.UNSPECIFIED); @@ -110,6 +118,32 @@ const Updates = () => { [setPermissions], ); + const handleUpgradePollError = useCallback( + (error: unknown) => { + handleAuthErrors({ + error, + onError: () => { + if (isPermissionDeniedError(error)) { + handlePermissionRevoked(isMounted.current); + } + }, + }); + }, + [handleAuthErrors, handlePermissionRevoked], + ); + + const upgrade = useUpgradeOperation({ + enabled: canUpdateInstance, + currentVersion: status?.currentVersion, + onPollError: handleUpgradePollError, + }); + const activeUpgrade = isUpgradeActive(upgrade.operation); + const succeededUpgrade = upgrade.operation?.phase === UpgradePhase.SUCCEEDED; + const upgradeRequestPending = upgrade.triggering || upgrade.reconciling; + const upgradeLocksConfiguration = upgradeRequestPending || Boolean(upgrade.operation); + const manualCommandDisabled = + isChannelChangePending || activeUpgrade || upgradeRequestPending || Boolean(succeededUpgrade); + const fetchStatus = useCallback(async () => { const requestId = ++latestStatusRequest.current; const authSession = captureAuthSession(); @@ -153,6 +187,45 @@ const Updates = () => { }; }, []); + useEffect(() => { + const operation = upgrade.operation; + if (!operation) { + return; + } + const terminal = operation.phase === UpgradePhase.SUCCEEDED || operation.phase === UpgradePhase.FAILED; + const autoOpenKey = `${operation.id}:${terminal ? "terminal" : "active"}`; + if (lastAutoOpenedOperation.current === autoOpenKey) { + return; + } + // Open once when an operation is first recovered, and once more when it + // becomes terminal. Intermediate phase updates must not keep stealing + // focus after an operator dismisses the progress modal. + lastAutoOpenedOperation.current = autoOpenKey; + setUpgradeModalOpen(true); + }, [upgrade.operation]); + + useEffect(() => { + const wasReconciling = previousReconciling.current; + if (upgrade.reconciling && !previousReconciling.current) { + setUpgradeModalOpen(true); + } + if (wasReconciling && !upgrade.reconciling && !upgrade.operation && upgrade.triggerError) { + // A reachable executor authoritatively found no matching operation. + // Refresh the offer before allowing a retry because the release/channel + // may have changed while the trigger outcome was being reconciled. + void fetchStatus(); + } + previousReconciling.current = upgrade.reconciling; + }, [fetchStatus, upgrade.operation, upgrade.reconciling, upgrade.triggerError]); + + useEffect(() => { + if (upgrade.triggerError && upgrade.triggerError !== previousTriggerError.current && !upgrade.reconciling) { + setUpgradeModalOpen(true); + void fetchStatus(); + } + previousTriggerError.current = upgrade.triggerError; + }, [fetchStatus, upgrade.reconciling, upgrade.triggerError]); + useEffect(() => { // The RPC is server-gated on instance:update; non-holders are redirected // below and must not fire it. @@ -169,7 +242,7 @@ const Updates = () => { const handleIncludeRCChange = async (includeRC: boolean) => { const nextChannel = includeRC ? ReleaseChannel.STABLE_AND_RC : ReleaseChannel.STABLE; - if (nextChannel === channel || isChannelChangePending) { + if (nextChannel === channel || isChannelChangePending || upgradeLocksConfiguration) { return; } const authSession = captureAuthSession(); @@ -233,10 +306,59 @@ const Updates = () => { } const release = status?.statusAvailable && status.updateAvailable ? status.latestEligible : undefined; + const modalRelease = upgrade.operation && upgrade.operation.targetVersion !== release?.version ? undefined : release; + const operationStatusLabel = upgrade.reconciling + ? upgrade.manualFallbackReady + ? "Upgrade outcome is unknown — host confirmation required" + : "Confirming whether the upgrade started" + : upgrade.operation?.phase === UpgradePhase.FAILED + ? "Upgrade failed" + : upgrade.operation?.phase === UpgradePhase.SUCCEEDED + ? "Upgrade complete — reload Fleet" + : upgrade.operation + ? upgrade.operation.message || `Upgrading Fleet to ${upgrade.operation.targetVersion}` + : upgrade.triggerError + ? "Upgrade request needs attention" + : null; + const hasUpgradeDetails = Boolean(upgrade.operation || upgrade.reconciling || upgrade.triggerError); return (
+ { + upgrade.acknowledgeOperation(); + setUpgradeModalOpen(false); + }} + onDismiss={() => setUpgradeModalOpen(false)} + onReload={upgrade.reloadFleet} + onUpgrade={upgrade.triggerUpgrade} + onUseManualFallback={() => { + upgrade.useManualFallback(); + setUpgradeModalOpen(false); + void fetchStatus(); + }} + open={upgradeModalOpen} + operation={upgrade.operation} + reconciling={upgrade.reconciling} + release={modalRelease} + targetVersion={ + upgrade.operation?.targetVersion ?? (upgrade.reconciling ? upgrade.trackedTargetVersion : release?.version) + } + triggerError={upgrade.triggerError} + triggering={upgrade.triggering} + /> + {loadError && hasUpgradeDetails ? ( +
+
+
Upgrade status
+
{operationStatusLabel}
+
+
+ ) : null} {loadError ? ( ) : ( @@ -268,13 +390,33 @@ const Updates = () => { ) : null}
+ {status?.oneClickAvailable || hasUpgradeDetails ? ( + +
+
{hasUpgradeDetails ? "Upgrade status" : "One-click upgrade"}
+ {operationStatusLabel ? ( +
{operationStatusLabel}
+ ) : ( +
+ Fleet validates the release before restarting the instance. +
+ )} +
+
@@ -303,7 +458,7 @@ const Updates = () => { void handleIncludeRCChange(e.target.checked)} /> Include release candidates diff --git a/client/src/protoFleet/features/settings/components/UpgradeOperationModal.test.tsx b/client/src/protoFleet/features/settings/components/UpgradeOperationModal.test.tsx new file mode 100644 index 0000000000..b061f0a33e --- /dev/null +++ b/client/src/protoFleet/features/settings/components/UpgradeOperationModal.test.tsx @@ -0,0 +1,252 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { create } from "@bufbuild/protobuf"; + +import { + ReleaseInfoSchema, + type UpgradeOperation, + UpgradeOperationSchema, + UpgradePhase, +} from "@/protoFleet/api/generated/instance/v1/updates_pb"; +import UpgradeOperationModal, { + type UpgradeOperationModalProps, +} from "@/protoFleet/features/settings/components/UpgradeOperationModal"; +import { pushToast, STATUSES } from "@/shared/features/toaster"; +import { copyToClipboard } from "@/shared/utils/utility"; + +vi.mock("@/shared/features/toaster", () => ({ + pushToast: vi.fn(), + STATUSES: { + error: "error", + success: "success", + }, +})); + +vi.mock("@/shared/utils/utility", () => ({ + copyToClipboard: vi.fn(), +})); + +const mockCopyToClipboard = vi.mocked(copyToClipboard); +const mockPushToast = vi.mocked(pushToast); + +const release = (version = "v1.3.0", prerelease = false) => + create(ReleaseInfoSchema, { + version, + prerelease, + }); + +type OperationOverrides = Partial< + Pick +>; + +const operation = (phase: UpgradePhase, overrides?: OperationOverrides) => + create(UpgradeOperationSchema, { + id: "operation-1", + targetVersion: "v1.3.0", + phase, + message: "Preparing upgrade", + ...overrides, + }); + +const renderModal = (overrides: Partial = {}) => { + const props: UpgradeOperationModalProps = { + connectionLost: false, + manualFallbackReady: false, + onAcknowledge: vi.fn(), + onDismiss: vi.fn(), + onReload: vi.fn(), + onUpgrade: vi.fn().mockResolvedValue(undefined), + onUseManualFallback: vi.fn(), + open: true, + reconciling: false, + release: release(), + triggerError: null, + triggering: false, + ...overrides, + }; + return { ...render(), props }; +}; + +describe("UpgradeOperationModal", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockCopyToClipboard.mockResolvedValue(undefined); + }); + + it("confirms the exact stable target before starting the upgrade", async () => { + const onUpgrade = vi.fn().mockResolvedValue(undefined); + renderModal({ onUpgrade }); + + const confirmButton = screen.getByRole("button", { name: "Confirm upgrade to v1.3.0" }); + expect(confirmButton).toBeInTheDocument(); + fireEvent.click(confirmButton); + + await waitFor(() => expect(onUpgrade).toHaveBeenCalledWith("v1.3.0")); + }); + + it("gives release candidates a strong forward-only migration warning", () => { + renderModal({ release: release("v1.3.0-rc.2", true) }); + + expect(screen.getByText(/This is a release candidate/)).toBeInTheDocument(); + expect(screen.getByText(/forward-only database migrations/)).toBeInTheDocument(); + expect(screen.getByText(/cannot downgrade this instance afterward/)).toBeInTheDocument(); + }); + + it("keeps a long-running upgrade dismissible", () => { + const onDismiss = vi.fn(); + renderModal({ + onDismiss, + operation: operation(UpgradePhase.PREFLIGHT, { message: "Validating the new stack" }), + }); + + expect(screen.getByRole("status")).toHaveTextContent("Phase: Preflight"); + fireEvent.click(screen.getByRole("button", { name: "Close dialog" })); + expect(onDismiss).toHaveBeenCalledOnce(); + }); + + it("announces the expected reconnect state", () => { + renderModal({ + connectionLost: true, + operation: operation(UpgradePhase.ACTIVATING), + }); + + const status = screen.getByRole("status"); + expect(status).toHaveAttribute("aria-live", "polite"); + expect(status).toHaveTextContent(/disconnect is expected while services restart/); + }); + + it("warns against manual installation while reconciling an ambiguous trigger", () => { + renderModal({ reconciling: true, triggerError: "The request timed out" }); + + expect(screen.getByRole("status")).toHaveTextContent(/Checking whether the upgrade started/); + expect(screen.getByRole("status")).toHaveTextContent(/Do not run the manual install command yet/); + expect(screen.queryByRole("button", { name: /Confirm upgrade/ })).not.toBeInTheDocument(); + }); + + it("requires explicit host confirmation before unlocking a manual fallback", () => { + const onUseManualFallback = vi.fn(); + renderModal({ + connectionLost: true, + manualFallbackReady: true, + onUseManualFallback, + reconciling: true, + triggerError: "Host updater did not confirm the upgrade", + }); + + expect(screen.getByRole("status")).toHaveTextContent(/checking the host and confirming no upgrade is running/i); + fireEvent.click(screen.getByRole("button", { name: "I confirmed — unlock manual install" })); + expect(onUseManualFallback).toHaveBeenCalledOnce(); + }); + + it("labels reconciliation with its tracked target rather than a newer offer", () => { + renderModal({ + reconciling: true, + release: release("v1.4.0"), + targetVersion: "v1.3.0", + triggerError: "The v1.3.0 request has an unknown outcome", + }); + + expect(screen.getByTestId("upgrade-operation-modal")).toHaveTextContent("Upgrade Fleet to v1.3.0"); + expect(screen.getByTestId("upgrade-operation-modal")).not.toHaveTextContent("Upgrade Fleet to v1.4.0"); + }); + + it("shows failed-operation details and copies the recovery command", async () => { + const recoveryCommand = "cd /opt/proto-fleet/deployment && ./run-fleet.sh --non-interactive --skip-build"; + renderModal({ + operation: operation(UpgradePhase.FAILED, { + message: "Upgrade failed", + error: "new stack failed to start", + hostLogPath: "/var/lib/proto-fleet-updater/logs/operation-1.log", + recoveryCommand, + }), + }); + + const alert = screen.getByRole("alert"); + expect(alert).toHaveTextContent("new stack failed to start"); + expect(alert).toHaveTextContent("operation-1.log"); + fireEvent.click(screen.getByRole("button", { name: "Copy recovery command" })); + + await waitFor(() => expect(mockCopyToClipboard).toHaveBeenCalledWith(recoveryCommand)); + expect(mockPushToast).toHaveBeenCalledWith({ + message: "Recovery command copied to clipboard", + status: STATUSES.success, + }); + }); + + it("reports a recovery-command copy failure", async () => { + mockCopyToClipboard.mockRejectedValue(new Error("copy failed")); + renderModal({ + operation: operation(UpgradePhase.FAILED, { + recoveryCommand: "./run-fleet.sh --non-interactive --skip-build", + }), + }); + + fireEvent.click(screen.getByRole("button", { name: "Copy recovery command" })); + + await waitFor(() => + expect(mockPushToast).toHaveBeenCalledWith({ + message: "Failed to copy recovery command", + status: STATUSES.error, + }), + ); + }); + + it("does not render an empty recovery fallback", () => { + renderModal({ + operation: operation(UpgradePhase.FAILED, { + error: "preflight failed", + recoveryCommand: " ", + }), + }); + + expect(screen.queryByText("Recovery command")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Copy recovery command" })).not.toBeInTheDocument(); + }); + + it("acknowledges a failure separately from hiding its details", () => { + const onAcknowledge = vi.fn(); + const onDismiss = vi.fn(); + renderModal({ + onAcknowledge, + onDismiss, + operation: operation(UpgradePhase.FAILED), + }); + + fireEvent.click(screen.getByRole("button", { name: "Dismiss failure" })); + expect(onAcknowledge).toHaveBeenCalledOnce(); + expect(onDismiss).not.toHaveBeenCalled(); + }); + + it("reloads Fleet after a successful upgrade", () => { + const onReload = vi.fn(); + renderModal({ + onReload, + operation: operation(UpgradePhase.SUCCEEDED, { message: "Fleet v1.3.0 is running" }), + }); + + fireEvent.click(screen.getByRole("button", { name: "Reload Fleet" })); + expect(onReload).toHaveBeenCalledOnce(); + }); + + it("announces a trigger error and allows retry after reconciliation", async () => { + const onUpgrade = vi.fn().mockResolvedValue(undefined); + renderModal({ + onUpgrade, + triggerError: "Host updater did not answer", + }); + + expect(screen.getByRole("alert")).toHaveTextContent("Host updater did not answer"); + fireEvent.click(screen.getByRole("button", { name: "Confirm upgrade to v1.3.0" })); + await waitFor(() => expect(onUpgrade).toHaveBeenCalledWith("v1.3.0")); + }); + + it("keeps a trigger error visible when there is no longer an eligible release", () => { + renderModal({ + release: undefined, + triggerError: "The eligible release changed before the request completed", + }); + + expect(screen.getByRole("alert")).toHaveTextContent("The eligible release changed before the request completed"); + expect(screen.queryByRole("button", { name: /Confirm upgrade/ })).not.toBeInTheDocument(); + }); +}); diff --git a/client/src/protoFleet/features/settings/components/UpgradeOperationModal.tsx b/client/src/protoFleet/features/settings/components/UpgradeOperationModal.tsx new file mode 100644 index 0000000000..6601c76728 --- /dev/null +++ b/client/src/protoFleet/features/settings/components/UpgradeOperationModal.tsx @@ -0,0 +1,268 @@ +import { + type ReleaseInfo, + type UpgradeOperation, + UpgradePhase, +} from "@/protoFleet/api/generated/instance/v1/updates_pb"; +import { Copy } from "@/shared/assets/icons"; +import Button, { variants } from "@/shared/components/Button"; +import Modal from "@/shared/components/Modal"; +import ProgressCircular from "@/shared/components/ProgressCircular"; +import { pushToast, STATUSES } from "@/shared/features/toaster"; +import { copyToClipboard } from "@/shared/utils/utility"; + +export interface UpgradeOperationModalProps { + connectionLost: boolean; + manualFallbackReady: boolean; + onAcknowledge: () => void; + onDismiss: () => void; + onReload: () => void; + onUpgrade: (targetVersion: string) => Promise; + onUseManualFallback: () => void; + open: boolean; + operation?: UpgradeOperation; + reconciling: boolean; + release?: ReleaseInfo; + targetVersion?: string; + triggerError: string | null; + triggering: boolean; +} + +const ACTIVE_PHASE_LABELS: Partial> = { + [UpgradePhase.QUEUED]: "Queued", + [UpgradePhase.DOWNLOADING]: "Downloading", + [UpgradePhase.VERIFYING]: "Verifying", + [UpgradePhase.STAGING]: "Staging", + [UpgradePhase.PREFLIGHT]: "Preflight", + [UpgradePhase.ACTIVATING]: "Activating", +}; + +const UpgradeOperationModal = ({ + connectionLost, + manualFallbackReady, + onAcknowledge, + onDismiss, + onReload, + onUpgrade, + onUseManualFallback, + open, + operation, + reconciling, + release, + targetVersion, + triggerError, + triggering, +}: UpgradeOperationModalProps) => { + if (!release && !operation && !reconciling && !triggerError) { + return null; + } + + const displayedTargetVersion = operation?.targetVersion || targetVersion || release?.version; + const failed = operation?.phase === UpgradePhase.FAILED; + const succeeded = operation?.phase === UpgradePhase.SUCCEEDED; + const active = Boolean(operation && !failed && !succeeded); + const activePhaseLabel = operation ? (ACTIVE_PHASE_LABELS[operation.phase] ?? "Starting") : undefined; + const error = operation?.error.trim() ?? ""; + const hostLogPath = operation?.hostLogPath.trim() ?? ""; + const recoveryCommand = operation?.recoveryCommand.trim() ?? ""; + + const handleUpgrade = () => { + if (!release || reconciling) return; + void onUpgrade(release.version).catch(() => { + // The route owns reconciliation and exposes a terminal triggerError. + }); + }; + + const handleCopyRecoveryCommand = () => { + if (!recoveryCommand) return; + void copyToClipboard(recoveryCommand) + .then(() => { + pushToast({ + message: "Recovery command copied to clipboard", + status: STATUSES.success, + }); + }) + .catch(() => { + pushToast({ + message: "Failed to copy recovery command", + status: STATUSES.error, + }); + }); + }; + + const buttons = manualFallbackReady + ? [ + { + text: "I confirmed — unlock manual install", + variant: variants.secondaryDanger, + onClick: onUseManualFallback, + dismissModalOnClick: false, + }, + ] + : succeeded + ? [ + { + text: "Reload Fleet", + variant: variants.primary, + onClick: onReload, + dismissModalOnClick: false, + }, + ] + : failed + ? [ + { + text: "Dismiss failure", + variant: variants.secondary, + onClick: onAcknowledge, + dismissModalOnClick: false, + }, + ] + : !active && !reconciling && release + ? [ + { + text: "Cancel", + variant: variants.secondary, + onClick: onDismiss, + dismissModalOnClick: false, + }, + { + text: `Confirm upgrade to ${release.version}`, + variant: variants.primary, + onClick: handleUpgrade, + loading: triggering, + dismissModalOnClick: false, + }, + ] + : undefined; + + return ( + +
+ {reconciling ? ( +
+
+ +
+ {manualFallbackReady + ? "Fleet could not confirm the upgrade outcome" + : "Checking whether the upgrade started"} +
+
+ {manualFallbackReady ? ( +

+ The host updater is still unreachable. Only unlock the manual command after checking the host and + confirming no upgrade is running; overlapping installs can leave the deployment unusable. +

+ ) : ( +

+ Fleet is reconciling the request with the host updater. Do not run the manual install command yet; wait + until this check finishes. +

+ )} +
+ ) : active ? ( +
+
+ +
{operation?.message || "Upgrade in progress"}
+
+
Phase: {activePhaseLabel}
+ {connectionLost ? ( +

+ Fleet is temporarily unreachable. A disconnect is expected while services restart; this page will keep + checking for progress. +

+ ) : ( +

+ You can close this dialog while Fleet downloads, validates, and activates the release. Return to this + Updates page to check progress. +

+ )} +
+ ) : failed ? ( +
+
{operation?.message || "Upgrade failed"}
+ {error ?

{error}

: null} + {hostLogPath ? ( +

+ Host log: {hostLogPath} +

+ ) : null} + {recoveryCommand ? ( +
+
Recovery command
+
+ + {recoveryCommand} + +
+
+ ) : null} +
+ ) : succeeded ? ( +
+
{operation?.message || "Upgrade complete"}
+

+ Reload Fleet to use the client bundled with {operation?.targetVersion}. +

+
+ ) : release ? ( +
+
Confirm upgrade to {release.version}
+

+ Fleet will validate and build this exact release first, then take the instance offline for several minutes + while containers restart and database migrations run. +

+ {release.prerelease ? ( +

+ This is a release candidate. The upgrade can run forward-only database migrations, and you cannot + downgrade this instance afterward. Continue only if you accept that recovery may require a newer + compatible release. +

+ ) : null} +
+ ) : null} + + {triggerError ? ( +

+ {triggerError} +

+ ) : null} +
+ + ); +}; + +export default UpgradeOperationModal; diff --git a/client/src/protoFleet/features/updates/api/useUpgradeOperation.test.tsx b/client/src/protoFleet/features/updates/api/useUpgradeOperation.test.tsx new file mode 100644 index 0000000000..d191b271f8 --- /dev/null +++ b/client/src/protoFleet/features/updates/api/useUpgradeOperation.test.tsx @@ -0,0 +1,247 @@ +import { act, renderHook, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { create } from "@bufbuild/protobuf"; + +import { instanceUpdateClient } from "@/protoFleet/api/clients"; +import { + GetUpgradeStatusResponseSchema, + TriggerUpgradeResponseSchema, + type UpgradeOperation, + UpgradeOperationSchema, + UpgradePhase, +} from "@/protoFleet/api/generated/instance/v1/updates_pb"; +import { useUpgradeOperation } from "@/protoFleet/features/updates/api/useUpgradeOperation"; + +vi.mock("@/protoFleet/api/clients", () => ({ + instanceUpdateClient: { + getUpgradeStatus: vi.fn(), + triggerUpgrade: vi.fn(), + }, +})); + +const mockGetUpgradeStatus = vi.mocked(instanceUpdateClient.getUpgradeStatus); +const mockTriggerUpgrade = vi.mocked(instanceUpdateClient.triggerUpgrade); +const TRACKED_OPERATION_KEY = "protoFleet:tracked-upgrade-operation"; +const ACKNOWLEDGED_OPERATION_KEY = "protoFleet:acknowledged-upgrade-operation"; + +type MessageOverrides = Omit, "$typeName" | "$unknown">; + +const operation = (phase: UpgradePhase, overrides?: MessageOverrides) => + create(UpgradeOperationSchema, { + id: "operation-1", + targetVersion: "v1.3.0", + phase, + message: "Preparing upgrade", + ...overrides, + }); + +const status = (executorAvailable = true, currentOperation?: UpgradeOperation) => + create(GetUpgradeStatusResponseSchema, { + executorAvailable, + operation: currentOperation, + }); + +const deferred = () => { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, reject, resolve }; +}; + +beforeEach(() => { + vi.clearAllMocks(); + window.sessionStorage.clear(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("useUpgradeOperation", () => { + it("recovers a durable active operation without a capability gate", async () => { + const activeOperation = operation(UpgradePhase.PREFLIGHT); + mockGetUpgradeStatus.mockResolvedValue(status(true, activeOperation)); + + const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + await waitFor(() => expect(result.current.operation?.id).toBe("operation-1")); + expect(result.current.connectionLost).toBe(false); + expect(mockGetUpgradeStatus).toHaveBeenCalledWith( + {}, + expect.objectContaining({ signal: expect.any(AbortSignal), timeoutMs: 10_000 }), + ); + }); + + it("does not overlap a pending poll and aborts it on cleanup", async () => { + vi.useFakeTimers(); + const request = deferred>(); + mockGetUpgradeStatus.mockReturnValue(request.promise); + + const hook = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await act(async () => Promise.resolve()); + expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1); + + await act(async () => vi.advanceTimersByTimeAsync(120_000)); + expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1); + const signal = mockGetUpgradeStatus.mock.calls[0]?.[1]?.signal; + + hook.unmount(); + expect(signal?.aborted).toBe(true); + }); + + it("starts the exact target and tracks the returned operation", async () => { + const activeOperation = operation(UpgradePhase.PREFLIGHT); + mockGetUpgradeStatus.mockResolvedValue(status()); + mockTriggerUpgrade.mockResolvedValue( + create(TriggerUpgradeResponseSchema, { + operation: activeOperation, + }), + ); + const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalled()); + await act(async () => result.current.triggerUpgrade("v1.3.0")); + + expect(mockTriggerUpgrade).toHaveBeenCalledWith({ targetVersion: "v1.3.0" }, { timeoutMs: 30_000 }); + expect(result.current.operation?.phase).toBe(UpgradePhase.PREFLIGHT); + expect(JSON.parse(window.sessionStorage.getItem(TRACKED_OPERATION_KEY) ?? "{}")).toEqual({ + id: "operation-1", + targetVersion: "v1.3.0", + }); + }); + + it("reconciles an ambiguous trigger rejection to the durable operation", async () => { + const activeOperation = operation(UpgradePhase.PREFLIGHT); + mockGetUpgradeStatus.mockResolvedValueOnce(status()).mockResolvedValue(status(true, activeOperation)); + mockTriggerUpgrade.mockRejectedValue(new Error("response lost")); + const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1)); + await act(async () => result.current.triggerUpgrade("v1.3.0")); + await waitFor(() => expect(result.current.operation?.id).toBe("operation-1")); + + expect(result.current.reconciling).toBe(false); + expect(result.current.triggerError).toBeNull(); + }); + + it("ends bounded reconciliation and preserves an actionable trigger error", async () => { + vi.useFakeTimers(); + mockGetUpgradeStatus.mockResolvedValue(status()); + mockTriggerUpgrade.mockRejectedValue(new Error("host did not confirm")); + const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await act(async () => Promise.resolve()); + + await act(async () => result.current.triggerUpgrade("v1.3.0")); + expect(result.current.reconciling).toBe(true); + + await act(async () => vi.advanceTimersByTimeAsync(17_000)); + + expect(result.current.reconciling).toBe(false); + expect(result.current.triggerError).toContain("host did not confirm"); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); + }); + + it("does not unlock an unknown trigger outcome while the executor is unreachable", async () => { + vi.useFakeTimers(); + mockGetUpgradeStatus.mockResolvedValueOnce(status()).mockResolvedValue(status(false)); + mockTriggerUpgrade.mockRejectedValue(new Error("host did not confirm")); + const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await act(async () => Promise.resolve()); + + await act(async () => result.current.triggerUpgrade("v1.3.0")); + await act(async () => vi.advanceTimersByTimeAsync(30_000)); + + expect(result.current.reconciling).toBe(true); + expect(result.current.connectionLost).toBe(true); + expect(result.current.manualFallbackReady).toBe(true); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).not.toBeNull(); + + act(() => result.current.useManualFallback()); + + expect(result.current.reconciling).toBe(false); + expect(result.current.triggerError).toBeNull(); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); + }); + + it("keeps progress through disconnect and accepts the terminal success", async () => { + vi.useFakeTimers(); + const activeOperation = operation(UpgradePhase.ACTIVATING); + const succeededOperation = operation(UpgradePhase.SUCCEEDED, { message: "Upgrade complete" }); + mockGetUpgradeStatus + .mockResolvedValueOnce(status(true, activeOperation)) + .mockRejectedValueOnce(new Error("Fleet restarting")) + .mockResolvedValue(status(true, succeededOperation)); + const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await act(async () => Promise.resolve()); + expect(result.current.operation?.phase).toBe(UpgradePhase.ACTIVATING); + + await act(async () => vi.advanceTimersByTimeAsync(2_000)); + expect(result.current.connectionLost).toBe(true); + expect(result.current.operation?.phase).toBe(UpgradePhase.ACTIVATING); + + await act(async () => vi.advanceTimersByTimeAsync(2_000)); + expect(result.current.operation?.phase).toBe(UpgradePhase.SUCCEEDED); + expect(result.current.connectionLost).toBe(false); + }); + + it("does not replay an untracked historical success", async () => { + mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.SUCCEEDED))); + const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.3.0" })); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalled()); + expect(result.current.operation).toBeUndefined(); + }); + + it("suppresses a stale failure after manual recovery installed its target", async () => { + window.sessionStorage.setItem( + TRACKED_OPERATION_KEY, + JSON.stringify({ id: "operation-1", targetVersion: "v1.3.0" }), + ); + mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.FAILED))); + const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.3.0" })); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalled()); + expect(result.current.operation).toBeUndefined(); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); + }); + + it("rechecks an unresolved failure as soon as the current version loads", async () => { + mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.FAILED))); + const initialProps: { currentVersion?: string } = {}; + const { rerender, result } = renderHook( + ({ currentVersion }: { currentVersion?: string }) => useUpgradeOperation({ enabled: true, currentVersion }), + { initialProps }, + ); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1)); + expect(result.current.operation).toBeUndefined(); + + rerender({ currentVersion: "v1.2.0" }); + + await waitFor(() => expect(result.current.operation?.phase).toBe(UpgradePhase.FAILED)); + }); + + it("acknowledges a terminal failure so it is not replayed", async () => { + mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.FAILED))); + const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await waitFor(() => expect(result.current.operation?.phase).toBe(UpgradePhase.FAILED)); + + act(() => result.current.acknowledgeOperation()); + + expect(result.current.operation).toBeUndefined(); + expect(window.sessionStorage.getItem(ACKNOWLEDGED_OPERATION_KEY)).toBe("operation-1"); + }); + + it("forwards poll errors for page-level auth handling", async () => { + const onPollError = vi.fn(); + const pollError = new Error("permission revoked"); + mockGetUpgradeStatus.mockRejectedValue(pollError); + + renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0", onPollError })); + + await waitFor(() => expect(onPollError).toHaveBeenCalledWith(pollError)); + }); +}); diff --git a/client/src/protoFleet/features/updates/api/useUpgradeOperation.ts b/client/src/protoFleet/features/updates/api/useUpgradeOperation.ts new file mode 100644 index 0000000000..b1a0ddcf63 --- /dev/null +++ b/client/src/protoFleet/features/updates/api/useUpgradeOperation.ts @@ -0,0 +1,371 @@ +import { useCallback, useEffect, useRef, useState } from "react"; + +import { instanceUpdateClient } from "@/protoFleet/api/clients"; +import { type UpgradeOperation, UpgradePhase } from "@/protoFleet/api/generated/instance/v1/updates_pb"; +import { getErrorMessage } from "@/protoFleet/api/getErrorMessage"; + +const ACTIVE_POLL_INTERVAL_MS = 2_000; +const IDLE_POLL_INTERVAL_MS = 60_000; +const STATUS_REQUEST_TIMEOUT_MS = 10_000; +const TRIGGER_REQUEST_TIMEOUT_MS = 30_000; +const TRIGGER_RECONCILIATION_TIMEOUT_MS = 15_000; +const TRACKED_OPERATION_KEY = "protoFleet:tracked-upgrade-operation"; +const ACKNOWLEDGED_OPERATION_KEY = "protoFleet:acknowledged-upgrade-operation"; + +interface TrackedOperation { + id?: string; + targetVersion: string; +} + +interface UseUpgradeOperationOptions { + currentVersion?: string; + enabled: boolean; + onPollError?: (error: unknown) => void; +} + +interface UseUpgradeOperationResult { + acknowledgeOperation: () => void; + connectionLost: boolean; + manualFallbackReady: boolean; + operation: UpgradeOperation | undefined; + reconciling: boolean; + reloadFleet: () => void; + triggerError: string | null; + triggering: boolean; + trackedTargetVersion: string | undefined; + triggerUpgrade: (targetVersion: string) => Promise; + useManualFallback: () => void; +} + +export const isUpgradeTerminal = (phase: UpgradePhase) => + phase === UpgradePhase.SUCCEEDED || phase === UpgradePhase.FAILED; + +export const isUpgradeActive = (operation?: UpgradeOperation) => + Boolean(operation && operation.phase !== UpgradePhase.UNSPECIFIED && !isUpgradeTerminal(operation.phase)); + +const readTrackedOperation = (): TrackedOperation | undefined => { + try { + const raw = window.sessionStorage.getItem(TRACKED_OPERATION_KEY); + if (!raw) return undefined; + const value = JSON.parse(raw) as Partial; + if (typeof value.targetVersion !== "string" || !value.targetVersion) { + window.sessionStorage.removeItem(TRACKED_OPERATION_KEY); + return undefined; + } + return { + targetVersion: value.targetVersion, + ...(typeof value.id === "string" && value.id ? { id: value.id } : {}), + }; + } catch { + return undefined; + } +}; + +const readAcknowledgedOperation = (): string | null => { + try { + return window.sessionStorage.getItem(ACKNOWLEDGED_OPERATION_KEY); + } catch { + return null; + } +}; + +export function useUpgradeOperation({ + currentVersion, + enabled, + onPollError, +}: UseUpgradeOperationOptions): UseUpgradeOperationResult { + const [operation, setOperation] = useState(); + const [triggering, setTriggering] = useState(false); + const [trackedOperation, setTrackedOperation] = useState(readTrackedOperation); + const recoveredUnknownOutcome = Boolean(trackedOperation && !trackedOperation.id); + const [reconciling, setReconciling] = useState(recoveredUnknownOutcome); + const [connectionLost, setConnectionLost] = useState(false); + const [manualFallbackReady, setManualFallbackReady] = useState(false); + const [triggerError, setTriggerError] = useState( + recoveredUnknownOutcome ? "Fleet did not confirm whether the previous upgrade request started" : null, + ); + const [pollRevision, setPollRevision] = useState(0); + const [acknowledgedOperation, setAcknowledgedOperation] = useState(readAcknowledgedOperation); + + const operationRef = useRef(operation); + const trackedOperationRef = useRef(trackedOperation); + const acknowledgedOperationRef = useRef(acknowledgedOperation); + const reconcilingRef = useRef(reconciling); + const currentVersionRef = useRef(currentVersion); + const onPollErrorRef = useRef(onPollError); + const reconciliationDeadlineRef = useRef(null); + + currentVersionRef.current = currentVersion; + onPollErrorRef.current = onPollError; + + useEffect(() => { + if (trackedOperationRef.current && !trackedOperationRef.current.id) { + reconciliationDeadlineRef.current = Date.now() + TRIGGER_RECONCILIATION_TIMEOUT_MS; + } + }, []); + + const updateOperation = useCallback((next: UpgradeOperation | undefined) => { + operationRef.current = next; + setOperation(next); + }, []); + + const updateReconciling = useCallback((next: boolean) => { + reconcilingRef.current = next; + setReconciling(next); + }, []); + + const updateTrackedOperation = useCallback((next: TrackedOperation | undefined) => { + trackedOperationRef.current = next; + setTrackedOperation(next); + try { + if (next) { + window.sessionStorage.setItem(TRACKED_OPERATION_KEY, JSON.stringify(next)); + } else { + window.sessionStorage.removeItem(TRACKED_OPERATION_KEY); + } + } catch { + // Browser storage is an optimization for route/reload recovery. The + // host operation remains durable when storage is unavailable. + } + }, []); + + const updateAcknowledgedOperation = useCallback((next: string | null) => { + acknowledgedOperationRef.current = next; + setAcknowledgedOperation(next); + try { + if (next) { + window.sessionStorage.setItem(ACKNOWLEDGED_OPERATION_KEY, next); + } else { + window.sessionStorage.removeItem(ACKNOWLEDGED_OPERATION_KEY); + } + } catch { + // See updateTrackedOperation: storage failure cannot affect host state. + } + }, []); + + const finishExpiredReconciliation = useCallback(() => { + const deadline = reconciliationDeadlineRef.current; + if (!reconcilingRef.current || deadline === null || Date.now() < deadline) { + return false; + } + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + if (!trackedOperationRef.current?.id) { + updateTrackedOperation(undefined); + } + return true; + }, [updateReconciling, updateTrackedOperation]); + + const allowManualFallbackAfterTimeout = useCallback(() => { + const deadline = reconciliationDeadlineRef.current; + if (reconcilingRef.current && deadline !== null && Date.now() >= deadline) { + setManualFallbackReady(true); + } + }, []); + + const acceptServerOperation = useCallback( + (next: UpgradeOperation) => { + const active = isUpgradeActive(next); + const tracked = trackedOperationRef.current; + const trackedMatches = tracked?.id ? tracked.id === next.id : tracked?.targetVersion === next.targetVersion; + + if (active) { + // A durable active operation is authoritative, including one started + // by another operator. It takes precedence over a newer release offer. + updateAcknowledgedOperation(null); + updateTrackedOperation({ id: next.id, targetVersion: next.targetVersion }); + updateOperation(next); + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + setConnectionLost(false); + setTriggerError(null); + return true; + } + + if (!isUpgradeTerminal(next.phase)) { + return false; + } + + if (acknowledgedOperationRef.current === next.id) { + return false; + } + + // A manual recovery may have installed the failed target successfully. + // Do not replay the updater's stale failure once Fleet reports that exact + // version as current. + if (next.phase === UpgradePhase.FAILED && currentVersionRef.current === next.targetVersion) { + if (trackedMatches) updateTrackedOperation(undefined); + if (operationRef.current?.id === next.id) updateOperation(undefined); + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + setConnectionLost(false); + return false; + } + + const unresolvedFailure = + next.phase === UpgradePhase.FAILED && + Boolean(currentVersionRef.current && currentVersionRef.current !== next.targetVersion); + if (!trackedMatches && !unresolvedFailure) { + return false; + } + + updateTrackedOperation({ id: next.id, targetVersion: next.targetVersion }); + updateOperation(next); + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + setConnectionLost(false); + setTriggerError(null); + return true; + }, + [updateAcknowledgedOperation, updateOperation, updateReconciling, updateTrackedOperation], + ); + + const pollStatus = useCallback( + async (signal: AbortSignal) => { + try { + const response = await instanceUpdateClient.getUpgradeStatus( + {}, + { signal, timeoutMs: STATUS_REQUEST_TIMEOUT_MS }, + ); + if (signal.aborted) return; + + if (response.operation && acceptServerOperation(response.operation)) { + return; + } + + if (!response.executorAvailable) { + if (isUpgradeActive(operationRef.current) || reconcilingRef.current || trackedOperationRef.current) { + setConnectionLost(true); + } + allowManualFallbackAfterTimeout(); + return; + } + + if (isUpgradeActive(operationRef.current)) { + // The executor is reachable but lost the operation that Fleet was + // following. Preserve the last durable phase rather than exposing a + // competing action while reconciliation continues. + setConnectionLost(true); + } else { + setConnectionLost(false); + } + finishExpiredReconciliation(); + } catch (error) { + if (signal.aborted) return; + onPollErrorRef.current?.(error); + if (isUpgradeActive(operationRef.current) || reconcilingRef.current || trackedOperationRef.current) { + setConnectionLost(true); + } + allowManualFallbackAfterTimeout(); + } + }, + [acceptServerOperation, allowManualFallbackAfterTimeout, finishExpiredReconciliation], + ); + + useEffect(() => { + if (!enabled) return; + + let alive = true; + let timer: number | undefined; + let controller: AbortController | undefined; + + const run = async () => { + controller = new AbortController(); + await pollStatus(controller.signal); + if (alive) { + const awaitingTrackedOperation = Boolean(trackedOperationRef.current && !operationRef.current); + const pollIntervalMs = + isUpgradeActive(operationRef.current) || reconcilingRef.current || awaitingTrackedOperation + ? ACTIVE_POLL_INTERVAL_MS + : IDLE_POLL_INTERVAL_MS; + timer = window.setTimeout(run, pollIntervalMs); + } + }; + + void run(); + return () => { + alive = false; + controller?.abort(); + if (timer !== undefined) window.clearTimeout(timer); + }; + }, [currentVersion, enabled, pollRevision, pollStatus]); + + const triggerUpgrade = useCallback( + async (targetVersion: string) => { + updateTrackedOperation({ targetVersion }); + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + setConnectionLost(false); + setTriggerError(null); + setTriggering(true); + try { + const response = await instanceUpdateClient.triggerUpgrade( + { targetVersion }, + { timeoutMs: TRIGGER_REQUEST_TIMEOUT_MS }, + ); + if (!response.operation) { + throw new Error("Host updater did not return an operation"); + } + acceptServerOperation(response.operation); + } catch (error) { + setTriggerError(getErrorMessage(error, "Failed to start upgrade")); + reconciliationDeadlineRef.current = Date.now() + TRIGGER_RECONCILIATION_TIMEOUT_MS; + updateReconciling(true); + } finally { + setTriggering(false); + // An idle status poll may already be scheduled far in the future. + // Wake it after trigger admission settles so success and ambiguous + // responses both transition immediately to the active cadence. + setPollRevision((revision) => revision + 1); + } + }, + [acceptServerOperation, updateReconciling, updateTrackedOperation], + ); + + const acknowledgeOperation = useCallback(() => { + if (operationRef.current && isUpgradeTerminal(operationRef.current.phase)) { + updateAcknowledgedOperation(operationRef.current.id); + } + updateTrackedOperation(undefined); + updateOperation(undefined); + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + setConnectionLost(false); + setTriggerError(null); + }, [updateAcknowledgedOperation, updateOperation, updateReconciling, updateTrackedOperation]); + + const useManualFallback = useCallback(() => { + if (!manualFallbackReady) return; + updateTrackedOperation(undefined); + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + setConnectionLost(false); + setTriggerError(null); + }, [manualFallbackReady, updateReconciling, updateTrackedOperation]); + + const reloadFleet = useCallback(() => { + updateTrackedOperation(undefined); + window.location.reload(); + }, [updateTrackedOperation]); + + return { + acknowledgeOperation, + connectionLost, + manualFallbackReady, + operation, + reconciling, + reloadFleet, + triggerError, + triggering, + trackedTargetVersion: trackedOperation?.targetVersion, + triggerUpgrade, + useManualFallback, + }; +} diff --git a/client/src/protoFleet/features/updates/copyInstallCommand.ts b/client/src/protoFleet/features/updates/copyInstallCommand.ts index 362bffc2bd..1292456180 100644 --- a/client/src/protoFleet/features/updates/copyInstallCommand.ts +++ b/client/src/protoFleet/features/updates/copyInstallCommand.ts @@ -1,8 +1,8 @@ import { pushToast, STATUSES } from "@/shared/features/toaster"; import { copyToClipboard } from "@/shared/utils/utility"; -// Shared by the update notification modal and the settings Updates page so -// both surfaces give identical feedback for the same action. +// Keep the manual fallback's clipboard feedback consistent wherever the +// Settings update flow renders it. export const copyInstallCommand = (installCommand: string) => { copyToClipboard(installCommand) .then(() => { diff --git a/docs/plans/archive/2026-07-29-one-click-upgrade-executor-plan.md b/docs/plans/archive/2026-07-29-one-click-upgrade-executor-plan.md new file mode 100644 index 0000000000..6f84ba91c5 --- /dev/null +++ b/docs/plans/archive/2026-07-29-one-click-upgrade-executor-plan.md @@ -0,0 +1,115 @@ +--- +title: One-Click Upgrade Host Executor - Plan +date: 2026-07-29 +status: completed +type: plan +topic: one-click-upgrade +artifact_contract: ce-unified-plan/v1 +artifact_readiness: implementation-ready +--- + +# One-Click Upgrade Host Executor + +## Outcome + +Proto Fleet now lets an authorized operator confirm an eligible upgrade in the +Updates settings route. A privileged process outside the Compose stack +validates and activates the release, while the client follows durable progress +through the expected Fleet restart. Hosts without a reachable executor retain +the release-specific manual install command as the authoritative fallback. + +## Final architecture + +### Client ownership + +- The normal shell owns only a passive version pill. It performs lightweight + release discovery and routes the operator to `/settings/updates`; it does not + retain installer commands or own upgrade progress globally. +- The Updates route fetches authoritative status before exposing actions. It + shows one-click controls only when Fleet reports a reachable host executor, + and otherwise keeps the manual command path intact. +- Confirmation names the exact target and explains the restart window. Release + candidates add the forward-only migration and no-downgrade warning. +- Confirmation, progress, reconnect, terminal error, recovery, and success are + route-owned states. Leaving the route stops only browser polling; it does not + cancel the durable host operation. Returning to the route recovers current + state through the upgrade-status RPC. +- An ambiguous trigger keeps conflicting controls locked while the executor is + unreachable. After a bounded wait, the manual path can be unlocked only by + explicitly confirming on-host that no upgrade is still running. + +### Privilege and execution boundary + +- `fleetd` re-checks `instance:update`, the selected release channel, and the + currently eligible target. The browser cannot provide a download URL, + command, downgrade, or arbitrary release tag. +- The root systemd service runs outside the Compose stack it restarts. Fleet + reaches its narrow HTTP API through + `/run/proto-fleet-updater/updater.sock`; the application container never + receives the host Docker socket. +- Before teardown, the updater downloads and verifies the release, safely + extracts it, preserves deployment configuration, builds release-specific + images, and completes preflight. Activation revalidates the staged manifest + and image identities before switching the deployment. +- The supported topology is one Proto Fleet installation per host on Linux + with systemd and rootful Docker, including WSL distributions configured with + systemd. macOS, rootless Docker, Linux without systemd, and alternate + multi-install layouts use the manual flow. + +### Trust and recovery boundary + +- The SHA-256 sidecar detects corruption and binds the expected asset name to + its digest. The bundle and sidecar share a GitHub Release origin, so GitHub + remains the publisher trust anchor; independent release signing is outside + this phase. +- Runtime configuration (`.env`, TLS material, Influx configuration, and + persisted optional-overlay settings) is carried into the staged deployment. +- Updater state and per-operation logs survive Fleet restarts at + `/var/lib/proto-fleet-updater/state.json` and + `/var/lib/proto-fleet-updater/logs/.log`. +- A terminal failure exposes the host log path and an explicit recovery + command. The previous deployment remains at + `/deployment.previous` for inspection, but automatic binary + rollback is disabled because migrations are forward-only. + +## End-to-end flow + +```mermaid +sequenceDiagram + participant U as "Authorized operator" + participant S as "Fleet shell" + participant R as "Updates route" + participant F as "fleetd" + participant X as "Host updater (systemd)" + participant G as "GitHub Releases" + + S->>F: "Discover eligible version" + F-->>S: "Version-only indicator data" + U->>S: "Select update pill" + S->>R: "Navigate to /settings/updates" + R->>F: "GetUpdateStatus" + F-->>R: "Eligible release, capability, manual command" + U->>R: "Confirm exact target" + R->>F: "TriggerUpgrade(target version)" + F->>F: "Re-check permission, channel, and eligibility" + F->>X: "Start operation over Unix socket" + X->>G: "Download bundle and checksum" + X->>X: "Verify, stage, preflight, and activate" + R-->>F: "Poll durable status; restart disconnect is expected" + opt "Operator leaves and later returns" + U->>R: "Open /settings/updates" + R->>F: "GetUpgradeStatus" + end + F-->>R: "Active or terminal durable operation" + R-->>U: "Progress, reload, or recovery guidance" +``` + +## Completed implementation units + +1. Durable single-flight updater manager and Unix-socket API. +2. Packaged updater binary, hardened systemd service, and installer lifecycle. +3. Non-interactive preflight and activation with immutable staging checks. +4. Release checksum publication and bounded download/extraction validation. +5. Permission-gated trigger/status RPCs with server-side target validation. +6. Passive shell discovery plus route-owned confirmation and operation states. +7. Focused host, API, installer, and client lifecycle tests. From 789da4caf243f2726a788d66f213c2a91f6c1e16 Mon Sep 17 00:00:00 2001 From: Marvin Charles Date: Sat, 8 Aug 2026 09:23:59 +0300 Subject: [PATCH 02/13] fix(updates): harden upgrade recovery state --- .../settings/components/Updates.test.tsx | 29 ++++- .../features/settings/components/Updates.tsx | 27 +++- .../components/UpgradeOperationModal.test.tsx | 2 +- .../components/UpgradeOperationModal.tsx | 8 +- .../updates/api/useUpgradeOperation.test.tsx | 120 +++++++++++++++--- .../updates/api/useUpgradeOperation.ts | 73 ++++++++++- 6 files changed, 218 insertions(+), 41 deletions(-) diff --git a/client/src/protoFleet/features/settings/components/Updates.test.tsx b/client/src/protoFleet/features/settings/components/Updates.test.tsx index d84627f967..9952e7583e 100644 --- a/client/src/protoFleet/features/settings/components/Updates.test.tsx +++ b/client/src/protoFleet/features/settings/components/Updates.test.tsx @@ -29,6 +29,7 @@ const permissionsMock = vi.hoisted(() => ({ isAuthenticated: true, sessionExpiry: new Date(1_000), setPermissions: vi.fn<(permissions: string[]) => void>(), + username: "operator-a", })); const authErrorsMock = vi.hoisted(() => ({ handleAuthErrors: vi.fn(), @@ -63,7 +64,9 @@ vi.mock("@/protoFleet/store", () => { return { useHasPermission: vi.fn((permission: string) => permissionsMock.current.includes(permission)), usePermissions: () => permissionsMock.current, + useSessionExpiry: () => permissionsMock.sessionExpiry, useSetPermissions: () => permissionsMock.setPermissions, + useUsername: () => permissionsMock.username, useAuthErrors: () => authErrorsMock, useFleetStore: { getState: () => ({ @@ -84,11 +87,13 @@ vi.mock("@/protoFleet/api/clients", () => ({ }, })); -vi.mock("@/protoFleet/features/updates/api/useUpgradeOperation", () => ({ - isUpgradeActive: (operation?: { phase: number }) => - Boolean(operation && operation.phase !== 0 && operation.phase !== 7 && operation.phase !== 8), - useUpgradeOperation: vi.fn(() => upgradeHookMock.current), -})); +vi.mock("@/protoFleet/features/updates/api/useUpgradeOperation", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + useUpgradeOperation: vi.fn(() => upgradeHookMock.current), + }; +}); vi.mock("@/shared/utils/utility", () => ({ copyToClipboard: vi.fn(), @@ -178,6 +183,7 @@ beforeEach(() => { permissionsMock.current = ["instance:update", "fleet:read"]; permissionsMock.isAuthenticated = true; permissionsMock.sessionExpiry = new Date(1_000); + permissionsMock.username = "operator-a"; permissionsMock.setPermissions.mockImplementation((permissions) => { permissionsMock.current = permissions; }); @@ -287,7 +293,18 @@ describe("Updates", () => { const page = render(); - expect(await page.findByText(/checking whether the upgrade started/i)).toBeInTheDocument(); + expect(await page.findByText(/checking upgrade status/i)).toBeInTheDocument(); + expect(page.getByRole("button", { name: "Copy install command" })).toBeDisabled(); + expect(page.getByRole("checkbox", { name: RC_CHECKBOX_NAME })).toBeDisabled(); + }); + + it("locks competing controls whenever a persisted operation remains unresolved", async () => { + upgradeHookMock.current.trackedTargetVersion = "v1.3.0"; + mockGetUpdateStatus.mockResolvedValue(buildStatus({ oneClickAvailable: true })); + + const page = render(); + + expect(await page.findByRole("button", { name: "Upgrade to v1.3.0" })).toBeDisabled(); expect(page.getByRole("button", { name: "Copy install command" })).toBeDisabled(); expect(page.getByRole("checkbox", { name: RC_CHECKBOX_NAME })).toBeDisabled(); }); diff --git a/client/src/protoFleet/features/settings/components/Updates.tsx b/client/src/protoFleet/features/settings/components/Updates.tsx index 844ede6fd4..a8052b9104 100644 --- a/client/src/protoFleet/features/settings/components/Updates.tsx +++ b/client/src/protoFleet/features/settings/components/Updates.tsx @@ -11,7 +11,15 @@ import SettingsPageHeader from "@/protoFleet/features/settings/components/Settin import UpgradeOperationModal from "@/protoFleet/features/settings/components/UpgradeOperationModal"; import { isUpgradeActive, useUpgradeOperation } from "@/protoFleet/features/updates/api/useUpgradeOperation"; import { copyInstallCommand } from "@/protoFleet/features/updates/copyInstallCommand"; -import { useAuthErrors, useFleetStore, useHasPermission, usePermissions, useSetPermissions } from "@/protoFleet/store"; +import { + useAuthErrors, + useFleetStore, + useHasPermission, + usePermissions, + useSessionExpiry, + useSetPermissions, + useUsername, +} from "@/protoFleet/store"; import { Copy } from "@/shared/assets/icons"; import Button, { variants } from "@/shared/components/Button"; import Checkbox from "@/shared/components/Checkbox"; @@ -87,7 +95,9 @@ const waitForReleaseChannelSave = async () => { const Updates = () => { const canUpdateInstance = useHasPermission(INSTANCE_UPDATE_PERMISSION); const permissions = usePermissions(); + const sessionExpiry = useSessionExpiry(); const setPermissions = useSetPermissions(); + const username = useUsername(); const { handleAuthErrors } = useAuthErrors(); const [status, setStatus] = useState(null); const [loadError, setLoadError] = useState(null); @@ -133,6 +143,7 @@ const Updates = () => { ); const upgrade = useUpgradeOperation({ + authSessionIdentity: `${username}:${sessionExpiry?.getTime() ?? "signed-out"}`, enabled: canUpdateInstance, currentVersion: status?.currentVersion, onPollError: handleUpgradePollError, @@ -140,9 +151,15 @@ const Updates = () => { const activeUpgrade = isUpgradeActive(upgrade.operation); const succeededUpgrade = upgrade.operation?.phase === UpgradePhase.SUCCEEDED; const upgradeRequestPending = upgrade.triggering || upgrade.reconciling; - const upgradeLocksConfiguration = upgradeRequestPending || Boolean(upgrade.operation); + const unresolvedTrackedUpgrade = Boolean(upgrade.trackedTargetVersion && !upgrade.operation); + const upgradeLocksConfiguration = upgradeRequestPending || unresolvedTrackedUpgrade || Boolean(upgrade.operation); + const upgradeActionDisabled = isChannelChangePending || upgradeRequestPending || unresolvedTrackedUpgrade; const manualCommandDisabled = - isChannelChangePending || activeUpgrade || upgradeRequestPending || Boolean(succeededUpgrade); + isChannelChangePending || + activeUpgrade || + upgradeRequestPending || + unresolvedTrackedUpgrade || + Boolean(succeededUpgrade); const fetchStatus = useCallback(async () => { const requestId = ++latestStatusRequest.current; @@ -310,7 +327,7 @@ const Updates = () => { const operationStatusLabel = upgrade.reconciling ? upgrade.manualFallbackReady ? "Upgrade outcome is unknown — host confirmation required" - : "Confirming whether the upgrade started" + : "Confirming upgrade status" : upgrade.operation?.phase === UpgradePhase.FAILED ? "Upgrade failed" : upgrade.operation?.phase === UpgradePhase.SUCCEEDED @@ -405,7 +422,7 @@ const Updates = () => {
{manualFallbackReady ? ( @@ -168,8 +166,8 @@ const UpgradeOperationModal = ({

) : (

- Fleet is reconciling the request with the host updater. Do not run the manual install command yet; wait - until this check finishes. + Fleet is reconciling the tracked upgrade with the host updater. Do not run the manual install command + yet; wait until this check finishes.

)} diff --git a/client/src/protoFleet/features/updates/api/useUpgradeOperation.test.tsx b/client/src/protoFleet/features/updates/api/useUpgradeOperation.test.tsx index d191b271f8..a4dee190d7 100644 --- a/client/src/protoFleet/features/updates/api/useUpgradeOperation.test.tsx +++ b/client/src/protoFleet/features/updates/api/useUpgradeOperation.test.tsx @@ -23,6 +23,12 @@ const mockGetUpgradeStatus = vi.mocked(instanceUpdateClient.getUpgradeStatus); const mockTriggerUpgrade = vi.mocked(instanceUpdateClient.triggerUpgrade); const TRACKED_OPERATION_KEY = "protoFleet:tracked-upgrade-operation"; const ACKNOWLEDGED_OPERATION_KEY = "protoFleet:acknowledged-upgrade-operation"; +const AUTH_SESSION_IDENTITY = "operator-a:1000"; + +const useTestUpgradeOperation = ( + options: Omit[0], "authSessionIdentity">, + authSessionIdentity = AUTH_SESSION_IDENTITY, +) => useUpgradeOperation({ authSessionIdentity, ...options }); type MessageOverrides = Omit, "$typeName" | "$unknown">; @@ -63,11 +69,17 @@ afterEach(() => { describe("useUpgradeOperation", () => { it("recovers a durable active operation without a capability gate", async () => { const activeOperation = operation(UpgradePhase.PREFLIGHT); + window.sessionStorage.setItem( + TRACKED_OPERATION_KEY, + JSON.stringify({ id: activeOperation.id, targetVersion: activeOperation.targetVersion }), + ); mockGetUpgradeStatus.mockResolvedValue(status(true, activeOperation)); - const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + expect(result.current.reconciling).toBe(true); await waitFor(() => expect(result.current.operation?.id).toBe("operation-1")); + expect(result.current.reconciling).toBe(false); expect(result.current.connectionLost).toBe(false); expect(mockGetUpgradeStatus).toHaveBeenCalledWith( {}, @@ -75,12 +87,21 @@ describe("useUpgradeOperation", () => { ); }); + it("removes a malformed tracked-operation record", () => { + window.sessionStorage.setItem(TRACKED_OPERATION_KEY, "{not-json"); + mockGetUpgradeStatus.mockResolvedValue(status()); + + renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); + }); + it("does not overlap a pending poll and aborts it on cleanup", async () => { vi.useFakeTimers(); const request = deferred>(); mockGetUpgradeStatus.mockReturnValue(request.promise); - const hook = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + const hook = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); await act(async () => Promise.resolve()); expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1); @@ -100,7 +121,7 @@ describe("useUpgradeOperation", () => { operation: activeOperation, }), ); - const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalled()); await act(async () => result.current.triggerUpgrade("v1.3.0")); @@ -117,7 +138,7 @@ describe("useUpgradeOperation", () => { const activeOperation = operation(UpgradePhase.PREFLIGHT); mockGetUpgradeStatus.mockResolvedValueOnce(status()).mockResolvedValue(status(true, activeOperation)); mockTriggerUpgrade.mockRejectedValue(new Error("response lost")); - const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1)); await act(async () => result.current.triggerUpgrade("v1.3.0")); @@ -131,7 +152,7 @@ describe("useUpgradeOperation", () => { vi.useFakeTimers(); mockGetUpgradeStatus.mockResolvedValue(status()); mockTriggerUpgrade.mockRejectedValue(new Error("host did not confirm")); - const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); await act(async () => Promise.resolve()); await act(async () => result.current.triggerUpgrade("v1.3.0")); @@ -148,7 +169,7 @@ describe("useUpgradeOperation", () => { vi.useFakeTimers(); mockGetUpgradeStatus.mockResolvedValueOnce(status()).mockResolvedValue(status(false)); mockTriggerUpgrade.mockRejectedValue(new Error("host did not confirm")); - const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); await act(async () => Promise.resolve()); await act(async () => result.current.triggerUpgrade("v1.3.0")); @@ -166,6 +187,47 @@ describe("useUpgradeOperation", () => { expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); }); + it("keeps a recovered operation ID locked until the unreachable host is explicitly confirmed", async () => { + vi.useFakeTimers(); + window.sessionStorage.setItem( + TRACKED_OPERATION_KEY, + JSON.stringify({ id: "operation-1", targetVersion: "v1.3.0" }), + ); + mockGetUpgradeStatus.mockResolvedValue(status(false)); + + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + expect(result.current.reconciling).toBe(true); + + await act(async () => vi.advanceTimersByTimeAsync(17_000)); + + expect(result.current.reconciling).toBe(true); + expect(result.current.connectionLost).toBe(true); + expect(result.current.manualFallbackReady).toBe(true); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).not.toBeNull(); + + act(() => result.current.useManualFallback()); + + expect(result.current.reconciling).toBe(false); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); + }); + + it("unlocks a recovered operation ID after an authoritative reachable miss", async () => { + window.sessionStorage.setItem( + TRACKED_OPERATION_KEY, + JSON.stringify({ id: "operation-1", targetVersion: "v1.3.0" }), + ); + mockGetUpgradeStatus.mockResolvedValue(status(true)); + + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + expect(result.current.reconciling).toBe(true); + + await waitFor(() => expect(result.current.reconciling).toBe(false)); + + expect(result.current.connectionLost).toBe(false); + expect(result.current.manualFallbackReady).toBe(false); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); + }); + it("keeps progress through disconnect and accepts the terminal success", async () => { vi.useFakeTimers(); const activeOperation = operation(UpgradePhase.ACTIVATING); @@ -174,7 +236,7 @@ describe("useUpgradeOperation", () => { .mockResolvedValueOnce(status(true, activeOperation)) .mockRejectedValueOnce(new Error("Fleet restarting")) .mockResolvedValue(status(true, succeededOperation)); - const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); await act(async () => Promise.resolve()); expect(result.current.operation?.phase).toBe(UpgradePhase.ACTIVATING); @@ -189,7 +251,7 @@ describe("useUpgradeOperation", () => { it("does not replay an untracked historical success", async () => { mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.SUCCEEDED))); - const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.3.0" })); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.3.0" })); await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalled()); expect(result.current.operation).toBeUndefined(); @@ -201,7 +263,7 @@ describe("useUpgradeOperation", () => { JSON.stringify({ id: "operation-1", targetVersion: "v1.3.0" }), ); mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.FAILED))); - const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.3.0" })); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.3.0" })); await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalled()); expect(result.current.operation).toBeUndefined(); @@ -212,7 +274,7 @@ describe("useUpgradeOperation", () => { mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.FAILED))); const initialProps: { currentVersion?: string } = {}; const { rerender, result } = renderHook( - ({ currentVersion }: { currentVersion?: string }) => useUpgradeOperation({ enabled: true, currentVersion }), + ({ currentVersion }: { currentVersion?: string }) => useTestUpgradeOperation({ enabled: true, currentVersion }), { initialProps }, ); @@ -224,15 +286,33 @@ describe("useUpgradeOperation", () => { await waitFor(() => expect(result.current.operation?.phase).toBe(UpgradePhase.FAILED)); }); - it("acknowledges a terminal failure so it is not replayed", async () => { + it("scopes an acknowledged terminal failure to the authenticated session", async () => { mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.FAILED))); - const { result } = renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); - await waitFor(() => expect(result.current.operation?.phase).toBe(UpgradePhase.FAILED)); + const firstSession = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await waitFor(() => expect(firstSession.result.current.operation?.phase).toBe(UpgradePhase.FAILED)); - act(() => result.current.acknowledgeOperation()); + act(() => firstSession.result.current.acknowledgeOperation()); - expect(result.current.operation).toBeUndefined(); - expect(window.sessionStorage.getItem(ACKNOWLEDGED_OPERATION_KEY)).toBe("operation-1"); + expect(firstSession.result.current.operation).toBeUndefined(); + expect(JSON.parse(window.sessionStorage.getItem(ACKNOWLEDGED_OPERATION_KEY) ?? "{}")).toEqual({ + authSessionIdentity: AUTH_SESSION_IDENTITY, + id: "operation-1", + }); + firstSession.unmount(); + + const sameSession = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(2)); + expect(sameSession.result.current.operation).toBeUndefined(); + sameSession.unmount(); + + const nextSession = renderHook(() => + useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" }, "operator-b:2000"), + ); + await waitFor(() => expect(nextSession.result.current.operation?.phase).toBe(UpgradePhase.FAILED)); + expect(JSON.parse(window.sessionStorage.getItem(ACKNOWLEDGED_OPERATION_KEY) ?? "{}")).toEqual({ + authSessionIdentity: AUTH_SESSION_IDENTITY, + id: "operation-1", + }); }); it("forwards poll errors for page-level auth handling", async () => { @@ -240,7 +320,13 @@ describe("useUpgradeOperation", () => { const pollError = new Error("permission revoked"); mockGetUpgradeStatus.mockRejectedValue(pollError); - renderHook(() => useUpgradeOperation({ enabled: true, currentVersion: "v1.2.0", onPollError })); + renderHook(() => + useTestUpgradeOperation({ + enabled: true, + currentVersion: "v1.2.0", + onPollError, + }), + ); await waitFor(() => expect(onPollError).toHaveBeenCalledWith(pollError)); }); diff --git a/client/src/protoFleet/features/updates/api/useUpgradeOperation.ts b/client/src/protoFleet/features/updates/api/useUpgradeOperation.ts index b1a0ddcf63..9c23caf3ad 100644 --- a/client/src/protoFleet/features/updates/api/useUpgradeOperation.ts +++ b/client/src/protoFleet/features/updates/api/useUpgradeOperation.ts @@ -17,7 +17,13 @@ interface TrackedOperation { targetVersion: string; } +interface AcknowledgedOperation { + authSessionIdentity: string; + id: string; +} + interface UseUpgradeOperationOptions { + authSessionIdentity: string; currentVersion?: string; enabled: boolean; onPollError?: (error: unknown) => void; @@ -57,19 +63,36 @@ const readTrackedOperation = (): TrackedOperation | undefined => { ...(typeof value.id === "string" && value.id ? { id: value.id } : {}), }; } catch { + try { + window.sessionStorage.removeItem(TRACKED_OPERATION_KEY); + } catch { + // Storage is best-effort; the host remains authoritative. + } return undefined; } }; -const readAcknowledgedOperation = (): string | null => { +const readAcknowledgedOperation = (authSessionIdentity: string): string | null => { try { - return window.sessionStorage.getItem(ACKNOWLEDGED_OPERATION_KEY); + const raw = window.sessionStorage.getItem(ACKNOWLEDGED_OPERATION_KEY); + if (!raw) return null; + const value = JSON.parse(raw) as Partial; + if (typeof value.id === "string" && value.id && value.authSessionIdentity === authSessionIdentity) { + return value.id; + } + return null; } catch { + try { + window.sessionStorage.removeItem(ACKNOWLEDGED_OPERATION_KEY); + } catch { + // Storage is best-effort; the host remains authoritative. + } return null; } }; export function useUpgradeOperation({ + authSessionIdentity, currentVersion, enabled, onPollError, @@ -77,15 +100,18 @@ export function useUpgradeOperation({ const [operation, setOperation] = useState(); const [triggering, setTriggering] = useState(false); const [trackedOperation, setTrackedOperation] = useState(readTrackedOperation); + const recoveredTrackedOperation = Boolean(trackedOperation); const recoveredUnknownOutcome = Boolean(trackedOperation && !trackedOperation.id); - const [reconciling, setReconciling] = useState(recoveredUnknownOutcome); + const [reconciling, setReconciling] = useState(recoveredTrackedOperation); const [connectionLost, setConnectionLost] = useState(false); const [manualFallbackReady, setManualFallbackReady] = useState(false); const [triggerError, setTriggerError] = useState( recoveredUnknownOutcome ? "Fleet did not confirm whether the previous upgrade request started" : null, ); const [pollRevision, setPollRevision] = useState(0); - const [acknowledgedOperation, setAcknowledgedOperation] = useState(readAcknowledgedOperation); + const [acknowledgedOperation, setAcknowledgedOperation] = useState(() => + readAcknowledgedOperation(authSessionIdentity), + ); const operationRef = useRef(operation); const trackedOperationRef = useRef(trackedOperation); @@ -94,16 +120,25 @@ export function useUpgradeOperation({ const currentVersionRef = useRef(currentVersion); const onPollErrorRef = useRef(onPollError); const reconciliationDeadlineRef = useRef(null); + const authSessionIdentityRef = useRef(authSessionIdentity); currentVersionRef.current = currentVersion; onPollErrorRef.current = onPollError; useEffect(() => { - if (trackedOperationRef.current && !trackedOperationRef.current.id) { + if (trackedOperationRef.current) { reconciliationDeadlineRef.current = Date.now() + TRIGGER_RECONCILIATION_TIMEOUT_MS; } }, []); + useEffect(() => { + if (authSessionIdentityRef.current === authSessionIdentity) return; + authSessionIdentityRef.current = authSessionIdentity; + const next = readAcknowledgedOperation(authSessionIdentity); + acknowledgedOperationRef.current = next; + setAcknowledgedOperation(next); + }, [authSessionIdentity]); + const updateOperation = useCallback((next: UpgradeOperation | undefined) => { operationRef.current = next; setOperation(next); @@ -134,7 +169,10 @@ export function useUpgradeOperation({ setAcknowledgedOperation(next); try { if (next) { - window.sessionStorage.setItem(ACKNOWLEDGED_OPERATION_KEY, next); + window.sessionStorage.setItem( + ACKNOWLEDGED_OPERATION_KEY, + JSON.stringify({ authSessionIdentity: authSessionIdentityRef.current, id: next }), + ); } else { window.sessionStorage.removeItem(ACKNOWLEDGED_OPERATION_KEY); } @@ -164,6 +202,19 @@ export function useUpgradeOperation({ } }, []); + const resolveRecoveredOperationMiss = useCallback(() => { + if (!reconcilingRef.current || !trackedOperationRef.current?.id) { + return false; + } + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateTrackedOperation(undefined); + updateReconciling(false); + setConnectionLost(false); + setTriggerError(null); + return true; + }, [updateReconciling, updateTrackedOperation]); + const acceptServerOperation = useCallback( (next: UpgradeOperation) => { const active = isUpgradeActive(next); @@ -253,6 +304,9 @@ export function useUpgradeOperation({ } else { setConnectionLost(false); } + if (resolveRecoveredOperationMiss()) { + return; + } finishExpiredReconciliation(); } catch (error) { if (signal.aborted) return; @@ -263,7 +317,12 @@ export function useUpgradeOperation({ allowManualFallbackAfterTimeout(); } }, - [acceptServerOperation, allowManualFallbackAfterTimeout, finishExpiredReconciliation], + [ + acceptServerOperation, + allowManualFallbackAfterTimeout, + finishExpiredReconciliation, + resolveRecoveredOperationMiss, + ], ); useEffect(() => { From cfab18509ed6aeadc18227aeaa630daf5b0cee9b Mon Sep 17 00:00:00 2001 From: Marvin Charles Date: Sat, 8 Aug 2026 09:53:59 +0300 Subject: [PATCH 03/13] fix(updates): bound lost-operation recovery --- .../settings/components/Updates.test.tsx | 11 +++ .../features/settings/components/Updates.tsx | 3 +- .../components/UpgradeOperationModal.tsx | 4 +- .../updates/api/useUpgradeOperation.test.tsx | 79 ++++++++++++++++++- .../updates/api/useUpgradeOperation.ts | 47 ++++++++--- 5 files changed, 127 insertions(+), 17 deletions(-) diff --git a/client/src/protoFleet/features/settings/components/Updates.test.tsx b/client/src/protoFleet/features/settings/components/Updates.test.tsx index 9952e7583e..8dfbf1c3df 100644 --- a/client/src/protoFleet/features/settings/components/Updates.test.tsx +++ b/client/src/protoFleet/features/settings/components/Updates.test.tsx @@ -296,6 +296,12 @@ describe("Updates", () => { expect(await page.findByText(/checking upgrade status/i)).toBeInTheDocument(); expect(page.getByRole("button", { name: "Copy install command" })).toBeDisabled(); expect(page.getByRole("checkbox", { name: RC_CHECKBOX_NAME })).toBeDisabled(); + + fireEvent.click(page.getByRole("button", { name: "Close dialog" })); + const detailsButton = page.getByRole("button", { name: "View upgrade details" }); + expect(detailsButton).toBeEnabled(); + fireEvent.click(detailsButton); + expect(page.getByTestId("upgrade-operation-modal")).toBeInTheDocument(); }); it("locks competing controls whenever a persisted operation remains unresolved", async () => { @@ -431,6 +437,11 @@ describe("Updates", () => { expect(await findByText("Unable to load update status")).toBeInTheDocument(); expect(getByText("release registry unreachable")).toBeInTheDocument(); + await waitFor(() => + expect(mockUseUpgradeOperation).toHaveBeenLastCalledWith( + expect.objectContaining({ currentVersionUnavailable: true }), + ), + ); }); it("saves a channel change and toasts success", async () => { diff --git a/client/src/protoFleet/features/settings/components/Updates.tsx b/client/src/protoFleet/features/settings/components/Updates.tsx index a8052b9104..19b1c03af2 100644 --- a/client/src/protoFleet/features/settings/components/Updates.tsx +++ b/client/src/protoFleet/features/settings/components/Updates.tsx @@ -146,6 +146,7 @@ const Updates = () => { authSessionIdentity: `${username}:${sessionExpiry?.getTime() ?? "signed-out"}`, enabled: canUpdateInstance, currentVersion: status?.currentVersion, + currentVersionUnavailable: Boolean(loadError && !status), onPollError: handleUpgradePollError, }); const activeUpgrade = isUpgradeActive(upgrade.operation); @@ -422,7 +423,7 @@ const Updates = () => {