diff --git a/client/src/protoFleet/features/settings/components/Updates.test.tsx b/client/src/protoFleet/features/settings/components/Updates.test.tsx index 2e5d31565a..e7a798b694 100644 --- a/client/src/protoFleet/features/settings/components/Updates.test.tsx +++ b/client/src/protoFleet/features/settings/components/Updates.test.tsx @@ -1,4 +1,4 @@ -import { act, fireEvent, render, waitFor } from "@testing-library/react"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { create } from "@bufbuild/protobuf"; import { Code, ConnectError } from "@connectrpc/connect"; @@ -9,13 +9,17 @@ import type { GetUpdateStatusResponse, ReleaseInfo, SetReleaseChannelResponse, + UpgradeOperation, } from "@/protoFleet/api/generated/instance/v1/updates_pb"; import { GetUpdateStatusResponseSchema, ReleaseChannel, ReleaseInfoSchema, SetReleaseChannelResponseSchema, + UpgradeOperationSchema, + UpgradePhase, } from "@/protoFleet/api/generated/instance/v1/updates_pb"; +import { useUpgradeOperation } from "@/protoFleet/features/updates/api/useUpgradeOperation"; import { useHasPermission } from "@/protoFleet/store"; import { pushToast } from "@/shared/features/toaster"; import { copyToClipboard } from "@/shared/utils/utility"; @@ -24,11 +28,30 @@ const permissionsMock = vi.hoisted(() => ({ current: ["instance:update", "fleet:read"], isAuthenticated: true, sessionExpiry: new Date(1_000), + sessionGeneration: 1, setPermissions: vi.fn<(permissions: string[]) => void>(), + username: "operator-a", })); const authErrorsMock = vi.hoisted(() => ({ handleAuthErrors: vi.fn(), })); +interface UpgradeHookMockState { + acknowledgeOperation: ReturnType; + connectionLost: boolean; + manualFallbackReady: boolean; + operation?: UpgradeOperation; + operationStatusPending: boolean; + reconciling: boolean; + reloadFleet: ReturnType; + triggerError: string | null; + triggering: boolean; + trackedTargetVersion?: string; + triggerUpgrade: ReturnType; + useManualFallback: ReturnType; +} +const upgradeHookMock = vi.hoisted(() => ({ + current: {} as UpgradeHookMockState, +})); vi.mock("react-router-dom", () => ({ Navigate: ({ to }: { to: string }) =>
, @@ -43,7 +66,10 @@ vi.mock("@/protoFleet/store", () => { return { useHasPermission: vi.fn((permission: string) => permissionsMock.current.includes(permission)), usePermissions: () => permissionsMock.current, + useSessionExpiry: () => permissionsMock.sessionExpiry, + useSessionGeneration: () => permissionsMock.sessionGeneration, useSetPermissions: () => permissionsMock.setPermissions, + useUsername: () => permissionsMock.username, useAuthErrors: () => authErrorsMock, useFleetStore: { getState: () => ({ @@ -51,6 +77,8 @@ vi.mock("@/protoFleet/store", () => { isAuthenticated: permissionsMock.isAuthenticated, permissions: permissionsMock.current, sessionExpiry: permissionsMock.sessionExpiry, + sessionGeneration: permissionsMock.sessionGeneration, + username: permissionsMock.username, }, }), }, @@ -64,6 +92,14 @@ vi.mock("@/protoFleet/api/clients", () => ({ }, })); +vi.mock("@/protoFleet/features/updates/api/useUpgradeOperation", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + useUpgradeOperation: vi.fn(() => upgradeHookMock.current), + }; +}); + vi.mock("@/shared/utils/utility", () => ({ copyToClipboard: vi.fn(), })); @@ -102,13 +138,24 @@ const buildStatus = (overrides?: MessageOverrides): Get ...overrides, }); +const buildOperation = (phase: UpgradePhase, overrides?: MessageOverrides): UpgradeOperation => + create(UpgradeOperationSchema, { + id: "operation-1", + targetVersion: "v1.3.0", + phase, + message: "Preparing upgrade", + ...overrides, + }); + const mockUseHasPermission = vi.mocked(useHasPermission); +const mockUseUpgradeOperation = vi.mocked(useUpgradeOperation); const mockGetUpdateStatus = vi.mocked(instanceUpdateClient.getUpdateStatus); const mockSetReleaseChannel = vi.mocked(instanceUpdateClient.setReleaseChannel); const mockCopyToClipboard = vi.mocked(copyToClipboard); const mockPushToast = vi.mocked(pushToast); const RC_CHECKBOX_NAME = "Include release candidates"; +const UPDATE_STATUS_REQUEST_TIMEOUT_MS = 10_000; const RELEASE_CHANNEL_SAVE_TIMEOUT_MS = 30_000; const PERMISSION_REVOKED_MESSAGE = "You no longer have permission to update this instance"; @@ -125,9 +172,26 @@ const createDeferred = () => { beforeEach(() => { vi.clearAllMocks(); localStorage.clear(); + sessionStorage.clear(); + upgradeHookMock.current = { + acknowledgeOperation: vi.fn(), + connectionLost: false, + manualFallbackReady: false, + operation: undefined, + operationStatusPending: false, + reconciling: false, + reloadFleet: vi.fn(), + triggerError: null, + triggering: false, + trackedTargetVersion: undefined, + triggerUpgrade: vi.fn().mockResolvedValue(undefined), + useManualFallback: vi.fn(), + }; permissionsMock.current = ["instance:update", "fleet:read"]; permissionsMock.isAuthenticated = true; permissionsMock.sessionExpiry = new Date(1_000); + permissionsMock.sessionGeneration = 1; + permissionsMock.username = "operator-a"; permissionsMock.setPermissions.mockImplementation((permissions) => { permissionsMock.current = permissions; }); @@ -150,6 +214,275 @@ describe("Updates", () => { expect(link).toHaveAttribute("rel", "noopener noreferrer"); expect(getByText(INSTALL_COMMAND)).toBeInTheDocument(); expect(getByRole("button", { name: "Copy install command" })).toBeInTheDocument(); + expect(getByRole("button", { name: "Copy install command" })).toBeEnabled(); + expect(screen.queryByRole("button", { name: "Upgrade to v1.3.0" })).not.toBeInTheDocument(); + expect(mockGetUpdateStatus).toHaveBeenCalledWith({}, { timeoutMs: UPDATE_STATUS_REQUEST_TIMEOUT_MS }); + }); + + it("confirms the exact offered version before starting a one-click upgrade", async () => { + mockGetUpdateStatus.mockResolvedValue(buildStatus({ oneClickAvailable: true })); + + const page = render(); + fireEvent.click(await page.findByRole("button", { name: "Upgrade to v1.3.0" })); + + expect(page.getByTestId("upgrade-operation-modal")).toHaveTextContent( + "Fleet will validate and build this exact release", + ); + expect(upgradeHookMock.current.triggerUpgrade).not.toHaveBeenCalled(); + + fireEvent.click(page.getByRole("button", { name: "Confirm upgrade to v1.3.0" })); + await waitFor(() => expect(upgradeHookMock.current.triggerUpgrade).toHaveBeenCalledWith("v1.3.0")); + }); + + it("keeps an active operation ahead of a newer release offer", async () => { + upgradeHookMock.current.operation = buildOperation(UpgradePhase.PREFLIGHT, { + targetVersion: "v1.3.0", + message: "Validating v1.3.0", + }); + mockGetUpdateStatus.mockResolvedValue( + buildStatus({ + oneClickAvailable: true, + installCommand: "install v1.4.0", + latestEligible: buildReleaseInfo({ version: "v1.4.0" }), + }), + ); + + const page = render(); + + expect((await page.findAllByText("Validating v1.3.0")).length).toBeGreaterThan(0); + expect(page.queryByRole("button", { name: "Upgrade to v1.4.0" })).not.toBeInTheDocument(); + expect(page.getByRole("button", { name: "Copy install command" })).toBeDisabled(); + expect(page.getByRole("checkbox", { name: RC_CHECKBOX_NAME })).toBeDisabled(); + + fireEvent.click(page.getByRole("button", { name: "Close dialog" })); + fireEvent.click(page.getByRole("button", { name: "View upgrade details" })); + expect(page.getAllByText("Validating v1.3.0").length).toBeGreaterThan(0); + }); + + it("keeps manual recovery usable after a failed operation and can acknowledge its durable record", async () => { + upgradeHookMock.current.operation = buildOperation(UpgradePhase.FAILED, { + error: "new stack failed to start", + hostLogPath: "/var/lib/proto-fleet-updater/logs/operation-1.log", + recoveryCommand: "cd /opt/proto-fleet/deployment && ./run-fleet.sh --skip-build", + }); + mockGetUpdateStatus.mockResolvedValue(buildStatus({ oneClickAvailable: true })); + + const page = render(); + + expect(await page.findByText("new stack failed to start")).toBeInTheDocument(); + expect(page.getByRole("button", { name: "Copy install command" })).toBeEnabled(); + fireEvent.click(page.getByRole("button", { name: "Dismiss failure" })); + expect(upgradeHookMock.current.acknowledgeOperation).toHaveBeenCalledTimes(1); + }); + + it("offers a reload after the watched operation succeeds", async () => { + upgradeHookMock.current.operation = buildOperation(UpgradePhase.SUCCEEDED, { + message: "Upgrade complete", + }); + mockGetUpdateStatus.mockResolvedValue( + buildStatus({ + currentVersion: "v1.3.0", + updateAvailable: false, + installCommand: "", + latestEligible: undefined, + oneClickAvailable: true, + }), + ); + + const page = render(); + fireEvent.click(await page.findByRole("button", { name: "Reload Fleet" })); + + expect(upgradeHookMock.current.reloadFleet).toHaveBeenCalledTimes(1); + }); + + it("locks competing controls while reconciling an ambiguous trigger", async () => { + upgradeHookMock.current.reconciling = true; + upgradeHookMock.current.triggerError = "Fleet did not confirm the request"; + mockGetUpdateStatus.mockResolvedValue(buildStatus({ oneClickAvailable: true })); + + const page = render(); + + expect(await page.findByText(/checking upgrade status/i)).toBeInTheDocument(); + expect(page.getByRole("button", { name: "Copy install command" })).toBeDisabled(); + expect(page.getByRole("checkbox", { name: RC_CHECKBOX_NAME })).toBeDisabled(); + + fireEvent.click(page.getByRole("button", { name: "Close dialog" })); + const detailsButton = page.getByRole("button", { name: "View upgrade details" }); + expect(detailsButton).toBeEnabled(); + fireEvent.click(detailsButton); + expect(page.getByTestId("upgrade-operation-modal")).toBeInTheDocument(); + }); + + it("keeps install controls locked while fallback refreshes the authoritative offer", async () => { + const refreshedStatus = createDeferred(); + upgradeHookMock.current.reconciling = true; + upgradeHookMock.current.manualFallbackReady = true; + upgradeHookMock.current.triggerError = "Fleet did not confirm the request"; + mockGetUpdateStatus.mockResolvedValueOnce(buildStatus({ oneClickAvailable: true })); + mockGetUpdateStatus.mockReturnValueOnce(refreshedStatus.promise); + + const page = render(); + expect(await page.findByText("v1.3.0")).toBeInTheDocument(); + fireEvent.click(page.getByRole("button", { name: "I confirmed — unlock manual install" })); + expect(upgradeHookMock.current.useManualFallback).toHaveBeenCalledTimes(1); + await waitFor(() => expect(mockGetUpdateStatus).toHaveBeenCalledTimes(2)); + + upgradeHookMock.current = { + ...upgradeHookMock.current, + manualFallbackReady: false, + reconciling: false, + triggerError: null, + }; + page.rerender(); + + expect(page.getByRole("button", { name: "Copy install command" })).toBeDisabled(); + expect(page.getByRole("checkbox", { name: RC_CHECKBOX_NAME })).toBeDisabled(); + + await act(async () => { + refreshedStatus.resolve( + buildStatus({ + installCommand: "install v1.4.0", + latestEligible: buildReleaseInfo({ version: "v1.4.0" }), + oneClickAvailable: true, + }), + ); + await refreshedStatus.promise; + }); + + expect(await page.findByText("v1.4.0")).toBeInTheDocument(); + expect(page.getByRole("button", { name: "Copy install command" })).toBeEnabled(); + expect(page.getByRole("checkbox", { name: RC_CHECKBOX_NAME })).toBeEnabled(); + }); + + it("keeps controls locked while an untracked success refreshes the installed version", async () => { + const refreshedStatus = createDeferred(); + mockGetUpdateStatus.mockResolvedValueOnce(buildStatus({ oneClickAvailable: true })); + mockGetUpdateStatus.mockReturnValueOnce(refreshedStatus.promise); + + const page = render(); + expect(await page.findByText("v1.3.0")).toBeInTheDocument(); + const hookCalls = mockUseUpgradeOperation.mock.calls; + const hookOptions = hookCalls[hookCalls.length - 1]?.[0]; + expect(hookOptions?.onUntrackedSuccess).toEqual(expect.any(Function)); + + act(() => hookOptions?.onUntrackedSuccess?.(buildOperation(UpgradePhase.SUCCEEDED))); + await waitFor(() => expect(mockGetUpdateStatus).toHaveBeenCalledTimes(2)); + + expect(page.getByRole("button", { name: "Copy install command" })).toBeDisabled(); + expect(page.getByRole("checkbox", { name: RC_CHECKBOX_NAME })).toBeDisabled(); + + await act(async () => { + refreshedStatus.resolve( + buildStatus({ + currentVersion: "v1.3.0", + updateAvailable: false, + installCommand: "", + latestEligible: undefined, + oneClickAvailable: true, + }), + ); + await refreshedStatus.promise; + }); + + expect(await page.findByText("You're on the latest version")).toBeInTheDocument(); + expect(page.queryByRole("button", { name: "Copy install command" })).not.toBeInTheDocument(); + expect(page.getByRole("checkbox", { name: RC_CHECKBOX_NAME })).toBeEnabled(); + }); + + it("locks competing controls until the initial operation status resolves", async () => { + upgradeHookMock.current.operationStatusPending = true; + mockGetUpdateStatus.mockResolvedValue(buildStatus({ oneClickAvailable: true })); + + const page = render(); + + expect(await page.findByRole("button", { name: "Upgrade to v1.3.0" })).toBeDisabled(); + expect(page.getByRole("button", { name: "Copy install command" })).toBeDisabled(); + expect(page.getByRole("checkbox", { name: RC_CHECKBOX_NAME })).toBeDisabled(); + }); + + it("locks competing controls whenever a persisted operation remains unresolved", async () => { + upgradeHookMock.current.trackedTargetVersion = "v1.3.0"; + mockGetUpdateStatus.mockResolvedValue(buildStatus({ oneClickAvailable: true })); + + const page = render(); + + expect(await page.findByRole("button", { name: "Upgrade to v1.3.0" })).toBeDisabled(); + expect(page.getByRole("button", { name: "Copy install command" })).toBeDisabled(); + expect(page.getByRole("checkbox", { name: RC_CHECKBOX_NAME })).toBeDisabled(); + }); + + it("keeps an ambiguous request's target when a newer release is now eligible", async () => { + upgradeHookMock.current.reconciling = true; + upgradeHookMock.current.trackedTargetVersion = "v1.3.0"; + upgradeHookMock.current.triggerError = "Fleet did not confirm the v1.3.0 request"; + mockGetUpdateStatus.mockResolvedValue( + buildStatus({ + installCommand: "install v1.4.0", + latestEligible: buildReleaseInfo({ version: "v1.4.0" }), + oneClickAvailable: true, + }), + ); + + const page = render(); + + expect(await page.findByText("v1.4.0")).toBeInTheDocument(); + expect(page.getByTestId("upgrade-operation-modal")).toHaveTextContent("Upgrade Fleet to v1.3.0"); + }); + + it("refreshes the eligible release after reconciliation finds no operation", async () => { + const refreshedStatus = createDeferred(); + upgradeHookMock.current.reconciling = true; + upgradeHookMock.current.triggerError = "Fleet did not confirm the request"; + mockGetUpdateStatus + .mockResolvedValueOnce(buildStatus({ oneClickAvailable: true })) + .mockReturnValueOnce(refreshedStatus.promise); + + const page = render(); + expect(await page.findByText("v1.3.0")).toBeInTheDocument(); + + upgradeHookMock.current = { ...upgradeHookMock.current, reconciling: false }; + page.rerender(); + + await waitFor(() => expect(mockGetUpdateStatus).toHaveBeenCalledTimes(2)); + expect(page.queryByRole("button", { name: "Confirm upgrade to v1.3.0" })).not.toBeInTheDocument(); + + await act(async () => { + refreshedStatus.resolve( + buildStatus({ + installCommand: "install v1.4.0", + latestEligible: buildReleaseInfo({ version: "v1.4.0" }), + oneClickAvailable: true, + }), + ); + await refreshedStatus.promise; + }); + + expect(await page.findByText("v1.4.0")).toBeInTheDocument(); + expect(page.getByRole("button", { name: "Confirm upgrade to v1.4.0" })).toBeEnabled(); + }); + + it("keeps a stale modal target disabled after its authoritative refresh fails", async () => { + const failedRefresh = createDeferred(); + upgradeHookMock.current.reconciling = true; + upgradeHookMock.current.triggerError = "Fleet did not confirm the request"; + mockGetUpdateStatus + .mockResolvedValueOnce(buildStatus({ oneClickAvailable: true })) + .mockReturnValueOnce(failedRefresh.promise); + + const page = render(); + expect(await page.findByText("v1.3.0")).toBeInTheDocument(); + + upgradeHookMock.current = { ...upgradeHookMock.current, reconciling: false }; + page.rerender(); + await waitFor(() => expect(mockGetUpdateStatus).toHaveBeenCalledTimes(2)); + + await act(async () => { + failedRefresh.reject(new Error("release status unavailable")); + await failedRefresh.promise.catch(() => undefined); + }); + + expect(page.getByRole("alert")).toHaveTextContent("Fleet did not confirm the request"); + expect(page.queryByRole("button", { name: /Confirm upgrade/ })).not.toBeInTheDocument(); }); it("omits the release notes link when the server provides no URL", async () => { @@ -235,6 +568,11 @@ describe("Updates", () => { expect(await findByText("Unable to load update status")).toBeInTheDocument(); expect(getByText("release registry unreachable")).toBeInTheDocument(); + await waitFor(() => + expect(mockUseUpgradeOperation).toHaveBeenLastCalledWith( + expect.objectContaining({ currentVersionUnavailable: true }), + ), + ); }); it("saves a channel change and toasts success", async () => { @@ -440,6 +778,36 @@ describe("Updates", () => { expect(mockPushToast).not.toHaveBeenCalled(); }); + it("restarts a pending status refresh when the authenticated session changes in place", async () => { + const previousRequest = createDeferred(); + const replacementRequest = createDeferred(); + mockGetUpdateStatus.mockReturnValueOnce(previousRequest.promise).mockReturnValueOnce(replacementRequest.promise); + + const page = render(); + await waitFor(() => expect(mockGetUpdateStatus).toHaveBeenCalledTimes(1)); + + permissionsMock.sessionExpiry = new Date(2_000); + permissionsMock.sessionGeneration = 2; + page.rerender(); + + await waitFor(() => expect(mockGetUpdateStatus).toHaveBeenCalledTimes(2)); + + await act(async () => { + replacementRequest.resolve(buildStatus()); + await replacementRequest.promise; + }); + + expect(await page.findByText("v1.3.0")).toBeInTheDocument(); + expect(page.getByRole("button", { name: "Copy install command" })).toBeEnabled(); + + await act(async () => { + previousRequest.reject(new ConnectError("old session expired", Code.Unauthenticated)); + await previousRequest.promise.catch(() => undefined); + }); + + expect(authErrorsMock.handleAuthErrors).not.toHaveBeenCalled(); + }); + it("disables channel and copy controls throughout the save and refetch", async () => { const save = createDeferred(); const refetch = createDeferred(); @@ -692,6 +1060,28 @@ describe("Updates", () => { expect(page.getByTestId("navigate")).toHaveAttribute("data-to", "/settings/network"); }); + it("invalidates stale client permission when upgrade polling is denied", async () => { + mockGetUpdateStatus.mockResolvedValue(buildStatus()); + + const page = render(); + await page.findByText("v1.2.0"); + const lastHookCall = mockUseUpgradeOperation.mock.calls[mockUseUpgradeOperation.mock.calls.length - 1]; + const onPollError = lastHookCall?.[0].onPollError; + + act(() => { + onPollError?.(new ConnectError("permission revoked", Code.PermissionDenied)); + }); + + expect(mockPushToast).toHaveBeenCalledWith({ + message: PERMISSION_REVOKED_MESSAGE, + status: "error", + }); + expect(permissionsMock.setPermissions).toHaveBeenCalledWith(["fleet:read"]); + + page.rerender(); + expect(page.getByTestId("navigate")).toHaveAttribute("data-to", "/settings/network"); + }); + it("toasts an error and leaves the checkbox unchecked when saving the channel fails", async () => { mockGetUpdateStatus .mockResolvedValueOnce(buildStatus({ channel: ReleaseChannel.STABLE })) diff --git a/client/src/protoFleet/features/settings/components/Updates.tsx b/client/src/protoFleet/features/settings/components/Updates.tsx index 7119e49a05..3f290d1685 100644 --- a/client/src/protoFleet/features/settings/components/Updates.tsx +++ b/client/src/protoFleet/features/settings/components/Updates.tsx @@ -1,16 +1,27 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { Navigate } from "react-router-dom"; import { instanceUpdateClient } from "@/protoFleet/api/clients"; -import { ReleaseChannel } from "@/protoFleet/api/generated/instance/v1/updates_pb"; +import { ReleaseChannel, UpgradePhase } from "@/protoFleet/api/generated/instance/v1/updates_pb"; import type { GetUpdateStatusResponse } from "@/protoFleet/api/generated/instance/v1/updates_pb"; import { getErrorMessage } from "@/protoFleet/api/getErrorMessage"; import { isAuthOrPermissionError, isPermissionDeniedError } from "@/protoFleet/api/requestErrors"; import { getSettingsLandingPath } from "@/protoFleet/config/navItems"; import SettingsEmptyState from "@/protoFleet/features/settings/components/SettingsEmptyState"; import SettingsPageHeader from "@/protoFleet/features/settings/components/SettingsPageHeader"; +import UpgradeOperationModal from "@/protoFleet/features/settings/components/UpgradeOperationModal"; +import { isUpgradeActive, useUpgradeOperation } from "@/protoFleet/features/updates/api/useUpgradeOperation"; import { copyInstallCommand } from "@/protoFleet/features/updates/copyInstallCommand"; -import { useAuthErrors, useFleetStore, useHasPermission, usePermissions, useSetPermissions } from "@/protoFleet/store"; +import { + useAuthErrors, + useFleetStore, + useHasPermission, + usePermissions, + useSessionGeneration, + useSetPermissions, + useUsername, +} from "@/protoFleet/store"; import { Copy } from "@/shared/assets/icons"; +import Button, { variants } from "@/shared/components/Button"; import Checkbox from "@/shared/components/Checkbox"; import Header from "@/shared/components/Header"; import Row from "@/shared/components/Row"; @@ -19,25 +30,30 @@ import { pushToast, STATUSES } from "@/shared/features/toaster"; const SkeletonLoader = ; const INSTANCE_UPDATE_PERMISSION = "instance:update"; +const UPDATE_STATUS_REQUEST_TIMEOUT_MS = 10_000; const RELEASE_CHANNEL_SAVE_TIMEOUT_MS = 30_000; const PERMISSION_REVOKED_MESSAGE = "You no longer have permission to update this instance"; -const UPDATES_PAGE_DESCRIPTION = "View the server version and choose which releases this instance installs."; +const UPDATES_PAGE_DESCRIPTION = + "View the server version, choose which releases this instance installs, and apply eligible updates."; interface AuthSessionSnapshot { + identity: string; isAuthenticated: boolean; sessionExpiry: Date | null; } -const captureAuthSession = (): AuthSessionSnapshot => { +const captureAuthSession = (identity: string): AuthSessionSnapshot => { const { isAuthenticated, sessionExpiry } = useFleetStore.getState().auth; - return { isAuthenticated, sessionExpiry }; + return { identity, isAuthenticated, sessionExpiry }; }; const isSameAuthSession = (snapshot: AuthSessionSnapshot) => { - const { isAuthenticated, sessionExpiry } = useFleetStore.getState().auth; - // Login installs a new Date object and logout clears it. Compare identity so - // even a replacement session for the same user cannot inherit old failures. - return isAuthenticated === snapshot.isAuthenticated && sessionExpiry === snapshot.sessionExpiry; + const { isAuthenticated, sessionExpiry, sessionGeneration, username } = useFleetStore.getState().auth; + return ( + `${username}:${sessionGeneration}` === snapshot.identity && + isAuthenticated === snapshot.isAuthenticated && + sessionExpiry === snapshot.sessionExpiry + ); }; // A route remount must not load status while the previous page instance is @@ -83,13 +99,21 @@ const waitForReleaseChannelSave = async () => { const Updates = () => { const canUpdateInstance = useHasPermission(INSTANCE_UPDATE_PERMISSION); const permissions = usePermissions(); + const sessionGeneration = useSessionGeneration(); const setPermissions = useSetPermissions(); + const username = useUsername(); + const authSessionIdentity = `${username}:${sessionGeneration}`; const { handleAuthErrors } = useAuthErrors(); const [status, setStatus] = useState(null); const [loadError, setLoadError] = useState(null); const [isChannelChangePending, setIsChannelChangePending] = useState(false); + const [isStatusRefreshPending, setIsStatusRefreshPending] = useState(false); + const [upgradeModalOpen, setUpgradeModalOpen] = useState(false); const latestStatusRequest = useRef(0); const isMounted = useRef(false); + const lastAutoOpenedOperation = useRef(null); + const previousReconciling = useRef(false); + const previousTriggerError = useRef(null); // The channel the server has persisted; the checkbox is controlled by it, // so a failed save never moves the control. const [channel, setChannel] = useState(ReleaseChannel.UNSPECIFIED); @@ -110,15 +134,30 @@ const Updates = () => { [setPermissions], ); + const handleUpgradePollError = useCallback( + (error: unknown) => { + handleAuthErrors({ + error, + onError: () => { + if (isPermissionDeniedError(error)) { + handlePermissionRevoked(isMounted.current); + } + }, + }); + }, + [handleAuthErrors, handlePermissionRevoked], + ); + const fetchStatus = useCallback(async () => { const requestId = ++latestStatusRequest.current; - const authSession = captureAuthSession(); - await waitForReleaseChannelSave(); - if (requestId !== latestStatusRequest.current || !isSameAuthSession(authSession)) { - return; - } + const authSession = captureAuthSession(authSessionIdentity); + setIsStatusRefreshPending(true); try { - const response = await instanceUpdateClient.getUpdateStatus({}); + await waitForReleaseChannelSave(); + if (requestId !== latestStatusRequest.current || !isSameAuthSession(authSession)) { + return; + } + const response = await instanceUpdateClient.getUpdateStatus({}, { timeoutMs: UPDATE_STATUS_REQUEST_TIMEOUT_MS }); if (requestId !== latestStatusRequest.current || !isSameAuthSession(authSession)) { return; } @@ -143,8 +182,47 @@ const Updates = () => { setLoadError(getErrorMessage(err, "Failed to load update status")); }, }); + } finally { + if (requestId === latestStatusRequest.current && isSameAuthSession(authSession) && isMounted.current) { + setIsStatusRefreshPending(false); + } } - }, [handleAuthErrors, handlePermissionRevoked]); + }, [authSessionIdentity, handleAuthErrors, handlePermissionRevoked]); + + const upgrade = useUpgradeOperation({ + authSessionIdentity, + enabled: canUpdateInstance, + currentVersion: status?.currentVersion, + currentVersionUnavailable: Boolean(loadError && !status), + onUntrackedSuccess: () => { + void fetchStatus(); + }, + onPollError: handleUpgradePollError, + }); + const activeUpgrade = isUpgradeActive(upgrade.operation); + const succeededUpgrade = upgrade.operation?.phase === UpgradePhase.SUCCEEDED; + const upgradeRequestPending = upgrade.triggering || upgrade.reconciling; + const unresolvedTrackedUpgrade = Boolean(upgrade.trackedTargetVersion && !upgrade.operation); + const upgradeLocksConfiguration = + isStatusRefreshPending || + upgrade.operationStatusPending || + upgradeRequestPending || + unresolvedTrackedUpgrade || + Boolean(upgrade.operation); + const upgradeActionDisabled = + isChannelChangePending || + isStatusRefreshPending || + upgrade.operationStatusPending || + upgradeRequestPending || + unresolvedTrackedUpgrade; + const manualCommandDisabled = + isChannelChangePending || + isStatusRefreshPending || + upgrade.operationStatusPending || + activeUpgrade || + upgradeRequestPending || + unresolvedTrackedUpgrade || + Boolean(succeededUpgrade); useEffect(() => { isMounted.current = true; @@ -153,6 +231,45 @@ const Updates = () => { }; }, []); + useEffect(() => { + const operation = upgrade.operation; + if (!operation) { + return; + } + const terminal = operation.phase === UpgradePhase.SUCCEEDED || operation.phase === UpgradePhase.FAILED; + const autoOpenKey = `${operation.id}:${terminal ? "terminal" : "active"}`; + if (lastAutoOpenedOperation.current === autoOpenKey) { + return; + } + // Open once when an operation is first recovered, and once more when it + // becomes terminal. Intermediate phase updates must not keep stealing + // focus after an operator dismisses the progress modal. + lastAutoOpenedOperation.current = autoOpenKey; + setUpgradeModalOpen(true); + }, [upgrade.operation]); + + useEffect(() => { + const wasReconciling = previousReconciling.current; + if (upgrade.reconciling && !previousReconciling.current) { + setUpgradeModalOpen(true); + } + if (wasReconciling && !upgrade.reconciling && !upgrade.operation && upgrade.triggerError) { + // A reachable executor authoritatively found no matching operation. + // Refresh the offer before allowing a retry because the release/channel + // may have changed while the trigger outcome was being reconciled. + void fetchStatus(); + } + previousReconciling.current = upgrade.reconciling; + }, [fetchStatus, upgrade.operation, upgrade.reconciling, upgrade.triggerError]); + + useEffect(() => { + if (upgrade.triggerError && upgrade.triggerError !== previousTriggerError.current && !upgrade.reconciling) { + setUpgradeModalOpen(true); + void fetchStatus(); + } + previousTriggerError.current = upgrade.triggerError; + }, [fetchStatus, upgrade.reconciling, upgrade.triggerError]); + useEffect(() => { // The RPC is server-gated on instance:update; non-holders are redirected // below and must not fire it. @@ -169,10 +286,10 @@ const Updates = () => { const handleIncludeRCChange = async (includeRC: boolean) => { const nextChannel = includeRC ? ReleaseChannel.STABLE_AND_RC : ReleaseChannel.STABLE; - if (nextChannel === channel || isChannelChangePending) { + if (nextChannel === channel || isChannelChangePending || upgradeLocksConfiguration) { return; } - const authSession = captureAuthSession(); + const authSession = captureAuthSession(authSessionIdentity); setIsChannelChangePending(true); try { let saveSucceeded = false; @@ -233,10 +350,62 @@ const Updates = () => { } const release = status?.statusAvailable && status.updateAvailable ? status.latestEligible : undefined; + const modalRelease = + isStatusRefreshPending || loadError || (upgrade.operation && upgrade.operation.targetVersion !== release?.version) + ? undefined + : release; + const operationStatusLabel = upgrade.reconciling + ? upgrade.manualFallbackReady + ? "Upgrade outcome is unknown — host confirmation required" + : "Confirming upgrade status" + : upgrade.operation?.phase === UpgradePhase.FAILED + ? "Upgrade failed" + : upgrade.operation?.phase === UpgradePhase.SUCCEEDED + ? "Upgrade complete — reload Fleet" + : upgrade.operation + ? upgrade.operation.message || `Upgrading Fleet to ${upgrade.operation.targetVersion}` + : upgrade.triggerError + ? "Upgrade request needs attention" + : null; + const hasUpgradeDetails = Boolean(upgrade.operation || upgrade.reconciling || upgrade.triggerError); return (
+ { + upgrade.acknowledgeOperation(); + setUpgradeModalOpen(false); + }} + onDismiss={() => setUpgradeModalOpen(false)} + onReload={upgrade.reloadFleet} + onUpgrade={upgrade.triggerUpgrade} + onUseManualFallback={() => { + upgrade.useManualFallback(); + setUpgradeModalOpen(false); + void fetchStatus(); + }} + open={upgradeModalOpen} + operation={upgrade.operation} + reconciling={upgrade.reconciling} + release={modalRelease} + targetVersion={ + upgrade.operation?.targetVersion ?? (upgrade.reconciling ? upgrade.trackedTargetVersion : release?.version) + } + triggerError={upgrade.triggerError} + triggering={upgrade.triggering} + /> + {loadError && hasUpgradeDetails ? ( +
+
+
Upgrade status
+
{operationStatusLabel}
+
+
+ ) : null} {loadError ? ( ) : ( @@ -268,13 +437,33 @@ const Updates = () => { ) : null}
+ {status?.oneClickAvailable || hasUpgradeDetails ? ( + +
+
{hasUpgradeDetails ? "Upgrade status" : "One-click upgrade"}
+ {operationStatusLabel ? ( +
{operationStatusLabel}
+ ) : ( +
+ Fleet validates the release before restarting the instance. +
+ )} +
+
@@ -303,7 +505,7 @@ const Updates = () => { void handleIncludeRCChange(e.target.checked)} /> Include release candidates diff --git a/client/src/protoFleet/features/settings/components/UpgradeOperationModal.stories.tsx b/client/src/protoFleet/features/settings/components/UpgradeOperationModal.stories.tsx new file mode 100644 index 0000000000..653f965141 --- /dev/null +++ b/client/src/protoFleet/features/settings/components/UpgradeOperationModal.stories.tsx @@ -0,0 +1,117 @@ +import { create } from "@bufbuild/protobuf"; +import type { Meta, StoryObj } from "@storybook/react"; +import { action } from "storybook/actions"; + +import { + ReleaseInfoSchema, + UpgradeOperationSchema, + UpgradePhase, +} from "@/protoFleet/api/generated/instance/v1/updates_pb"; +import UpgradeOperationModal from "@/protoFleet/features/settings/components/UpgradeOperationModal"; + +const release = (version = "v1.3.0", prerelease = false) => + create(ReleaseInfoSchema, { + version, + prerelease, + }); + +const operation = (phase: UpgradePhase, message: string) => + create(UpgradeOperationSchema, { + id: "operation-1", + targetVersion: "v1.3.0", + phase, + message, + }); + +const meta = { + title: "Proto Fleet/Settings/UpgradeOperationModal", + component: UpgradeOperationModal, + parameters: { + layout: "fullscreen", + }, + args: { + connectionLost: false, + manualFallbackReady: false, + onAcknowledge: action("acknowledge failure"), + onDismiss: action("dismiss modal"), + onReload: action("reload Fleet"), + onUpgrade: (targetVersion: string) => { + action("start upgrade")(targetVersion); + return Promise.resolve(); + }, + onUseManualFallback: action("unlock manual install"), + open: true, + reconciling: false, + release: release(), + triggerError: null, + triggering: false, + }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const StableConfirmation: Story = {}; + +export const ReleaseCandidateConfirmation: Story = { + args: { + release: release("v1.3.0-rc.2", true), + }, +}; + +export const CheckingStatus: Story = { + args: { + reconciling: true, + targetVersion: "v1.3.0", + triggerError: "The upgrade request timed out before Fleet received a response.", + }, +}; + +export const ManualFallbackConfirmation: Story = { + args: { + connectionLost: true, + manualFallbackReady: true, + reconciling: true, + targetVersion: "v1.3.0", + triggerError: "The host updater is not reporting the tracked upgrade.", + }, +}; + +export const Preflight: Story = { + args: { + operation: operation(UpgradePhase.PREFLIGHT, "Validating the new stack"), + }, +}; + +export const RestartingServices: Story = { + args: { + connectionLost: true, + operation: operation(UpgradePhase.ACTIVATING, "Restarting Fleet services"), + }, +}; + +export const FailedWithRecovery: Story = { + args: { + operation: create(UpgradeOperationSchema, { + id: "operation-1", + targetVersion: "v1.3.0", + phase: UpgradePhase.FAILED, + message: "Upgrade failed", + error: "The replacement stack did not become ready.", + hostLogPath: "/var/lib/proto-fleet-updater/logs/operation-1.log", + recoveryCommand: "cd /opt/proto-fleet/deployment && ./run-fleet.sh --non-interactive --skip-build", + }), + }, +}; + +export const Succeeded: Story = { + args: { + operation: operation(UpgradePhase.SUCCEEDED, "Fleet v1.3.0 is running"), + }, +}; + +export const TriggerError: Story = { + args: { + triggerError: "The host updater rejected the request. Review the host logs and try again.", + }, +}; diff --git a/client/src/protoFleet/features/settings/components/UpgradeOperationModal.test.tsx b/client/src/protoFleet/features/settings/components/UpgradeOperationModal.test.tsx new file mode 100644 index 0000000000..052b7ee548 --- /dev/null +++ b/client/src/protoFleet/features/settings/components/UpgradeOperationModal.test.tsx @@ -0,0 +1,263 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { create } from "@bufbuild/protobuf"; + +import { + ReleaseInfoSchema, + type UpgradeOperation, + UpgradeOperationSchema, + UpgradePhase, +} from "@/protoFleet/api/generated/instance/v1/updates_pb"; +import UpgradeOperationModal, { + type UpgradeOperationModalProps, +} from "@/protoFleet/features/settings/components/UpgradeOperationModal"; +import { pushToast, STATUSES } from "@/shared/features/toaster"; +import { copyToClipboard } from "@/shared/utils/utility"; + +vi.mock("@/shared/features/toaster", () => ({ + pushToast: vi.fn(), + STATUSES: { + error: "error", + success: "success", + }, +})); + +vi.mock("@/shared/utils/utility", () => ({ + copyToClipboard: vi.fn(), +})); + +const mockCopyToClipboard = vi.mocked(copyToClipboard); +const mockPushToast = vi.mocked(pushToast); + +const release = (version = "v1.3.0", prerelease = false) => + create(ReleaseInfoSchema, { + version, + prerelease, + }); + +type OperationOverrides = Partial< + Pick +>; + +const operation = (phase: UpgradePhase, overrides?: OperationOverrides) => + create(UpgradeOperationSchema, { + id: "operation-1", + targetVersion: "v1.3.0", + phase, + message: "Preparing upgrade", + ...overrides, + }); + +const renderModal = (overrides: Partial = {}) => { + const props: UpgradeOperationModalProps = { + connectionLost: false, + manualFallbackReady: false, + onAcknowledge: vi.fn(), + onDismiss: vi.fn(), + onReload: vi.fn(), + onUpgrade: vi.fn().mockResolvedValue(undefined), + onUseManualFallback: vi.fn(), + open: true, + reconciling: false, + release: release(), + triggerError: null, + triggering: false, + ...overrides, + }; + return { ...render(), props }; +}; + +describe("UpgradeOperationModal", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockCopyToClipboard.mockResolvedValue(undefined); + }); + + it("confirms the exact stable target before starting the upgrade", async () => { + const onUpgrade = vi.fn().mockResolvedValue(undefined); + renderModal({ onUpgrade }); + + const confirmButton = screen.getByRole("button", { name: "Confirm upgrade to v1.3.0" }); + expect(confirmButton).toBeInTheDocument(); + fireEvent.click(confirmButton); + + await waitFor(() => expect(onUpgrade).toHaveBeenCalledWith("v1.3.0")); + }); + + it("gives release candidates a strong forward-only migration warning", () => { + renderModal({ release: release("v1.3.0-rc.2", true) }); + + expect(screen.getByText(/This is a release candidate/)).toBeInTheDocument(); + expect(screen.getByText(/forward-only database migrations/)).toBeInTheDocument(); + expect(screen.getByText(/cannot downgrade this instance afterward/)).toBeInTheDocument(); + }); + + it("keeps a long-running upgrade dismissible", () => { + const onDismiss = vi.fn(); + renderModal({ + onDismiss, + operation: operation(UpgradePhase.PREFLIGHT, { message: "Validating the new stack" }), + }); + + expect(screen.getByRole("status")).toHaveTextContent("Phase: Preflight"); + fireEvent.click(screen.getByRole("button", { name: "Close dialog" })); + expect(onDismiss).toHaveBeenCalledOnce(); + }); + + it("announces the expected reconnect state", () => { + renderModal({ + connectionLost: true, + operation: operation(UpgradePhase.ACTIVATING), + }); + + const status = screen.getByRole("status"); + expect(status).toHaveAttribute("aria-live", "polite"); + expect(status).toHaveTextContent(/disconnect is expected while services restart/); + }); + + it("warns against manual installation while reconciling an ambiguous trigger", () => { + renderModal({ reconciling: true, triggerError: "The request timed out" }); + + expect(screen.getByRole("status")).toHaveTextContent(/Checking upgrade status/); + expect(screen.getByRole("status")).toHaveTextContent(/Do not run the manual install command yet/); + expect(screen.queryByRole("button", { name: /Confirm upgrade/ })).not.toBeInTheDocument(); + }); + + it("keeps an unknown host-updater phase visibly locked", () => { + renderModal({ + operation: operation(UpgradePhase.UNSPECIFIED), + reconciling: true, + }); + + expect(screen.getByRole("status")).toHaveTextContent(/reported an unknown upgrade state/i); + expect(screen.getByRole("status")).toHaveTextContent(/Manual installation remains locked/i); + expect(screen.queryByRole("button", { name: /unlock manual install/i })).not.toBeInTheDocument(); + }); + + it("requires explicit host confirmation before unlocking a manual fallback", () => { + const onUseManualFallback = vi.fn(); + renderModal({ + connectionLost: true, + manualFallbackReady: true, + onUseManualFallback, + reconciling: true, + triggerError: "Host updater did not confirm the upgrade", + }); + + expect(screen.getByRole("status")).toHaveTextContent(/checking the host and confirming no upgrade is running/i); + fireEvent.click(screen.getByRole("button", { name: "I confirmed — unlock manual install" })); + expect(onUseManualFallback).toHaveBeenCalledOnce(); + }); + + it("labels reconciliation with its tracked target rather than a newer offer", () => { + renderModal({ + reconciling: true, + release: release("v1.4.0"), + targetVersion: "v1.3.0", + triggerError: "The v1.3.0 request has an unknown outcome", + }); + + expect(screen.getByTestId("upgrade-operation-modal")).toHaveTextContent("Upgrade Fleet to v1.3.0"); + expect(screen.getByTestId("upgrade-operation-modal")).not.toHaveTextContent("Upgrade Fleet to v1.4.0"); + }); + + it("shows failed-operation details and copies the recovery command", async () => { + const recoveryCommand = "cd /opt/proto-fleet/deployment && ./run-fleet.sh --non-interactive --skip-build"; + renderModal({ + operation: operation(UpgradePhase.FAILED, { + message: "Upgrade failed", + error: "new stack failed to start", + hostLogPath: "/var/lib/proto-fleet-updater/logs/operation-1.log", + recoveryCommand, + }), + }); + + const alert = screen.getByRole("alert"); + expect(alert).toHaveTextContent("new stack failed to start"); + expect(alert).toHaveTextContent("operation-1.log"); + fireEvent.click(screen.getByRole("button", { name: "Copy recovery command" })); + + await waitFor(() => expect(mockCopyToClipboard).toHaveBeenCalledWith(recoveryCommand)); + expect(mockPushToast).toHaveBeenCalledWith({ + message: "Recovery command copied to clipboard", + status: STATUSES.success, + }); + }); + + it("reports a recovery-command copy failure", async () => { + mockCopyToClipboard.mockRejectedValue(new Error("copy failed")); + renderModal({ + operation: operation(UpgradePhase.FAILED, { + recoveryCommand: "./run-fleet.sh --non-interactive --skip-build", + }), + }); + + fireEvent.click(screen.getByRole("button", { name: "Copy recovery command" })); + + await waitFor(() => + expect(mockPushToast).toHaveBeenCalledWith({ + message: "Failed to copy recovery command", + status: STATUSES.error, + }), + ); + }); + + it("does not render an empty recovery fallback", () => { + renderModal({ + operation: operation(UpgradePhase.FAILED, { + error: "preflight failed", + recoveryCommand: " ", + }), + }); + + expect(screen.queryByText("Recovery command")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Copy recovery command" })).not.toBeInTheDocument(); + }); + + it("acknowledges a failure separately from hiding its details", () => { + const onAcknowledge = vi.fn(); + const onDismiss = vi.fn(); + renderModal({ + onAcknowledge, + onDismiss, + operation: operation(UpgradePhase.FAILED), + }); + + fireEvent.click(screen.getByRole("button", { name: "Dismiss failure" })); + expect(onAcknowledge).toHaveBeenCalledOnce(); + expect(onDismiss).not.toHaveBeenCalled(); + }); + + it("reloads Fleet after a successful upgrade", () => { + const onReload = vi.fn(); + renderModal({ + onReload, + operation: operation(UpgradePhase.SUCCEEDED, { message: "Fleet v1.3.0 is running" }), + }); + + fireEvent.click(screen.getByRole("button", { name: "Reload Fleet" })); + expect(onReload).toHaveBeenCalledOnce(); + }); + + it("announces a trigger error and allows retry after reconciliation", async () => { + const onUpgrade = vi.fn().mockResolvedValue(undefined); + renderModal({ + onUpgrade, + triggerError: "Host updater did not answer", + }); + + expect(screen.getByRole("alert")).toHaveTextContent("Host updater did not answer"); + fireEvent.click(screen.getByRole("button", { name: "Confirm upgrade to v1.3.0" })); + await waitFor(() => expect(onUpgrade).toHaveBeenCalledWith("v1.3.0")); + }); + + it("keeps a trigger error visible when there is no longer an eligible release", () => { + renderModal({ + release: undefined, + triggerError: "The eligible release changed before the request completed", + }); + + expect(screen.getByRole("alert")).toHaveTextContent("The eligible release changed before the request completed"); + expect(screen.queryByRole("button", { name: /Confirm upgrade/ })).not.toBeInTheDocument(); + }); +}); diff --git a/client/src/protoFleet/features/settings/components/UpgradeOperationModal.tsx b/client/src/protoFleet/features/settings/components/UpgradeOperationModal.tsx new file mode 100644 index 0000000000..fe2efc6f95 --- /dev/null +++ b/client/src/protoFleet/features/settings/components/UpgradeOperationModal.tsx @@ -0,0 +1,382 @@ +import type { ComponentProps, ReactNode } from "react"; +import { + type ReleaseInfo, + type UpgradeOperation, + UpgradePhase, +} from "@/protoFleet/api/generated/instance/v1/updates_pb"; +import { Copy } from "@/shared/assets/icons"; +import Button, { variants } from "@/shared/components/Button"; +import Modal from "@/shared/components/Modal"; +import ProgressCircular from "@/shared/components/ProgressCircular"; +import { pushToast, STATUSES } from "@/shared/features/toaster"; +import { copyToClipboard } from "@/shared/utils/utility"; + +export interface UpgradeOperationModalProps { + connectionLost: boolean; + manualFallbackReady: boolean; + onAcknowledge: () => void; + onDismiss: () => void; + onReload: () => void; + onUpgrade: (targetVersion: string) => Promise; + onUseManualFallback: () => void; + open: boolean; + operation?: UpgradeOperation; + reconciling: boolean; + release?: ReleaseInfo; + targetVersion?: string; + triggerError: string | null; + triggering: boolean; +} + +const ACTIVE_PHASE_LABELS: Partial> = { + [UpgradePhase.QUEUED]: "Queued", + [UpgradePhase.DOWNLOADING]: "Downloading", + [UpgradePhase.VERIFYING]: "Verifying", + [UpgradePhase.STAGING]: "Staging", + [UpgradePhase.PREFLIGHT]: "Preflight", + [UpgradePhase.ACTIVATING]: "Activating", +}; + +type ModalButtons = NonNullable["buttons"]>; + +interface ProgressPanelProps { + children: ReactNode; + heading: string; +} + +const ProgressPanel = ({ children, heading }: ProgressPanelProps) => ( +
+
+ +
{heading}
+
+ {children} +
+); + +interface ReconciliationPanelProps { + manualFallbackReady: boolean; + unknownPhase: boolean; +} + +const ReconciliationPanel = ({ manualFallbackReady, unknownPhase }: ReconciliationPanelProps) => ( + + {manualFallbackReady ? ( +

+ {unknownPhase + ? "Fleet cannot interpret the host updater's current state. " + : "The host updater is not reporting this upgrade. "} + Only unlock the manual command after checking the host and confirming no upgrade is running. Overlapping + installs can leave the deployment unusable. +

+ ) : unknownPhase ? ( +

+ This version of Fleet does not recognize the state returned by the host updater. Manual installation remains + locked while Fleet continues checking. +

+ ) : ( +

+ Fleet is reconciling the tracked upgrade with the host updater. Do not run the manual install command yet; wait + until this check finishes. +

+ )} +
+); + +interface ActiveUpgradePanelProps { + connectionLost: boolean; + operation: UpgradeOperation; +} + +const ActiveUpgradePanel = ({ connectionLost, operation }: ActiveUpgradePanelProps) => ( + +
Phase: {ACTIVE_PHASE_LABELS[operation.phase] ?? "Starting"}
+ {connectionLost ? ( +

+ Fleet is temporarily unreachable. A disconnect is expected while services restart; this page will keep checking + for progress. +

+ ) : ( +

+ You can close this dialog while Fleet downloads, validates, and activates the release. Return to this Updates + page to check progress. +

+ )} +
+); + +const copyRecoveryCommand = (recoveryCommand: string) => { + void copyToClipboard(recoveryCommand) + .then(() => { + pushToast({ + message: "Recovery command copied to clipboard", + status: STATUSES.success, + }); + }) + .catch(() => { + pushToast({ + message: "Failed to copy recovery command", + status: STATUSES.error, + }); + }); +}; + +const FailedUpgradePanel = ({ operation }: { operation: UpgradeOperation }) => { + const error = operation.error.trim(); + const hostLogPath = operation.hostLogPath.trim(); + const recoveryCommand = operation.recoveryCommand.trim(); + + return ( +
+
{operation.message || "Upgrade failed"}
+ {error ?

{error}

: null} + {hostLogPath ? ( +

+ Host log: {hostLogPath} +

+ ) : null} + {recoveryCommand ? ( +
+
Recovery command
+
+ {recoveryCommand} +
+
+ ) : null} +
+ ); +}; + +const SucceededUpgradePanel = ({ operation }: { operation: UpgradeOperation }) => ( +
+
{operation.message || "Upgrade complete"}
+

+ Reload Fleet to use the client bundled with {operation.targetVersion}. +

+
+); + +const UpgradeConfirmationPanel = ({ release }: { release: ReleaseInfo }) => ( +
+
Confirm upgrade to {release.version}
+

+ Fleet will validate and build this exact release first, then take the instance offline for several minutes while + containers restart and database migrations run. +

+ {release.prerelease ? ( +

+ This is a release candidate. The upgrade can run forward-only database migrations, and you cannot downgrade this + instance afterward. Continue only if you accept that recovery may require a newer compatible release. +

+ ) : null} +
+); + +interface UpgradeOperationContentProps { + connectionLost: boolean; + manualFallbackReady: boolean; + operation?: UpgradeOperation; + reconciling: boolean; + release?: ReleaseInfo; +} + +const UpgradeOperationContent = ({ + connectionLost, + manualFallbackReady, + operation, + reconciling, + release, +}: UpgradeOperationContentProps) => { + if (reconciling) { + return ( + + ); + } + if (!operation) { + return release ? : null; + } + if (operation.phase === UpgradePhase.FAILED) { + return ; + } + if (operation.phase === UpgradePhase.SUCCEEDED) { + return ; + } + return ; +}; + +interface GetModalButtonsOptions { + handleUpgrade: () => void; + manualFallbackReady: boolean; + onAcknowledge: () => void; + onDismiss: () => void; + onReload: () => void; + onUseManualFallback: () => void; + operation?: UpgradeOperation; + reconciling: boolean; + release?: ReleaseInfo; + triggering: boolean; +} + +const getModalButtons = ({ + handleUpgrade, + manualFallbackReady, + onAcknowledge, + onDismiss, + onReload, + onUseManualFallback, + operation, + reconciling, + release, + triggering, +}: GetModalButtonsOptions): ModalButtons | undefined => { + if (manualFallbackReady) { + return [ + { + text: "I confirmed — unlock manual install", + variant: variants.secondaryDanger, + onClick: onUseManualFallback, + dismissModalOnClick: false, + }, + ]; + } + if (operation?.phase === UpgradePhase.SUCCEEDED) { + return [ + { + text: "Reload Fleet", + variant: variants.primary, + onClick: onReload, + dismissModalOnClick: false, + }, + ]; + } + if (operation?.phase === UpgradePhase.FAILED) { + return [ + { + text: "Dismiss failure", + variant: variants.secondary, + onClick: onAcknowledge, + dismissModalOnClick: false, + }, + ]; + } + if (operation || reconciling || !release) { + return undefined; + } + return [ + { + text: "Cancel", + variant: variants.secondary, + onClick: onDismiss, + dismissModalOnClick: false, + }, + { + text: `Confirm upgrade to ${release.version}`, + variant: variants.primary, + onClick: handleUpgrade, + loading: triggering, + dismissModalOnClick: false, + }, + ]; +}; + +const UpgradeOperationModal = ({ + connectionLost, + manualFallbackReady, + onAcknowledge, + onDismiss, + onReload, + onUpgrade, + onUseManualFallback, + open, + operation, + reconciling, + release, + targetVersion, + triggerError, + triggering, +}: UpgradeOperationModalProps) => { + if (!release && !operation && !reconciling && !triggerError) { + return null; + } + + const displayedTargetVersion = operation?.targetVersion || targetVersion || release?.version; + + const handleUpgrade = () => { + if (!release || reconciling) return; + void onUpgrade(release.version).catch(() => { + // The route owns reconciliation and exposes a terminal triggerError. + }); + }; + + const buttons = getModalButtons({ + handleUpgrade, + manualFallbackReady, + onAcknowledge, + onDismiss, + onReload, + onUseManualFallback, + operation, + reconciling, + release, + triggering, + }); + + return ( + +
+ + + {triggerError ? ( +

+ {triggerError} +

+ ) : null} +
+
+ ); +}; + +export default UpgradeOperationModal; diff --git a/client/src/protoFleet/features/updates/api/useUpgradeOperation.test.tsx b/client/src/protoFleet/features/updates/api/useUpgradeOperation.test.tsx new file mode 100644 index 0000000000..074c869da0 --- /dev/null +++ b/client/src/protoFleet/features/updates/api/useUpgradeOperation.test.tsx @@ -0,0 +1,708 @@ +import { act, renderHook, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { create } from "@bufbuild/protobuf"; +import { TimestampSchema } from "@bufbuild/protobuf/wkt"; +import { Code, ConnectError } from "@connectrpc/connect"; + +import { instanceUpdateClient } from "@/protoFleet/api/clients"; +import { + GetUpgradeStatusResponseSchema, + TriggerUpgradeResponseSchema, + type UpgradeOperation, + UpgradeOperationSchema, + UpgradePhase, +} from "@/protoFleet/api/generated/instance/v1/updates_pb"; +import { isUpgradeActive, useUpgradeOperation } from "@/protoFleet/features/updates/api/useUpgradeOperation"; + +vi.mock("@/protoFleet/api/clients", () => ({ + instanceUpdateClient: { + getUpgradeStatus: vi.fn(), + triggerUpgrade: vi.fn(), + }, +})); + +const mockGetUpgradeStatus = vi.mocked(instanceUpdateClient.getUpgradeStatus); +const mockTriggerUpgrade = vi.mocked(instanceUpdateClient.triggerUpgrade); +const TRACKED_OPERATION_KEY = "protoFleet:tracked-upgrade-operation"; +const ACKNOWLEDGED_OPERATION_KEY = "protoFleet:acknowledged-upgrade-operation"; +const AUTH_SESSION_IDENTITY = "operator-a:1"; + +type TestUpgradeOperationOptions = Omit< + Parameters[0], + "authSessionIdentity" | "currentVersionUnavailable" +> & { + currentVersionUnavailable?: boolean; +}; + +const useTestUpgradeOperation = (options: TestUpgradeOperationOptions, authSessionIdentity = AUTH_SESSION_IDENTITY) => + useUpgradeOperation({ authSessionIdentity, currentVersionUnavailable: false, ...options }); + +type MessageOverrides = Omit, "$typeName" | "$unknown">; + +const timestamp = (seconds: number) => create(TimestampSchema, { seconds: BigInt(seconds) }); + +const operation = (phase: UpgradePhase, overrides?: MessageOverrides) => + create(UpgradeOperationSchema, { + id: "operation-1", + targetVersion: "v1.3.0", + phase, + message: "Preparing upgrade", + startedAt: timestamp(100), + updatedAt: timestamp(100), + ...overrides, + }); + +const status = (executorAvailable = true, currentOperation?: UpgradeOperation) => + create(GetUpgradeStatusResponseSchema, { + executorAvailable, + operation: currentOperation, + }); + +const deferred = () => { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, reject, resolve }; +}; + +beforeEach(() => { + vi.clearAllMocks(); + window.sessionStorage.clear(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("useUpgradeOperation", () => { + it("recovers a durable active operation without a capability gate", async () => { + const activeOperation = operation(UpgradePhase.PREFLIGHT); + window.sessionStorage.setItem( + TRACKED_OPERATION_KEY, + JSON.stringify({ id: activeOperation.id, targetVersion: activeOperation.targetVersion }), + ); + mockGetUpgradeStatus.mockResolvedValue(status(true, activeOperation)); + + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + expect(result.current.reconciling).toBe(true); + await waitFor(() => expect(result.current.operation?.id).toBe("operation-1")); + expect(result.current.reconciling).toBe(false); + expect(result.current.connectionLost).toBe(false); + expect(mockGetUpgradeStatus).toHaveBeenCalledWith( + {}, + expect.objectContaining({ signal: expect.any(AbortSignal), timeoutMs: 10_000 }), + ); + }); + + it("removes a malformed tracked-operation record", () => { + window.sessionStorage.setItem(TRACKED_OPERATION_KEY, "{not-json"); + mockGetUpgradeStatus.mockResolvedValue(status()); + + renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); + }); + + it("does not overlap a pending poll and aborts it on cleanup", async () => { + vi.useFakeTimers(); + const request = deferred>(); + mockGetUpgradeStatus.mockReturnValue(request.promise); + + const hook = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await act(async () => Promise.resolve()); + expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1); + + await act(async () => vi.advanceTimersByTimeAsync(120_000)); + expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1); + const signal = mockGetUpgradeStatus.mock.calls[0]?.[1]?.signal; + + hook.unmount(); + expect(signal?.aborted).toBe(true); + }); + + it("keeps operation status pending until the first authoritative response", async () => { + const request = deferred>(); + mockGetUpgradeStatus.mockReturnValue(request.promise); + + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + expect(result.current.operationStatusPending).toBe(true); + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1)); + + await act(async () => { + request.resolve(status()); + await request.promise; + }); + + expect(result.current.operationStatusPending).toBe(false); + }); + + it("restarts polling when the authenticated session generation changes", async () => { + const previousRequest = deferred>(); + const onPollError = vi.fn(); + mockGetUpgradeStatus.mockReturnValueOnce(previousRequest.promise).mockResolvedValue(status()); + + const { rerender, result } = renderHook( + ({ authSessionIdentity }: { authSessionIdentity: string }) => + useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0", onPollError }, authSessionIdentity), + { initialProps: { authSessionIdentity: AUTH_SESSION_IDENTITY } }, + ); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1)); + const previousSignal = mockGetUpgradeStatus.mock.calls[0]?.[1]?.signal; + const abortSpy = vi.spyOn(AbortController.prototype, "abort").mockImplementation(() => undefined); + + rerender({ authSessionIdentity: "operator-a:2" }); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(2)); + expect(abortSpy).toHaveBeenCalled(); + expect(previousSignal?.aborted).toBe(false); + await waitFor(() => expect(result.current.operationStatusPending).toBe(false)); + + await act(async () => { + previousRequest.reject(new Error("previous session expired")); + await Promise.resolve(); + }); + + expect(onPollError).not.toHaveBeenCalled(); + abortSpy.mockRestore(); + }); + + it("starts the exact target and tracks the returned operation", async () => { + const activeOperation = operation(UpgradePhase.PREFLIGHT); + mockGetUpgradeStatus.mockResolvedValue(status()); + mockTriggerUpgrade.mockResolvedValue( + create(TriggerUpgradeResponseSchema, { + operation: activeOperation, + }), + ); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalled()); + await act(async () => result.current.triggerUpgrade("v1.3.0")); + + expect(mockTriggerUpgrade).toHaveBeenCalledWith({ targetVersion: "v1.3.0" }, { timeoutMs: 30_000 }); + expect(result.current.operation?.phase).toBe(UpgradePhase.PREFLIGHT); + expect(JSON.parse(window.sessionStorage.getItem(TRACKED_OPERATION_KEY) ?? "{}")).toEqual({ + id: "operation-1", + targetVersion: "v1.3.0", + }); + }); + + it("keeps an unknown phase locked until explicit host confirmation", async () => { + vi.useFakeTimers(); + const unknownOperation = operation(UpgradePhase.UNSPECIFIED); + mockGetUpgradeStatus.mockResolvedValueOnce(status()).mockResolvedValue(status(true, unknownOperation)); + mockTriggerUpgrade.mockResolvedValue( + create(TriggerUpgradeResponseSchema, { + operation: unknownOperation, + }), + ); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await act(async () => Promise.resolve()); + + await act(async () => result.current.triggerUpgrade("v1.3.0")); + + expect(result.current.reconciling).toBe(true); + expect(result.current.operation?.phase).toBe(UpgradePhase.UNSPECIFIED); + expect(isUpgradeActive(result.current.operation)).toBe(true); + expect(result.current.triggerError).toBeNull(); + expect(JSON.parse(window.sessionStorage.getItem(TRACKED_OPERATION_KEY) ?? "{}")).toEqual({ + id: "operation-1", + targetVersion: "v1.3.0", + }); + + await act(async () => vi.advanceTimersByTimeAsync(17_000)); + + expect(result.current.reconciling).toBe(true); + expect(result.current.manualFallbackReady).toBe(true); + expect(result.current.operation?.phase).toBe(UpgradePhase.UNSPECIFIED); + + act(() => result.current.useManualFallback()); + + expect(result.current.reconciling).toBe(false); + expect(result.current.operation).toBeUndefined(); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); + expect(JSON.parse(window.sessionStorage.getItem(ACKNOWLEDGED_OPERATION_KEY) ?? "{}")).toEqual({ + authSessionIdentity: AUTH_SESSION_IDENTITY, + id: "operation-1", + phase: UpgradePhase.UNSPECIFIED, + revision: "100:0", + }); + + await act(async () => vi.advanceTimersByTimeAsync(2_000)); + expect(result.current.operation).toBeUndefined(); + }); + + it.each([UpgradePhase.FAILED, UpgradePhase.SUCCEEDED])( + "surfaces terminal phase %s after manually unlocking an unknown operation", + async (terminalPhase) => { + vi.useFakeTimers(); + const unknownOperation = operation(UpgradePhase.UNSPECIFIED); + const terminalOperation = operation(terminalPhase); + mockGetUpgradeStatus.mockResolvedValueOnce(status()).mockResolvedValue(status(true, unknownOperation)); + mockTriggerUpgrade.mockResolvedValue( + create(TriggerUpgradeResponseSchema, { + operation: unknownOperation, + }), + ); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await act(async () => Promise.resolve()); + + await act(async () => result.current.triggerUpgrade("v1.3.0")); + await act(async () => vi.advanceTimersByTimeAsync(17_000)); + act(() => result.current.useManualFallback()); + + mockGetUpgradeStatus.mockReset(); + mockGetUpgradeStatus.mockResolvedValue(status(true, terminalOperation)); + await act(async () => vi.advanceTimersByTimeAsync(2_000)); + + expect(result.current.operation?.phase).toBe(terminalPhase); + expect(window.sessionStorage.getItem(ACKNOWLEDGED_OPERATION_KEY)).toBeNull(); + }, + ); + + it("accepts a terminal operation returned directly by the trigger RPC", async () => { + const failedOperation = operation(UpgradePhase.FAILED, { message: "Preflight failed" }); + mockGetUpgradeStatus.mockResolvedValue(status()); + mockTriggerUpgrade.mockResolvedValue( + create(TriggerUpgradeResponseSchema, { + operation: failedOperation, + }), + ); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalled()); + + await act(async () => result.current.triggerUpgrade("v1.3.0")); + + expect(result.current.operation?.id).toBe("operation-1"); + expect(result.current.operation?.phase).toBe(UpgradePhase.FAILED); + expect(result.current.reconciling).toBe(false); + }); + + it("reconciles an ambiguous trigger rejection to the durable operation", async () => { + const activeOperation = operation(UpgradePhase.PREFLIGHT); + mockGetUpgradeStatus.mockResolvedValueOnce(status()).mockResolvedValue(status(true, activeOperation)); + mockTriggerUpgrade.mockRejectedValue(new Error("response lost")); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1)); + await act(async () => result.current.triggerUpgrade("v1.3.0")); + await waitFor(() => expect(result.current.operation?.id).toBe("operation-1")); + + expect(result.current.reconciling).toBe(false); + expect(result.current.triggerError).toBeNull(); + }); + + it("unlocks immediately when the server definitively rejects a stale target", async () => { + mockGetUpgradeStatus.mockResolvedValue(status()); + mockTriggerUpgrade.mockRejectedValue( + new ConnectError('target "v1.3.0" is no longer the eligible update', Code.FailedPrecondition), + ); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalled()); + await act(async () => result.current.triggerUpgrade("v1.3.0")); + + expect(result.current.reconciling).toBe(false); + expect(result.current.trackedTargetVersion).toBeUndefined(); + expect(result.current.triggerError).toContain("no longer the eligible update"); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); + }); + + it("reconciles an already-existing operation instead of treating the rejection as safe to unlock", async () => { + mockGetUpgradeStatus.mockResolvedValue(status()); + mockTriggerUpgrade.mockRejectedValue(new ConnectError("another upgrade is active", Code.AlreadyExists)); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalled()); + await act(async () => result.current.triggerUpgrade("v1.3.0")); + + expect(result.current.reconciling).toBe(true); + expect(result.current.trackedTargetVersion).toBe("v1.3.0"); + }); + + it("reconciles an ambiguous trigger rejection to a completed upgrade", async () => { + const succeededOperation = operation(UpgradePhase.SUCCEEDED, { message: "Upgrade complete" }); + mockGetUpgradeStatus.mockResolvedValueOnce(status()).mockResolvedValue(status(true, succeededOperation)); + mockTriggerUpgrade.mockRejectedValue(new Error("response lost")); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1)); + await act(async () => result.current.triggerUpgrade("v1.3.0")); + await waitFor(() => expect(result.current.operation?.phase).toBe(UpgradePhase.SUCCEEDED)); + + expect(result.current.operation?.id).toBe("operation-1"); + expect(result.current.reconciling).toBe(false); + expect(result.current.triggerError).toBeNull(); + }); + + it("reconciles an ambiguous trigger rejection to a newer same-target failure", async () => { + const previousFailure = operation(UpgradePhase.FAILED, { id: "operation-old" }); + const failedOperation = operation(UpgradePhase.FAILED, { + id: "operation-new", + updatedAt: timestamp(101), + recoveryCommand: "./run-fleet.sh --skip-build", + }); + window.sessionStorage.setItem( + ACKNOWLEDGED_OPERATION_KEY, + JSON.stringify({ + authSessionIdentity: AUTH_SESSION_IDENTITY, + id: previousFailure.id, + phase: previousFailure.phase, + revision: "100:0", + }), + ); + mockGetUpgradeStatus + .mockResolvedValueOnce(status(true, previousFailure)) + .mockResolvedValue(status(true, failedOperation)); + mockTriggerUpgrade.mockRejectedValue(new Error("response lost")); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1)); + await act(async () => result.current.triggerUpgrade("v1.3.0")); + await waitFor(() => expect(result.current.operation?.id).toBe("operation-new")); + + expect(result.current.operation?.phase).toBe(UpgradePhase.FAILED); + expect(result.current.operation?.recoveryCommand).toBe("./run-fleet.sh --skip-build"); + expect(result.current.reconciling).toBe(false); + }); + + it("does not let a stale terminal failure resolve an ambiguous trigger", async () => { + const staleFailure = operation(UpgradePhase.FAILED, { + id: "operation-old", + targetVersion: "v1.3.0", + }); + window.sessionStorage.setItem( + ACKNOWLEDGED_OPERATION_KEY, + JSON.stringify({ + authSessionIdentity: AUTH_SESSION_IDENTITY, + id: staleFailure.id, + phase: staleFailure.phase, + revision: "100:0", + }), + ); + mockGetUpgradeStatus.mockResolvedValue(status(true, staleFailure)); + mockTriggerUpgrade.mockRejectedValue(new Error("response lost")); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1)); + await act(async () => result.current.triggerUpgrade("v1.3.0")); + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(2)); + + expect(result.current.operation).toBeUndefined(); + expect(result.current.reconciling).toBe(true); + expect(result.current.trackedTargetVersion).toBe("v1.3.0"); + }); + + it("ends bounded reconciliation and preserves an actionable trigger error", async () => { + vi.useFakeTimers(); + mockGetUpgradeStatus.mockResolvedValue(status()); + mockTriggerUpgrade.mockRejectedValue(new Error("host did not confirm")); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await act(async () => Promise.resolve()); + + await act(async () => result.current.triggerUpgrade("v1.3.0")); + expect(result.current.reconciling).toBe(true); + + await act(async () => vi.advanceTimersByTimeAsync(17_000)); + + expect(result.current.reconciling).toBe(false); + expect(result.current.triggerError).toContain("host did not confirm"); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); + }); + + it("does not unlock an unknown trigger outcome while the executor is unreachable", async () => { + vi.useFakeTimers(); + mockGetUpgradeStatus.mockResolvedValueOnce(status()).mockResolvedValue(status(false)); + mockTriggerUpgrade.mockRejectedValue(new Error("host did not confirm")); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await act(async () => Promise.resolve()); + + await act(async () => result.current.triggerUpgrade("v1.3.0")); + await act(async () => vi.advanceTimersByTimeAsync(30_000)); + + expect(result.current.reconciling).toBe(true); + expect(result.current.connectionLost).toBe(true); + expect(result.current.manualFallbackReady).toBe(true); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).not.toBeNull(); + + act(() => result.current.useManualFallback()); + + expect(result.current.reconciling).toBe(false); + expect(result.current.triggerError).toBeNull(); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); + }); + + it("keeps a recovered operation ID locked until the unreachable host is explicitly confirmed", async () => { + vi.useFakeTimers(); + window.sessionStorage.setItem( + TRACKED_OPERATION_KEY, + JSON.stringify({ id: "operation-1", targetVersion: "v1.3.0" }), + ); + mockGetUpgradeStatus.mockResolvedValue(status(false)); + + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + expect(result.current.reconciling).toBe(true); + + await act(async () => vi.advanceTimersByTimeAsync(17_000)); + + expect(result.current.reconciling).toBe(true); + expect(result.current.connectionLost).toBe(true); + expect(result.current.manualFallbackReady).toBe(true); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).not.toBeNull(); + + act(() => result.current.useManualFallback()); + + expect(result.current.reconciling).toBe(false); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); + }); + + it("unlocks a recovered operation ID after an authoritative reachable miss", async () => { + window.sessionStorage.setItem( + TRACKED_OPERATION_KEY, + JSON.stringify({ id: "operation-1", targetVersion: "v1.3.0" }), + ); + mockGetUpgradeStatus.mockResolvedValue(status(true)); + + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + expect(result.current.reconciling).toBe(true); + + await waitFor(() => expect(result.current.reconciling).toBe(false)); + + expect(result.current.connectionLost).toBe(false); + expect(result.current.manualFallbackReady).toBe(false); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); + }); + + it("keeps progress through disconnect and accepts the terminal success", async () => { + vi.useFakeTimers(); + const activeOperation = operation(UpgradePhase.ACTIVATING); + const succeededOperation = operation(UpgradePhase.SUCCEEDED, { message: "Upgrade complete" }); + mockGetUpgradeStatus + .mockResolvedValueOnce(status(true, activeOperation)) + .mockRejectedValueOnce(new Error("Fleet restarting")) + .mockResolvedValue(status(true, succeededOperation)); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await act(async () => Promise.resolve()); + expect(result.current.operation?.phase).toBe(UpgradePhase.ACTIVATING); + + await act(async () => vi.advanceTimersByTimeAsync(2_000)); + expect(result.current.connectionLost).toBe(true); + expect(result.current.operation?.phase).toBe(UpgradePhase.ACTIVATING); + + await act(async () => vi.advanceTimersByTimeAsync(2_000)); + expect(result.current.operation?.phase).toBe(UpgradePhase.SUCCEEDED); + expect(result.current.connectionLost).toBe(false); + }); + + it.each([ + ["an unreachable executor", status(false)], + ["a reachable executor with no operation", status(true)], + ])("offers explicit fallback when an active operation is lost by %s", async (_scenario, missingStatus) => { + vi.useFakeTimers(); + const activeOperation = operation(UpgradePhase.ACTIVATING); + mockGetUpgradeStatus.mockResolvedValueOnce(status(true, activeOperation)).mockResolvedValue(missingStatus); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await act(async () => Promise.resolve()); + expect(result.current.operation?.phase).toBe(UpgradePhase.ACTIVATING); + + await act(async () => vi.advanceTimersByTimeAsync(2_000)); + expect(result.current.reconciling).toBe(true); + expect(result.current.connectionLost).toBe(true); + expect(result.current.manualFallbackReady).toBe(false); + + await act(async () => vi.advanceTimersByTimeAsync(16_000)); + expect(result.current.manualFallbackReady).toBe(true); + expect(result.current.operation?.phase).toBe(UpgradePhase.ACTIVATING); + + act(() => result.current.useManualFallback()); + expect(result.current.reconciling).toBe(false); + expect(result.current.operation).toBeUndefined(); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).toBeNull(); + }); + + it("does not replay an untracked historical success", async () => { + mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.SUCCEEDED))); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.3.0" })); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalled()); + expect(result.current.operation).toBeUndefined(); + }); + + it("requests one authoritative offer refresh for an untracked success revision", async () => { + vi.useFakeTimers(); + const onUntrackedSuccess = vi.fn(); + mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.SUCCEEDED))); + const { result } = renderHook(() => + useTestUpgradeOperation({ + enabled: true, + currentVersion: "v1.2.0", + onUntrackedSuccess, + }), + ); + + await act(async () => Promise.resolve()); + expect(onUntrackedSuccess).toHaveBeenCalledTimes(1); + expect(result.current.operation).toBeUndefined(); + + await act(async () => vi.advanceTimersByTimeAsync(60_000)); + expect(onUntrackedSuccess).toHaveBeenCalledTimes(1); + }); + + it("keeps a tracked activation failure visible when Fleet reports its target version", async () => { + window.sessionStorage.setItem( + TRACKED_OPERATION_KEY, + JSON.stringify({ id: "operation-1", targetVersion: "v1.3.0" }), + ); + mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.FAILED))); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.3.0" })); + + await waitFor(() => expect(result.current.operation?.phase).toBe(UpgradePhase.FAILED)); + expect(window.sessionStorage.getItem(TRACKED_OPERATION_KEY)).not.toBeNull(); + }); + + it("keeps an untracked activation failure visible when Fleet reports its target version", async () => { + mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.FAILED))); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.3.0" })); + + await waitFor(() => expect(result.current.operation?.phase).toBe(UpgradePhase.FAILED)); + }); + + it("suppresses a stale failure after manual recovery installed a newer release", async () => { + mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.FAILED))); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.4.0" })); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalled()); + expect(result.current.operation).toBeUndefined(); + }); + + it("treats a stable release as newer than its failed release candidate", async () => { + mockGetUpgradeStatus.mockResolvedValue( + status(true, operation(UpgradePhase.FAILED, { targetVersion: "v1.3.0-rc.4" })), + ); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.3.0" })); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalled()); + expect(result.current.operation).toBeUndefined(); + }); + + it("retains a failed operation when the current version is not canonical", async () => { + mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.FAILED))); + const { result } = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "dev" })); + + await waitFor(() => expect(result.current.operation?.phase).toBe(UpgradePhase.FAILED)); + }); + + it("rechecks an unresolved failure as soon as the current version loads", async () => { + mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.FAILED))); + const initialProps: { currentVersion?: string } = {}; + const { rerender, result } = renderHook( + ({ currentVersion }: { currentVersion?: string }) => useTestUpgradeOperation({ enabled: true, currentVersion }), + { initialProps }, + ); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1)); + expect(result.current.operation).toBeUndefined(); + + rerender({ currentVersion: "v1.2.0" }); + + await waitFor(() => expect(result.current.operation?.phase).toBe(UpgradePhase.FAILED)); + }); + + it("shows an untracked failure after current-version loading definitively fails", async () => { + const failedOperation = operation(UpgradePhase.FAILED, { + hostLogPath: "/var/lib/proto-fleet-updater/logs/operation-1.log", + recoveryCommand: "./run-fleet.sh --skip-build", + }); + mockGetUpgradeStatus.mockResolvedValue(status(true, failedOperation)); + const { rerender, result } = renderHook( + ({ currentVersionUnavailable }: { currentVersionUnavailable: boolean }) => + useTestUpgradeOperation({ enabled: true, currentVersionUnavailable }), + { initialProps: { currentVersionUnavailable: false } }, + ); + + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(1)); + expect(result.current.operation).toBeUndefined(); + + rerender({ currentVersionUnavailable: true }); + + await waitFor(() => expect(result.current.operation?.phase).toBe(UpgradePhase.FAILED)); + expect(result.current.operation?.recoveryCommand).toBe("./run-fleet.sh --skip-build"); + expect(result.current.operation?.hostLogPath).toContain("operation-1.log"); + }); + + it("scopes an acknowledged terminal failure to the authenticated session", async () => { + mockGetUpgradeStatus.mockResolvedValue(status(true, operation(UpgradePhase.FAILED))); + const firstSession = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await waitFor(() => expect(firstSession.result.current.operation?.phase).toBe(UpgradePhase.FAILED)); + + act(() => firstSession.result.current.acknowledgeOperation()); + + expect(firstSession.result.current.operation).toBeUndefined(); + const acknowledgedRecord = JSON.parse(window.sessionStorage.getItem(ACKNOWLEDGED_OPERATION_KEY) ?? "{}"); + expect(acknowledgedRecord).toEqual({ + authSessionIdentity: AUTH_SESSION_IDENTITY, + id: "operation-1", + phase: UpgradePhase.FAILED, + revision: "100:0", + }); + firstSession.unmount(); + + const sameSession = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await waitFor(() => expect(mockGetUpgradeStatus).toHaveBeenCalledTimes(2)); + expect(sameSession.result.current.operation).toBeUndefined(); + sameSession.unmount(); + + const nextSession = renderHook(() => + useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" }, "operator-a:2"), + ); + await waitFor(() => expect(nextSession.result.current.operation?.phase).toBe(UpgradePhase.FAILED)); + expect(JSON.parse(window.sessionStorage.getItem(ACKNOWLEDGED_OPERATION_KEY) ?? "{}")).toEqual(acknowledgedRecord); + }); + + it("resurfaces acknowledged failure details when startup reconciliation advances the revision", async () => { + const originalFailure = operation(UpgradePhase.FAILED, { recoveryCommand: "old recovery" }); + mockGetUpgradeStatus.mockResolvedValue(status(true, originalFailure)); + const firstSession = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + await waitFor(() => expect(firstSession.result.current.operation?.phase).toBe(UpgradePhase.FAILED)); + + act(() => firstSession.result.current.acknowledgeOperation()); + firstSession.unmount(); + + mockGetUpgradeStatus.mockReset(); + mockGetUpgradeStatus.mockResolvedValue( + status( + true, + operation(UpgradePhase.FAILED, { + updatedAt: timestamp(101), + recoveryCommand: "new recovery", + }), + ), + ); + const reconciledSession = renderHook(() => useTestUpgradeOperation({ enabled: true, currentVersion: "v1.2.0" })); + + await waitFor(() => expect(reconciledSession.result.current.operation?.recoveryCommand).toBe("new recovery")); + expect(window.sessionStorage.getItem(ACKNOWLEDGED_OPERATION_KEY)).toBeNull(); + }); + + it("forwards poll errors for page-level auth handling", async () => { + const onPollError = vi.fn(); + const pollError = new Error("permission revoked"); + mockGetUpgradeStatus.mockRejectedValue(pollError); + + renderHook(() => + useTestUpgradeOperation({ + enabled: true, + currentVersion: "v1.2.0", + onPollError, + }), + ); + + await waitFor(() => expect(onPollError).toHaveBeenCalledWith(pollError)); + }); +}); diff --git a/client/src/protoFleet/features/updates/api/useUpgradeOperation.ts b/client/src/protoFleet/features/updates/api/useUpgradeOperation.ts new file mode 100644 index 0000000000..ecf62eb36c --- /dev/null +++ b/client/src/protoFleet/features/updates/api/useUpgradeOperation.ts @@ -0,0 +1,611 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { Code, ConnectError } from "@connectrpc/connect"; + +import { instanceUpdateClient } from "@/protoFleet/api/clients"; +import { type UpgradeOperation, UpgradePhase } from "@/protoFleet/api/generated/instance/v1/updates_pb"; +import { getErrorMessage } from "@/protoFleet/api/getErrorMessage"; + +const ACTIVE_POLL_INTERVAL_MS = 2_000; +const IDLE_POLL_INTERVAL_MS = 60_000; +const STATUS_REQUEST_TIMEOUT_MS = 10_000; +const TRIGGER_REQUEST_TIMEOUT_MS = 30_000; +const TRIGGER_RECONCILIATION_TIMEOUT_MS = 15_000; +const TRACKED_OPERATION_KEY = "protoFleet:tracked-upgrade-operation"; +const ACKNOWLEDGED_OPERATION_KEY = "protoFleet:acknowledged-upgrade-operation"; +const CANONICAL_RELEASE_PATTERN = /^v(\d+)\.(\d+)\.(\d+)(?:-rc\.(\d+))?$/; +const DEFINITIVE_TRIGGER_REJECTION_CODES = new Set([ + Code.InvalidArgument, + Code.FailedPrecondition, + Code.PermissionDenied, + Code.Unauthenticated, + Code.Unimplemented, +]); + +const isDefinitiveTriggerRejection = (error: unknown) => + error instanceof ConnectError && DEFINITIVE_TRIGGER_REJECTION_CODES.has(error.code); + +interface CanonicalRelease { + core: [number, number, number]; + rc?: number; +} + +const parseCanonicalRelease = (version?: string): CanonicalRelease | undefined => { + const match = version?.match(CANONICAL_RELEASE_PATTERN); + if (!match) return undefined; + const parts = match.slice(1).map((part) => (part === undefined ? undefined : Number(part))); + if (parts.some((part) => part !== undefined && !Number.isSafeInteger(part))) return undefined; + return { + core: [parts[0]!, parts[1]!, parts[2]!], + ...(parts[3] === undefined ? {} : { rc: parts[3] }), + }; +}; + +const isReleaseNewer = (currentVersion: string | undefined, targetVersion: string) => { + const current = parseCanonicalRelease(currentVersion); + const target = parseCanonicalRelease(targetVersion); + if (!current || !target) return false; + for (let index = 0; index < current.core.length; index += 1) { + if (current.core[index] !== target.core[index]) { + return current.core[index] > target.core[index]; + } + } + if (current.rc === undefined) return target.rc !== undefined; + if (target.rc === undefined) return false; + return current.rc > target.rc; +}; + +interface TrackedOperation { + id?: string; + targetVersion: string; +} + +interface AcknowledgedOperation { + authSessionIdentity: string; + id: string; + phase: UpgradePhase; + revision: string; +} + +type AcknowledgedOperationInput = Pick; + +const ACKNOWLEDGEABLE_PHASES = new Set([ + UpgradePhase.UNSPECIFIED, + UpgradePhase.SUCCEEDED, + UpgradePhase.FAILED, +]); + +interface UseUpgradeOperationOptions { + authSessionIdentity: string; + currentVersion?: string; + currentVersionUnavailable: boolean; + enabled: boolean; + onUntrackedSuccess?: (operation: UpgradeOperation) => void; + onPollError?: (error: unknown) => void; +} + +interface UseUpgradeOperationResult { + acknowledgeOperation: () => void; + connectionLost: boolean; + manualFallbackReady: boolean; + operation: UpgradeOperation | undefined; + operationStatusPending: boolean; + reconciling: boolean; + reloadFleet: () => void; + triggerError: string | null; + triggering: boolean; + trackedTargetVersion: string | undefined; + triggerUpgrade: (targetVersion: string) => Promise; + useManualFallback: () => void; +} + +export const isUpgradeTerminal = (phase: UpgradePhase) => + phase === UpgradePhase.SUCCEEDED || phase === UpgradePhase.FAILED; + +export const isUpgradeActive = (operation?: UpgradeOperation) => + Boolean(operation && !isUpgradeTerminal(operation.phase)); + +const operationRevision = (operation: UpgradeOperation) => { + const updatedAt = operation.updatedAt; + if (updatedAt) { + return `${updatedAt.seconds}:${updatedAt.nanos}`; + } + return JSON.stringify([operation.message, operation.error, operation.recoveryCommand, operation.hostLogPath]); +}; + +const readTrackedOperation = (): TrackedOperation | undefined => { + try { + const raw = window.sessionStorage.getItem(TRACKED_OPERATION_KEY); + if (!raw) return undefined; + const value = JSON.parse(raw) as Partial; + if (typeof value.targetVersion !== "string" || !value.targetVersion) { + window.sessionStorage.removeItem(TRACKED_OPERATION_KEY); + return undefined; + } + return { + targetVersion: value.targetVersion, + ...(typeof value.id === "string" && value.id ? { id: value.id } : {}), + }; + } catch { + try { + window.sessionStorage.removeItem(TRACKED_OPERATION_KEY); + } catch { + // Storage is best-effort; the host remains authoritative. + } + return undefined; + } +}; + +const readAcknowledgedOperation = (authSessionIdentity: string): AcknowledgedOperationInput | null => { + try { + const raw = window.sessionStorage.getItem(ACKNOWLEDGED_OPERATION_KEY); + if (!raw) return null; + const value = JSON.parse(raw) as Partial; + if ( + typeof value.id === "string" && + value.id && + value.authSessionIdentity === authSessionIdentity && + typeof value.phase === "number" && + ACKNOWLEDGEABLE_PHASES.has(value.phase) && + typeof value.revision === "string" + ) { + return { id: value.id, phase: value.phase, revision: value.revision }; + } + return null; + } catch { + try { + window.sessionStorage.removeItem(ACKNOWLEDGED_OPERATION_KEY); + } catch { + // Storage is best-effort; the host remains authoritative. + } + return null; + } +}; + +export function useUpgradeOperation({ + authSessionIdentity, + currentVersion, + currentVersionUnavailable, + enabled, + onUntrackedSuccess, + onPollError, +}: UseUpgradeOperationOptions): UseUpgradeOperationResult { + const [operation, setOperation] = useState(); + const [triggering, setTriggering] = useState(false); + const [trackedOperation, setTrackedOperation] = useState(readTrackedOperation); + const recoveredTrackedOperation = Boolean(trackedOperation); + const recoveredUnknownOutcome = Boolean(trackedOperation && !trackedOperation.id); + const [reconciling, setReconciling] = useState(recoveredTrackedOperation); + const [connectionLost, setConnectionLost] = useState(false); + const [manualFallbackReady, setManualFallbackReady] = useState(false); + const [triggerError, setTriggerError] = useState( + recoveredUnknownOutcome ? "Fleet did not confirm whether the previous upgrade request started" : null, + ); + const [pollRevision, setPollRevision] = useState(0); + const [resolvedStatusSessionIdentity, setResolvedStatusSessionIdentity] = useState(null); + + const operationRef = useRef(operation); + const trackedOperationRef = useRef(trackedOperation); + const acknowledgedOperationRef = useRef(null); + const acknowledgedOperationSessionRef = useRef(null); + const reconcilingRef = useRef(reconciling); + const currentVersionRef = useRef(currentVersion); + const currentVersionUnavailableRef = useRef(currentVersionUnavailable); + const onPollErrorRef = useRef(onPollError); + const onUntrackedSuccessRef = useRef(onUntrackedSuccess); + const reconciliationDeadlineRef = useRef(null); + const lastObservedOperationIDRef = useRef(undefined); + const triggerBaselineOperationIDRef = useRef(undefined); + const refreshedUntrackedSuccessRef = useRef(null); + const authSessionIdentityRef = useRef(authSessionIdentity); + const resolvedStatusSessionIdentityRef = useRef(null); + + currentVersionRef.current = currentVersion; + currentVersionUnavailableRef.current = currentVersionUnavailable; + onPollErrorRef.current = onPollError; + onUntrackedSuccessRef.current = onUntrackedSuccess; + authSessionIdentityRef.current = authSessionIdentity; + if (acknowledgedOperationSessionRef.current !== authSessionIdentity) { + acknowledgedOperationSessionRef.current = authSessionIdentity; + acknowledgedOperationRef.current = readAcknowledgedOperation(authSessionIdentity); + } + + const operationStatusPending = enabled && resolvedStatusSessionIdentity !== authSessionIdentity; + + useEffect(() => { + if (trackedOperationRef.current) { + reconciliationDeadlineRef.current = Date.now() + TRIGGER_RECONCILIATION_TIMEOUT_MS; + } + }, []); + + const updateOperation = useCallback((next: UpgradeOperation | undefined) => { + operationRef.current = next; + setOperation(next); + }, []); + + const updateReconciling = useCallback((next: boolean) => { + reconcilingRef.current = next; + setReconciling(next); + }, []); + + const updateTrackedOperation = useCallback((next: TrackedOperation | undefined) => { + trackedOperationRef.current = next; + setTrackedOperation(next); + try { + if (next) { + window.sessionStorage.setItem(TRACKED_OPERATION_KEY, JSON.stringify(next)); + } else { + window.sessionStorage.removeItem(TRACKED_OPERATION_KEY); + } + } catch { + // Browser storage is an optimization for route/reload recovery. The + // host operation remains durable when storage is unavailable. + } + }, []); + + const updateAcknowledgedOperation = useCallback((next: AcknowledgedOperationInput | null) => { + acknowledgedOperationRef.current = next; + try { + if (next) { + window.sessionStorage.setItem( + ACKNOWLEDGED_OPERATION_KEY, + JSON.stringify({ authSessionIdentity: authSessionIdentityRef.current, ...next }), + ); + } else { + window.sessionStorage.removeItem(ACKNOWLEDGED_OPERATION_KEY); + } + } catch { + // See updateTrackedOperation: storage failure cannot affect host state. + } + }, []); + + const finishExpiredReconciliation = useCallback(() => { + const deadline = reconciliationDeadlineRef.current; + if (!reconcilingRef.current || deadline === null || Date.now() < deadline) { + return false; + } + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + if (!trackedOperationRef.current?.id) { + updateTrackedOperation(undefined); + } + return true; + }, [updateReconciling, updateTrackedOperation]); + + const allowManualFallbackAfterTimeout = useCallback(() => { + const deadline = reconciliationDeadlineRef.current; + if (reconcilingRef.current && deadline !== null && Date.now() >= deadline) { + setManualFallbackReady(true); + } + }, []); + + const reconcileMissingActiveOperation = useCallback(() => { + if (!isUpgradeActive(operationRef.current)) { + return false; + } + if (!reconcilingRef.current) { + reconciliationDeadlineRef.current = Date.now() + TRIGGER_RECONCILIATION_TIMEOUT_MS; + setManualFallbackReady(false); + updateReconciling(true); + } + setConnectionLost(true); + allowManualFallbackAfterTimeout(); + return true; + }, [allowManualFallbackAfterTimeout, updateReconciling]); + + const resolveRecoveredOperationMiss = useCallback(() => { + if (!reconcilingRef.current || !trackedOperationRef.current?.id) { + return false; + } + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateTrackedOperation(undefined); + updateReconciling(false); + setConnectionLost(false); + setTriggerError(null); + return true; + }, [updateReconciling, updateTrackedOperation]); + + const acceptServerOperation = useCallback( + (next: UpgradeOperation, fromTriggerResponse = false) => { + if (!next.id) { + return false; + } + const acknowledged = acknowledgedOperationRef.current; + const revision = operationRevision(next); + const acknowledgedExactRevision = + acknowledged?.id === next.id && acknowledged.phase === next.phase && acknowledged.revision === revision; + const acknowledgedTransition = acknowledged?.id === next.id && !acknowledgedExactRevision; + if (acknowledgedExactRevision) { + return false; + } + const active = isUpgradeActive(next); + const tracked = trackedOperationRef.current; + const reconciledTerminal = + reconcilingRef.current && + !tracked?.id && + tracked?.targetVersion === next.targetVersion && + isUpgradeTerminal(next.phase) && + triggerBaselineOperationIDRef.current !== undefined && + triggerBaselineOperationIDRef.current !== next.id; + const trackedMatches = tracked?.id + ? tracked.id === next.id + : (fromTriggerResponse && tracked?.targetVersion === next.targetVersion) || + reconciledTerminal || + acknowledgedTransition; + + if (active) { + // A durable active operation is authoritative, including one started + // by another operator. It takes precedence over a newer release offer. + updateAcknowledgedOperation(null); + updateTrackedOperation({ id: next.id, targetVersion: next.targetVersion }); + updateOperation(next); + if (next.phase === UpgradePhase.UNSPECIFIED) { + if (!reconcilingRef.current) { + reconciliationDeadlineRef.current = Date.now() + TRIGGER_RECONCILIATION_TIMEOUT_MS; + setManualFallbackReady(false); + updateReconciling(true); + } + setConnectionLost(false); + setTriggerError(null); + allowManualFallbackAfterTimeout(); + return true; + } + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + setConnectionLost(false); + setTriggerError(null); + return true; + } + + if (!isUpgradeTerminal(next.phase)) { + return false; + } + + if (tracked && !trackedMatches) { + return false; + } + + if (next.phase === UpgradePhase.SUCCEEDED && !trackedMatches) { + const successRevision = `${next.id}:${revision}`; + if (refreshedUntrackedSuccessRef.current !== successRevision) { + refreshedUntrackedSuccessRef.current = successRevision; + onUntrackedSuccessRef.current?.(next); + } + return false; + } + + // A strictly newer release proves a later recovery. An exact target + // match does not: activation can expose the target version before a + // later service failure records the operation as failed. + if (next.phase === UpgradePhase.FAILED && isReleaseNewer(currentVersionRef.current, next.targetVersion)) { + if (acknowledgedTransition) updateAcknowledgedOperation(null); + if (trackedMatches) updateTrackedOperation(undefined); + if (operationRef.current?.id === next.id) updateOperation(undefined); + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + setConnectionLost(false); + return false; + } + + const unresolvedFailure = + next.phase === UpgradePhase.FAILED && + (currentVersionUnavailableRef.current || Boolean(currentVersionRef.current)); + if (!trackedMatches && !unresolvedFailure) { + return false; + } + + if (acknowledgedTransition) updateAcknowledgedOperation(null); + updateTrackedOperation({ id: next.id, targetVersion: next.targetVersion }); + updateOperation(next); + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + setConnectionLost(false); + setTriggerError(null); + return true; + }, + [ + allowManualFallbackAfterTimeout, + updateAcknowledgedOperation, + updateOperation, + updateReconciling, + updateTrackedOperation, + ], + ); + + const pollStatus = useCallback( + async (signal: AbortSignal, pollingAuthSessionIdentity: string) => { + try { + const response = await instanceUpdateClient.getUpgradeStatus( + {}, + { signal, timeoutMs: STATUS_REQUEST_TIMEOUT_MS }, + ); + if (signal.aborted || authSessionIdentityRef.current !== pollingAuthSessionIdentity) { + return; + } + + resolvedStatusSessionIdentityRef.current = pollingAuthSessionIdentity; + setResolvedStatusSessionIdentity(pollingAuthSessionIdentity); + lastObservedOperationIDRef.current = response.operation?.id || null; + + if (response.operation && acceptServerOperation(response.operation)) { + return; + } + + if (!response.executorAvailable) { + if (reconcileMissingActiveOperation()) { + return; + } + if (isUpgradeActive(operationRef.current) || reconcilingRef.current || trackedOperationRef.current) { + setConnectionLost(true); + } + allowManualFallbackAfterTimeout(); + return; + } + + if (reconcileMissingActiveOperation()) { + return; + } + setConnectionLost(false); + if (resolveRecoveredOperationMiss()) { + return; + } + finishExpiredReconciliation(); + } catch (error) { + if (signal.aborted || authSessionIdentityRef.current !== pollingAuthSessionIdentity) return; + onPollErrorRef.current?.(error); + if (reconcileMissingActiveOperation()) { + return; + } + if (isUpgradeActive(operationRef.current) || reconcilingRef.current || trackedOperationRef.current) { + setConnectionLost(true); + } + allowManualFallbackAfterTimeout(); + } + }, + [ + acceptServerOperation, + allowManualFallbackAfterTimeout, + finishExpiredReconciliation, + reconcileMissingActiveOperation, + resolveRecoveredOperationMiss, + ], + ); + + useEffect(() => { + if (!enabled) return; + + let alive = true; + let timer: number | undefined; + let controller: AbortController | undefined; + + const run = async () => { + controller = new AbortController(); + await pollStatus(controller.signal, authSessionIdentity); + if (alive) { + const awaitingTrackedOperation = Boolean(trackedOperationRef.current && !operationRef.current); + const awaitingInitialStatus = resolvedStatusSessionIdentityRef.current !== authSessionIdentity; + const pollIntervalMs = + isUpgradeActive(operationRef.current) || + reconcilingRef.current || + awaitingTrackedOperation || + awaitingInitialStatus + ? ACTIVE_POLL_INTERVAL_MS + : IDLE_POLL_INTERVAL_MS; + timer = window.setTimeout(run, pollIntervalMs); + } + }; + + void run(); + return () => { + alive = false; + controller?.abort(); + if (timer !== undefined) window.clearTimeout(timer); + }; + }, [authSessionIdentity, currentVersion, currentVersionUnavailable, enabled, pollRevision, pollStatus]); + + const triggerUpgrade = useCallback( + async (targetVersion: string) => { + triggerBaselineOperationIDRef.current = lastObservedOperationIDRef.current; + updateTrackedOperation({ targetVersion }); + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + setConnectionLost(false); + setTriggerError(null); + setTriggering(true); + try { + const response = await instanceUpdateClient.triggerUpgrade( + { targetVersion }, + { timeoutMs: TRIGGER_REQUEST_TIMEOUT_MS }, + ); + if (!response.operation) { + throw new Error("Host updater did not return an operation"); + } + if (!acceptServerOperation(response.operation, true)) { + throw new Error( + "Fleet couldn't confirm the upgrade state. Fleet will check the host before unlocking other install options.", + ); + } + } catch (error) { + setTriggerError(getErrorMessage(error, "Failed to start upgrade")); + if (isDefinitiveTriggerRejection(error)) { + updateTrackedOperation(undefined); + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + setConnectionLost(false); + } else { + reconciliationDeadlineRef.current = Date.now() + TRIGGER_RECONCILIATION_TIMEOUT_MS; + updateReconciling(true); + } + } finally { + setTriggering(false); + // An idle status poll may already be scheduled far in the future. + // Wake it after trigger admission settles so success and ambiguous + // responses both transition immediately to the active cadence. + setPollRevision((revision) => revision + 1); + } + }, + [acceptServerOperation, updateReconciling, updateTrackedOperation], + ); + + const acknowledgeOperation = useCallback(() => { + const currentOperation = operationRef.current; + if (currentOperation && isUpgradeTerminal(currentOperation.phase)) { + updateAcknowledgedOperation({ + id: currentOperation.id, + phase: currentOperation.phase, + revision: operationRevision(currentOperation), + }); + } + updateTrackedOperation(undefined); + updateOperation(undefined); + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + setConnectionLost(false); + setTriggerError(null); + }, [updateAcknowledgedOperation, updateOperation, updateReconciling, updateTrackedOperation]); + + const useManualFallback = useCallback(() => { + if (!manualFallbackReady) return; + if (operationRef.current?.phase === UpgradePhase.UNSPECIFIED) { + updateAcknowledgedOperation({ + id: operationRef.current.id, + phase: operationRef.current.phase, + revision: operationRevision(operationRef.current), + }); + } + updateTrackedOperation(undefined); + updateOperation(undefined); + reconciliationDeadlineRef.current = null; + setManualFallbackReady(false); + updateReconciling(false); + setConnectionLost(false); + setTriggerError(null); + }, [manualFallbackReady, updateAcknowledgedOperation, updateOperation, updateReconciling, updateTrackedOperation]); + + const reloadFleet = useCallback(() => { + updateTrackedOperation(undefined); + window.location.reload(); + }, [updateTrackedOperation]); + + return { + acknowledgeOperation, + connectionLost, + manualFallbackReady, + operation, + operationStatusPending, + reconciling, + reloadFleet, + triggerError, + triggering, + trackedTargetVersion: trackedOperation?.targetVersion, + triggerUpgrade, + useManualFallback, + }; +} diff --git a/client/src/protoFleet/features/updates/copyInstallCommand.ts b/client/src/protoFleet/features/updates/copyInstallCommand.ts index 362bffc2bd..1292456180 100644 --- a/client/src/protoFleet/features/updates/copyInstallCommand.ts +++ b/client/src/protoFleet/features/updates/copyInstallCommand.ts @@ -1,8 +1,8 @@ import { pushToast, STATUSES } from "@/shared/features/toaster"; import { copyToClipboard } from "@/shared/utils/utility"; -// Shared by the update notification modal and the settings Updates page so -// both surfaces give identical feedback for the same action. +// Keep the manual fallback's clipboard feedback consistent wherever the +// Settings update flow renders it. export const copyInstallCommand = (installCommand: string) => { copyToClipboard(installCommand) .then(() => { diff --git a/client/src/protoFleet/store/hooks/useAuth.ts b/client/src/protoFleet/store/hooks/useAuth.ts index caad0e2f71..70096ba1c3 100644 --- a/client/src/protoFleet/store/hooks/useAuth.ts +++ b/client/src/protoFleet/store/hooks/useAuth.ts @@ -8,6 +8,8 @@ import { useFleetStore } from "../useFleetStore"; export const useSessionExpiry = () => useFleetStore((state) => state.auth.sessionExpiry); +export const useSessionGeneration = () => useFleetStore((state) => state.auth.sessionGeneration); + export const useIsAuthenticated = () => useFleetStore((state) => state.auth.isAuthenticated); export const useUsername = () => useFleetStore((state) => state.auth.username); diff --git a/client/src/protoFleet/store/index.ts b/client/src/protoFleet/store/index.ts index c520eaf966..944b8d55ac 100644 --- a/client/src/protoFleet/store/index.ts +++ b/client/src/protoFleet/store/index.ts @@ -11,6 +11,7 @@ export type { FleetStore } from "./useFleetStore"; export { useSessionExpiry, + useSessionGeneration, useIsAuthenticated, useUsername, useRole, diff --git a/client/src/protoFleet/store/slices/authSlice.ts b/client/src/protoFleet/store/slices/authSlice.ts index 85556bc065..4e3941528b 100644 --- a/client/src/protoFleet/store/slices/authSlice.ts +++ b/client/src/protoFleet/store/slices/authSlice.ts @@ -9,6 +9,7 @@ import { resetActiveCurtailmentData } from "@/protoFleet/api/activeCurtailmentDa export interface AuthSlice { sessionExpiry: Date | null; + sessionGeneration: number; isAuthenticated: boolean; username: string; role: string; @@ -38,6 +39,7 @@ export interface AuthSlice { export const createAuthSlice: StateCreator = (set) => ({ // Initial state sessionExpiry: null, + sessionGeneration: 0, isAuthenticated: false, username: "", role: "", @@ -49,6 +51,9 @@ export const createAuthSlice: StateCreator set((state) => { state.auth.sessionExpiry = expiry; + if (expiry) { + state.auth.sessionGeneration += 1; + } }), setIsAuthenticated: (isAuthenticated) => diff --git a/client/src/protoFleet/store/useFleetStore.test.ts b/client/src/protoFleet/store/useFleetStore.test.ts index dc2ee7ee05..e4291dfd64 100644 --- a/client/src/protoFleet/store/useFleetStore.test.ts +++ b/client/src/protoFleet/store/useFleetStore.test.ts @@ -58,6 +58,7 @@ describe("useFleetStore persistence", () => { it("preserves persisted org-scoped permissions", async () => { seedPersistedAuth({ sessionExpiry: new Date(Date.now() + 60_000), + sessionGeneration: 7, isAuthenticated: true, username: "alice@example.com", role: "ADMIN", @@ -70,6 +71,18 @@ describe("useFleetStore persistence", () => { expect(useFleetStore.getState().auth.permissions).toEqual(["site:read"]); expect(useFleetStore.getState().auth.isAuthenticated).toBe(true); + expect(useFleetStore.getState().auth.sessionGeneration).toBe(7); + }); + + it("advances the session generation for replacement sessions with the same expiry", async () => { + const { useFleetStore } = await import("./useFleetStore"); + const expiry = new Date(1_000); + + useFleetStore.getState().auth.setSessionExpiry(expiry); + expect(useFleetStore.getState().auth.sessionGeneration).toBe(1); + + useFleetStore.getState().auth.setSessionExpiry(new Date(expiry.getTime())); + expect(useFleetStore.getState().auth.sessionGeneration).toBe(2); }); it("preserves org-scoped sessions with no permissions", async () => { diff --git a/client/src/protoFleet/store/useFleetStore.ts b/client/src/protoFleet/store/useFleetStore.ts index 22b2f1a9fe..820cf37acf 100644 --- a/client/src/protoFleet/store/useFleetStore.ts +++ b/client/src/protoFleet/store/useFleetStore.ts @@ -29,7 +29,10 @@ export interface FleetStore { const ORG_PERMISSIONS_SCOPE = "org" as const; -type PersistedAuthState = Pick & { +type PersistedAuthState = Pick< + AuthSlice, + "sessionExpiry" | "sessionGeneration" | "isAuthenticated" | "username" | "role" | "permissions" +> & { // Guards against rehydrating old sessions where permissions meant a flat // "has this anywhere" projection. Current permissions are org/default scope. permissionsScope?: typeof ORG_PERMISSIONS_SCOPE; @@ -103,6 +106,7 @@ const createMultiKeyStorage = (): PersistStorage => { state: { auth: { sessionExpiry: state.auth.sessionExpiry, + sessionGeneration: state.auth.sessionGeneration, isAuthenticated: state.auth.isAuthenticated, username: state.auth.username, role: state.auth.role, @@ -181,6 +185,7 @@ export const useFleetStore = create()( partialize: (state) => ({ auth: { sessionExpiry: state.auth.sessionExpiry, + sessionGeneration: state.auth.sessionGeneration, isAuthenticated: state.auth.isAuthenticated, username: state.auth.username, role: state.auth.role, @@ -222,6 +227,9 @@ export const useFleetStore = create()( sessionExpiry: sessionIsStalePreOrgDefault ? currentState.auth.sessionExpiry : (persisted?.auth?.sessionExpiry ?? currentState.auth.sessionExpiry), + sessionGeneration: sessionIsStalePreOrgDefault + ? currentState.auth.sessionGeneration + : (persisted?.auth?.sessionGeneration ?? currentState.auth.sessionGeneration), isAuthenticated: sessionIsStalePreOrgDefault ? false : (persisted?.auth?.isAuthenticated ?? currentState.auth.isAuthenticated), diff --git a/docs/plans/archive/2026-07-29-one-click-upgrade-executor-plan.md b/docs/plans/archive/2026-07-29-one-click-upgrade-executor-plan.md new file mode 100644 index 0000000000..6f84ba91c5 --- /dev/null +++ b/docs/plans/archive/2026-07-29-one-click-upgrade-executor-plan.md @@ -0,0 +1,115 @@ +--- +title: One-Click Upgrade Host Executor - Plan +date: 2026-07-29 +status: completed +type: plan +topic: one-click-upgrade +artifact_contract: ce-unified-plan/v1 +artifact_readiness: implementation-ready +--- + +# One-Click Upgrade Host Executor + +## Outcome + +Proto Fleet now lets an authorized operator confirm an eligible upgrade in the +Updates settings route. A privileged process outside the Compose stack +validates and activates the release, while the client follows durable progress +through the expected Fleet restart. Hosts without a reachable executor retain +the release-specific manual install command as the authoritative fallback. + +## Final architecture + +### Client ownership + +- The normal shell owns only a passive version pill. It performs lightweight + release discovery and routes the operator to `/settings/updates`; it does not + retain installer commands or own upgrade progress globally. +- The Updates route fetches authoritative status before exposing actions. It + shows one-click controls only when Fleet reports a reachable host executor, + and otherwise keeps the manual command path intact. +- Confirmation names the exact target and explains the restart window. Release + candidates add the forward-only migration and no-downgrade warning. +- Confirmation, progress, reconnect, terminal error, recovery, and success are + route-owned states. Leaving the route stops only browser polling; it does not + cancel the durable host operation. Returning to the route recovers current + state through the upgrade-status RPC. +- An ambiguous trigger keeps conflicting controls locked while the executor is + unreachable. After a bounded wait, the manual path can be unlocked only by + explicitly confirming on-host that no upgrade is still running. + +### Privilege and execution boundary + +- `fleetd` re-checks `instance:update`, the selected release channel, and the + currently eligible target. The browser cannot provide a download URL, + command, downgrade, or arbitrary release tag. +- The root systemd service runs outside the Compose stack it restarts. Fleet + reaches its narrow HTTP API through + `/run/proto-fleet-updater/updater.sock`; the application container never + receives the host Docker socket. +- Before teardown, the updater downloads and verifies the release, safely + extracts it, preserves deployment configuration, builds release-specific + images, and completes preflight. Activation revalidates the staged manifest + and image identities before switching the deployment. +- The supported topology is one Proto Fleet installation per host on Linux + with systemd and rootful Docker, including WSL distributions configured with + systemd. macOS, rootless Docker, Linux without systemd, and alternate + multi-install layouts use the manual flow. + +### Trust and recovery boundary + +- The SHA-256 sidecar detects corruption and binds the expected asset name to + its digest. The bundle and sidecar share a GitHub Release origin, so GitHub + remains the publisher trust anchor; independent release signing is outside + this phase. +- Runtime configuration (`.env`, TLS material, Influx configuration, and + persisted optional-overlay settings) is carried into the staged deployment. +- Updater state and per-operation logs survive Fleet restarts at + `/var/lib/proto-fleet-updater/state.json` and + `/var/lib/proto-fleet-updater/logs/.log`. +- A terminal failure exposes the host log path and an explicit recovery + command. The previous deployment remains at + `/deployment.previous` for inspection, but automatic binary + rollback is disabled because migrations are forward-only. + +## End-to-end flow + +```mermaid +sequenceDiagram + participant U as "Authorized operator" + participant S as "Fleet shell" + participant R as "Updates route" + participant F as "fleetd" + participant X as "Host updater (systemd)" + participant G as "GitHub Releases" + + S->>F: "Discover eligible version" + F-->>S: "Version-only indicator data" + U->>S: "Select update pill" + S->>R: "Navigate to /settings/updates" + R->>F: "GetUpdateStatus" + F-->>R: "Eligible release, capability, manual command" + U->>R: "Confirm exact target" + R->>F: "TriggerUpgrade(target version)" + F->>F: "Re-check permission, channel, and eligibility" + F->>X: "Start operation over Unix socket" + X->>G: "Download bundle and checksum" + X->>X: "Verify, stage, preflight, and activate" + R-->>F: "Poll durable status; restart disconnect is expected" + opt "Operator leaves and later returns" + U->>R: "Open /settings/updates" + R->>F: "GetUpgradeStatus" + end + F-->>R: "Active or terminal durable operation" + R-->>U: "Progress, reload, or recovery guidance" +``` + +## Completed implementation units + +1. Durable single-flight updater manager and Unix-socket API. +2. Packaged updater binary, hardened systemd service, and installer lifecycle. +3. Non-interactive preflight and activation with immutable staging checks. +4. Release checksum publication and bounded download/extraction validation. +5. Permission-gated trigger/status RPCs with server-side target validation. +6. Passive shell discovery plus route-owned confirmation and operation states. +7. Focused host, API, installer, and client lifecycle tests.