diff --git a/packages/client/src/__tests__/session-manager-edge-coverage.test.ts b/packages/client/src/__tests__/session-manager-edge-coverage.test.ts index 6372744b1..8df4f4e04 100644 --- a/packages/client/src/__tests__/session-manager-edge-coverage.test.ts +++ b/packages/client/src/__tests__/session-manager-edge-coverage.test.ts @@ -3326,6 +3326,203 @@ describe("SessionManager edge coverage", () => { await sm.shutdown(); }); + it("establishes inbox recovery debt before resuming after an operator suspend timeout", async () => { + vi.useFakeTimers(); + let initialCtx: SessionContext | undefined; + let initialHead: SessionMessage | undefined; + const oldHandler = handler({ + start: vi.fn().mockImplementation((message, ctx, token) => { + initialCtx = ctx; + initialHead = message; + token?.processingStarted(message); + return new Promise(() => {}); + }), + inject: vi.fn().mockReturnValue({ kind: "owned", mode: "queued" } as const), + suspend: vi.fn(() => new Promise(() => {})), + shutdown: vi.fn(() => new Promise(() => {})), + }); + let freshCtx: SessionContext | undefined; + let freshMessage: SessionMessage | undefined; + const freshHandler = handler({ + start: vi.fn().mockImplementation(async (message, ctx) => { + freshCtx = ctx; + freshMessage = message; + return { sessionId: "fresh-session", route: { kind: "owned" as const, mode: "queued" as const } }; + }), + }); + const ackEntry = vi.fn<(entryId: number) => Promise>().mockResolvedValue(undefined); + const recoverChat = vi.fn<(chatId: string) => Promise>().mockResolvedValue(undefined); + const sm = makeManager({ handlers: [oldHandler, freshHandler], ackEntry, recoverChat }); + const i = internals(sm); + const chatId = "chat-timeout-recovery-before-resume"; + const headEntry = mockEntry({ id: 9100, chatId, messageId: "msg-timeout-head" }); + const queuedTailEntry = mockEntry({ id: 9101, chatId, messageId: "msg-timeout-queued-tail" }); + + const initialDispatch = sm.dispatch(headEntry); + void initialDispatch; + await vi.waitFor(() => expect(oldHandler.start).toHaveBeenCalledTimes(1)); + if (!initialCtx || !initialHead) throw new Error("initial route was not captured"); + await sm.dispatch(queuedTailEntry); + + await sm.handleCommand(chatId, "session:suspend"); + const laterDispatch = sm.dispatch(mockEntry({ id: 9102, chatId, messageId: "msg-after-timeout" })); + + await vi.advanceTimersByTimeAsync(29_999); + expect(freshHandler.start).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(1); + await laterDispatch; + + expect(i.sessions.has(chatId)).toBe(false); + expect(ackEntry).toHaveBeenCalledWith(9100); + expect(ackEntry).not.toHaveBeenCalledWith(9101); + expect(recoverChat).toHaveBeenCalledTimes(1); + expect(freshHandler.start).not.toHaveBeenCalled(); + + await sm.dispatch(queuedTailEntry); + + const recoveryOrder = recoverChat.mock.invocationCallOrder[0]; + const resumeOrder = vi.mocked(freshHandler.start).mock.invocationCallOrder[0]; + expect(recoveryOrder).toBeDefined(); + expect(resumeOrder).toBeDefined(); + expect(Number(recoveryOrder)).toBeLessThan(Number(resumeOrder)); + expect(i.sessions.get(chatId)?.handler).toBe(freshHandler); + expect(freshHandler.start).toHaveBeenCalledTimes(1); + if (!freshCtx || !freshMessage) throw new Error("fresh route was not captured"); + await freshCtx.finishTurn(freshMessage, { status: "success", terminal: true }); + + await expect(sm.handleCommand(chatId, "session:terminate")).rejects.toThrow( + "timed-out route producer is not confirmed settled", + ); + await sm.shutdown(); + }); + + it("keeps an abandoned suspend teardown from blocking later resumes or manager shutdown", async () => { + vi.useFakeTimers(); + const oldHandler = handler({ + suspend: vi.fn(() => new Promise(() => {})), + shutdown: vi.fn(() => new Promise(() => {})), + }); + let freshCtx: SessionContext | undefined; + let freshMessage: SessionMessage | undefined; + const freshHandler = handler({ + resume: vi.fn().mockImplementation(async (message, _sessionId, ctx) => { + freshCtx = ctx; + freshMessage = message; + return { sessionId: "fresh-session", route: { kind: "owned" as const, mode: "queued" as const } }; + }), + }); + const sm = makeManager({ handlers: [freshHandler] }); + const i = internals(sm); + const chatId = "chat-abandoned-suspend-teardown"; + i.sessions.set(chatId, makeSessionRecord(chatId, { handler: oldHandler, status: "active" })); + i._activeCount = 1; + + await sm.handleCommand(chatId, "session:suspend"); + const firstDispatch = sm.dispatch(mockEntry({ id: 9110, chatId, messageId: "msg-first-resume" })); + await vi.advanceTimersByTimeAsync(30_000); + await firstDispatch; + + expect(oldHandler.shutdown).toHaveBeenCalledTimes(1); + expect(freshHandler.resume).toHaveBeenCalledTimes(1); + if (!freshCtx || !freshMessage) throw new Error("first fresh resume was not captured"); + await freshCtx.finishTurn(freshMessage, { status: "success", terminal: true }); + + await sm.handleCommand(chatId, "session:suspend"); + await i.sessions.get(chatId)?.suspending; + await sm.dispatch(mockEntry({ id: 9111, chatId, messageId: "msg-second-resume" })); + + expect(freshHandler.resume).toHaveBeenCalledTimes(2); + await expect(sm.handleCommand(chatId, "session:terminate")).rejects.toThrow("not confirmed stopped"); + await sm.shutdown(); + }); + + it("lets Reset strictly retry an abandoned suspend teardown after its first shutdown rejects", async () => { + vi.useFakeTimers(); + const oldHandler = handler({ + suspend: vi.fn(() => new Promise(() => {})), + shutdown: vi.fn().mockRejectedValueOnce(new Error("transient shutdown failure")).mockResolvedValueOnce(undefined), + }); + const sm = makeManager(); + const i = internals(sm); + const chatId = "chat-abandoned-suspend-reset-retry"; + i.sessions.set(chatId, makeSessionRecord(chatId, { handler: oldHandler, status: "active" })); + i._activeCount = 1; + + await sm.handleCommand(chatId, "session:suspend"); + await vi.advanceTimersByTimeAsync(30_000); + await i.sessions.get(chatId)?.suspending; + await vi.waitFor(() => expect(oldHandler.shutdown).toHaveBeenCalledTimes(1)); + expect(i.pendingTeardowns.get(chatId)?.has(oldHandler)).toBe(true); + + await expect(sm.handleCommand(chatId, "session:terminate")).resolves.toBeUndefined(); + + expect(oldHandler.shutdown).toHaveBeenCalledTimes(2); + expect(i.sessions.has(chatId)).toBe(false); + expect(i.pendingTeardowns.has(chatId)).toBe(false); + await sm.shutdown(); + }); + + it("refuses an abandoned teardown retry until its producer materializes late teardown debt", async () => { + vi.useFakeTimers(); + let signalResumeStarted: (() => void) | undefined; + let resolveResume: (() => void) | undefined; + const resumeStarted = new Promise((resolve) => { + signalResumeStarted = resolve; + }); + const resumeGate = new Promise((resolve) => { + resolveResume = resolve; + }); + const oldHandler = handler({ + resume: vi.fn().mockImplementation(async (message, _sessionId, _ctx, token) => { + token?.processingStarted(message); + signalResumeStarted?.(); + await resumeGate; + return { sessionId: "late-session", route: { kind: "owned" as const, mode: "queued" as const } }; + }), + suspend: vi.fn(() => new Promise(() => {})), + shutdown: vi + .fn() + .mockRejectedValueOnce(new Error("initial shutdown failure")) + .mockImplementation(() => new Promise(() => {})), + }); + const sm = makeManager(); + const i = internals(sm); + const chatId = "chat-abandoned-producer-before-reset-retry"; + i.sessions.set( + chatId, + makeSessionRecord(chatId, { + handler: oldHandler, + status: "suspended", + claudeSessionId: "existing-session", + }), + ); + + const resumeDispatch = sm.dispatch(mockEntry({ id: 9115, chatId, messageId: "msg-late-reset-race" })); + await resumeStarted; + await sm.handleCommand(chatId, "session:suspend"); + const suspendBoundary = i.sessions.get(chatId)?.suspending; + if (!suspendBoundary) throw new Error("operator suspend boundary was not created"); + await vi.advanceTimersByTimeAsync(30_000); + await suspendBoundary; + await vi.waitFor(() => expect(oldHandler.shutdown).toHaveBeenCalledTimes(1)); + + const firstReset = sm.handleCommand(chatId, "session:terminate"); + await Promise.resolve(); + expect(oldHandler.shutdown).toHaveBeenCalledTimes(1); + await expect(firstReset).rejects.toThrow("timed-out route producer is not confirmed settled"); + + resolveResume?.(); + await resumeDispatch; + await vi.waitFor(() => expect(oldHandler.shutdown).toHaveBeenCalledTimes(2)); + expect(i.routeProducers.has(chatId)).toBe(false); + expect(i.pendingTeardowns.get(chatId)?.has(oldHandler)).toBe(true); + + await expect(sm.handleCommand(chatId, "session:terminate")).rejects.toThrow("not confirmed stopped"); + expect(i.pendingTeardowns.get(chatId)?.has(oldHandler)).toBe(true); + await sm.shutdown(); + }); + it("retains teardown proof when a canceled fresh-start shutdown fails, and converges on terminate", async () => { const boom = new Error("start-cancel shutdown failed"); const startHandler = handler({ diff --git a/packages/client/src/runtime/session-manager.ts b/packages/client/src/runtime/session-manager.ts index 9611f4f75..814791462 100644 --- a/packages/client/src/runtime/session-manager.ts +++ b/packages/client/src/runtime/session-manager.ts @@ -468,6 +468,35 @@ type SessionManagerConfig = { /** Maximum number of evicted session mappings to retain for resume recovery. */ const MAX_EVICTED_MAPPINGS = 500; +/** + * A provider abort is best-effort and its completion callback can disappear + * across host sleep or transport loss. Do not let that callback permanently + * fence later inbox delivery for the chat. + */ +const OPERATOR_SUSPEND_TIMEOUT_MS = 30_000; + +class HandlerSuspendTimeoutError extends Error { + constructor(chatId: string) { + super(`handler suspend timed out after ${OPERATOR_SUSPEND_TIMEOUT_MS}ms for chat ${chatId}`); + this.name = "HandlerSuspendTimeoutError"; + } +} + +async function waitForHandlerSuspend(chatId: string, suspend: () => Promise): Promise { + let timer: ReturnType | null = null; + try { + await Promise.race([ + Promise.resolve().then(suspend), + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new HandlerSuspendTimeoutError(chatId)), OPERATOR_SUSPEND_TIMEOUT_MS); + timer.unref?.(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + /** * Minimum spacing between gate-triggered replay-fence reconciliations for * one chat. Withheld dispatches retry the server-truth readback so a @@ -642,12 +671,21 @@ export class SessionManager { * Per-chat teardown debt: handlers detached from their SessionEntry (LRU * eviction, failSessionForRecovery, abortUnownedRoute, terminal cleanup, * canceled fresh-start, resume/retry handler replacement) without a - * CONFIRMED stop. A ref'd terminate joins/strictly tears down every - * pending handler of the chat before it may ack; confirmed stops drop out - * of the set. Entries are registered at the detach point (see + * CONFIRMED stop. A ref'd terminate joins/strictly tears down ordinary + * pending handlers before it may ack; an abandoned suspend handler makes + * that terminate fail closed without joining the lost callback. Confirmed + * stops drop out of the set. Entries are registered at the detach point (see * `detachHandlerWithPendingTeardown` / `registerPendingTeardown`). */ private readonly pendingTeardowns = new Map>(); + /** + * A manual suspend request whose completion timed out has lost its + * trustworthy join boundary. The handler remains retired and tracked as + * teardown debt, but ordinary route admission must not join that same raw + * callback forever. Reset fails closed while the raw attempt is pending and + * may strictly retry only after a failed attempt has settled. + */ + private readonly abandonedSuspendHandlers = new WeakSet(); /** * In-flight route producers (start/resume/retry provider calls), per chat. * Tracked from `beginRouteTransition` until the route settles: a canceled @@ -655,9 +693,12 @@ export class SessionManager { * its pre-materialization shutdown ran as a no-op), and only the * producer's settle funnels that materialization into * `discardStaleRouteTransition` → teardown debt. Terminate and manager - * shutdown join these before they may ack/return. + * shutdown join these before they may ack/return, except for a producer + * explicitly abandoned by the timed-out suspend generation boundary. */ private readonly routeProducers = new Map>>(); + /** Producer joins invalidated by the same generation boundary as a timed-out manual suspend. */ + private readonly abandonedRouteProducers = new WeakSet>(); /** * Per-chat single-flight registry for `runRetry` executions. An * overlapping trigger (timer fire + immediate delivery trigger) joins the @@ -1081,6 +1122,18 @@ export class SessionManager { if (joinedSuspend && session?.suspendError) throw asTerminateError("suspend", session.suspendError.error); const activeSlotHeld = session?.activeSlotHeld === true; if (session) this.releaseActiveSlot(session); + // An abandoned producer can still materialize the SAME handler and + // register a new after-prior teardown while a strict abandoned-handler + // retry is awaiting. Because teardown debt is handler-keyed, letting + // that retry start now could make its success delete the newer debt. + // Fail before any handler teardown; once the producer settles, its + // stale-completion debt is registered before the producer disappears. + if (this.hasAbandonedRouteProducer(chatId)) { + throw asTerminateError( + "teardown", + new Error(`timed-out route producer is not confirmed settled for chat ${chatId}`), + ); + } // The teardown boundary — strict in every case, because a // handler.shutdown rejection must fail the apply (applied:false), // never resolve into an ack over a possibly-live handler. Beyond the @@ -1099,6 +1152,15 @@ export class SessionManager { // entry (boxed: rejections can be falsey) so a genuine retry // re-attempts it instead of turning into a false success; on success // the entry is deleted below, retiring the recorded errors with it. + if (session && this.abandonedSuspendHandlers.has(session.handler)) { + try { + await this.retryAbandonedSuspendTeardownForTerminate(chatId, session.handler); + session.handlerStoppedBySuspend = session.handler; + } catch (err) { + session.teardownError = { error: err }; + throw asTerminateError("teardown", err); + } + } const needsTeardown = session != null && session.handlerStoppedBySuspend !== session.handler && @@ -1118,6 +1180,12 @@ export class SessionManager { // `discardStaleRouteTransition` → fresh teardown debt. The drain // below must see that complete debt, so this quiesce runs first. await this.quiesceRouteProducers(chatId); + if (this.hasAbandonedRouteProducer(chatId)) { + throw asTerminateError( + "teardown", + new Error(`timed-out route producer is not confirmed settled for chat ${chatId}`), + ); + } // Settle every pending teardown debt for this chat: handlers detached // from their entry (eviction / recovery / abort / terminal cleanup / @@ -1134,6 +1202,14 @@ export class SessionManager { const pendingTeardown = this.pendingTeardowns.get(chatId); if (!pendingTeardown || pendingTeardown.size === 0) break; for (const pendingHandler of [...pendingTeardown]) { + if (this.abandonedSuspendHandlers.has(pendingHandler)) { + try { + await this.retryAbandonedSuspendTeardownForTerminate(chatId, pendingHandler); + } catch (err) { + throw asTerminateError("teardown", err); + } + continue; + } try { await this.shutdownHandler(pendingHandler, "session_terminated", { observeFailure: true }); this.dropPendingTeardown(chatId, pendingHandler); @@ -1345,6 +1421,10 @@ export class SessionManager { if (!session.activeSlotHeld && session.suspending === null && !stopUnconfirmedAfterFailedBoundary) { return Promise.resolve(); } + // A timed-out suspend already has a best-effort shutdown in flight. + // Joining its untrustworthy raw callback would make daemon shutdown + // unbounded again. + if (this.abandonedSuspendHandlers.has(session.handler)) return Promise.resolve(); attemptedHandlers.add(session.handler); return this.shutdownHandler(session.handler, reason ?? "manager_shutdown", { ...(session.activeSlotHeld ? { settleProviderEntered: true } : {}), @@ -1370,6 +1450,7 @@ export class SessionManager { } } for (const [pendingHandler, chatIds] of debtChatsByHandler) { + if (this.abandonedSuspendHandlers.has(pendingHandler)) continue; if (attemptedHandlers.has(pendingHandler)) continue; attemptedHandlers.add(pendingHandler); shutdowns.push( @@ -1407,11 +1488,14 @@ export class SessionManager { ...[...this.sessions.values()] .map((session) => session.suspending) .filter((pending): pending is Promise => pending !== null), - ...[...this.routeProducers.values()].flatMap((producers) => [...producers]), + ...[...this.routeProducers.values()].flatMap((producers) => + [...producers].filter((producer) => !this.abandonedRouteProducers.has(producer)), + ), ]); const retriedHandlers = new Set(); for (const pending of this.pendingTeardowns.values()) { for (const pendingHandler of [...pending]) { + if (this.abandonedSuspendHandlers.has(pendingHandler)) continue; if (retriedHandlers.has(pendingHandler)) continue; retriedHandlers.add(pendingHandler); // Each attempt joins a still in-flight shutdown when one exists; a @@ -2192,6 +2276,10 @@ export class SessionManager { }; } + private hasAbandonedRouteProducer(chatId: string): boolean { + return [...(this.routeProducers.get(chatId) ?? [])].some((producer) => this.abandonedRouteProducers.has(producer)); + } + /** * Join every in-flight route producer for the chat, draining to a stable * quiet point. A producer's settle can reveal late materialization debt @@ -2202,18 +2290,21 @@ export class SessionManager { for (;;) { const producers = this.routeProducers.get(chatId); if (!producers || producers.size === 0) return; - await Promise.allSettled([...producers]); + const joinable = [...producers].filter((producer) => !this.abandonedRouteProducers.has(producer)); + if (joinable.length === 0) return; + await Promise.allSettled(joinable); } } /** * Route admission fence: before a chat may create a new provider route, - * its teardown authority must be clean — otherwise the chat could run - * "old handler never confirmed stopped + new provider route started". - * Settles every pending handler strictly (coalescing joins in-flight - * shutdowns). Returns false when a stop fails: the debt stays registered - * and the caller must keep the delivery's recovery/retry custody instead - * of routing. + * its teardown authority must normally be clean — otherwise the chat could + * run "old handler never confirmed stopped + new provider route started". + * The sole exception is an operator suspend generation whose completion + * already exceeded its bound: late output is generation-fenced, its stop + * remains Reset-failing debt, and ordinary routes skip the lost join. + * Other pending handlers settle strictly. Returns false when a strict stop + * fails so the caller keeps recovery/retry custody instead of routing. */ private async settleTeardownDebtBeforeRoute(chatId: string): Promise { // Quiesce in-flight route producers FIRST: a canceled start/resume can @@ -2228,8 +2319,12 @@ export class SessionManager { for (;;) { const pending = this.pendingTeardowns.get(chatId); if (!pending || pending.size === 0) return true; + const routeBlockingHandlers = [...pending].filter( + (pendingHandler) => !this.abandonedSuspendHandlers.has(pendingHandler), + ); + if (routeBlockingHandlers.length === 0) return true; let settled = true; - for (const pendingHandler of [...pending]) { + for (const pendingHandler of routeBlockingHandlers) { try { await this.shutdownHandler(pendingHandler, "route_admission_teardown", { observeFailure: true }); this.dropPendingTeardown(chatId, pendingHandler); @@ -2253,13 +2348,53 @@ export class SessionManager { private detachHandlerWithPendingTeardown(chatId: string, handler: AgentHandler, reason: string): void { this.registerPendingTeardown(chatId, handler); void this.shutdownHandler(handler, reason, { observeFailure: true }).then( - () => this.dropPendingTeardown(chatId, handler), + () => { + this.dropPendingTeardown(chatId, handler); + this.abandonedSuspendHandlers.delete(handler); + }, () => { // Failure keeps the debt — a later ref'd terminate strictly retries. }, ); } + private abandonTimedOutSuspendHandler(entry: SessionEntry): void { + const handler = entry.handler; + this.retiredHandlers.add(handler); + this.abandonedSuspendHandlers.add(handler); + for (const producer of this.routeProducers.get(entry.chatId) ?? []) { + this.abandonedRouteProducers.add(producer); + } + this.registerPendingTeardown(entry.chatId, handler); + void this.shutdownHandler(handler, "operator_suspend_timeout", { observeFailure: true }).then( + () => { + this.dropPendingTeardown(entry.chatId, handler); + this.abandonedSuspendHandlers.delete(handler); + if (entry.handler === handler) entry.handlerStoppedBySuspend = handler; + }, + () => { + // Keep the abandoned debt as fail-closed Reset proof. Ordinary routes + // rely on the retired generation fence instead of joining it. + }, + ); + } + + /** + * Reset may retry an abandoned suspend teardown only after the previous raw + * shutdown attempt has settled. A still-registered attempt is the lost + * callback that made the generation abandoned, so joining it would hang the + * Reset. Once a rejection has settled, a fresh strict attempt is safe and + * restores the ordinary retry-to-convergence contract. + */ + private async retryAbandonedSuspendTeardownForTerminate(chatId: string, handler: AgentHandler): Promise { + if (this.handlerShutdowns.has(handler)) { + throw new Error(`timed-out suspend handler is not confirmed stopped for chat ${chatId}`); + } + await this.shutdownHandler(handler, "session_terminated", { observeFailure: true }); + this.dropPendingTeardown(chatId, handler); + this.abandonedSuspendHandlers.delete(handler); + } + private retireTransitionHandler(transition: RouteTransitionToken, reason: string): void { this.retiredHandlers.add(transition.handler); void this.shutdownHandler(transition.handler, reason); @@ -3117,20 +3252,25 @@ export class SessionManager { return; } - // A failed suspend left the current handler never confirmed stopped. - // Stop it strictly (joining any in-flight shutdown's raw face) BEFORE - // the route below reuses or replaces the reference — reusing or - // overwriting an unconfirmed-stop handler loses the teardown authority - // while the old run may still be alive. A teardown failure propagates - // into resume's existing error semantics (routeMessage retries the - // delivery / operator resume logs the command error); it must not - // silently continue. On success the handler is retired so the route - // installs a fresh one, and the recorded suspend failure is cleared. - if (entry.suspendError && entry.handlerStoppedBySuspend !== entry.handler) { - await this.shutdownHandler(entry.handler, "session_resume_after_failed_suspend", { observeFailure: true }); - entry.handlerStoppedBySuspend = entry.handler; + // A failed suspend normally leaves the current handler unconfirmed, so + // stop it strictly before installing a replacement. A suspend timeout is + // different: the suspend request was dispatched, but its completion was + // never observed. Retire that handler and let the existing route + // generation + teardown-debt fences contain any late completion instead + // of waiting on the same callback forever. + if (entry.suspendError) { + if (entry.handlerStoppedBySuspend !== entry.handler) { + if (entry.suspendError.error instanceof HandlerSuspendTimeoutError) { + // The timed-out handler was already retired and moved to the + // non-route-blocking teardown set by suspendSession(). + this.retiredHandlers.add(entry.handler); + } else { + await this.shutdownHandler(entry.handler, "session_resume_after_failed_suspend", { observeFailure: true }); + entry.handlerStoppedBySuspend = entry.handler; + this.retiredHandlers.add(entry.handler); + } + } entry.suspendError = null; - this.retiredHandlers.add(entry.handler); } // Route admission fence: settle this chat's teardown authority before @@ -4133,11 +4273,14 @@ export class SessionManager { this.recomputeRuntimeState(); entry.suspending = (async () => { let settled = false; + let timedOut = false; try { // settleProviderEntered keeps already-issued DeliveryTokens on the // settlement lease (including active/deferred inject) so they can // post durable notice+ACK before prepareOperatorSuspend runs. - await entry.handler.suspend(opts.reason, { settleProviderEntered: true }); + await waitForHandlerSuspend(entry.chatId, () => + entry.handler.suspend(opts.reason, { settleProviderEntered: true }), + ); // If settle captured a terminal notice but could not persist it (or // could not claim token settlement), transfer the obligation onto // the inbox ledger so prepareOperatorSuspend / recovery cannot ACK @@ -4147,10 +4290,13 @@ export class SessionManager { } settled = true; } catch (err) { - // Settle failure leaves the handler joinable for a strict terminate / - // resume stop — do not start teardown here or suspend will hang on a - // gated shutdown the terminate owns. + // An ordinary settle failure leaves the handler joinable for a + // strict terminate/resume stop. A timeout has lost that join + // boundary, so start only a non-blocking best-effort teardown and + // retain fail-closed Reset debt. entry.suspendError = { error: err }; + timedOut = err instanceof HandlerSuspendTimeoutError; + if (timedOut) this.abandonTimedOutSuspendHandler(entry); try { this.config.log.warn({ chatId: entry.chatId, err }, "operator suspend settlement error"); } catch (logErr) { @@ -4167,7 +4313,7 @@ export class SessionManager { this.retiredHandlers.add(inFlightTransition.handler); } - if (!settled) { + if (!settled && !timedOut) { entry.suspending = null; if (unestablishedStart) { if (entry.handlerStoppedBySuspend !== entry.handler) { @@ -4182,7 +4328,7 @@ export class SessionManager { } const stopPromise = - inFlightTransition || !this.retiredHandlers.has(entry.handler) + settled && (inFlightTransition || !this.retiredHandlers.has(entry.handler)) ? this.shutdownHandler(target, opts.reason, { observeFailure: true }) : Promise.resolve(); @@ -4194,7 +4340,7 @@ export class SessionManager { await Promise.resolve(); await this.inboxDelivery.prepareOperatorSuspend(entry.chatId); await stopPromise; - if (target === entry.handler) { + if (settled && target === entry.handler) { entry.handlerStoppedBySuspend = entry.handler; } } catch (err) {