diff --git a/src/services/ai-e2e-test-gen.ts b/src/services/ai-e2e-test-gen.ts index 01049135b0..9e714af11c 100644 --- a/src/services/ai-e2e-test-gen.ts +++ b/src/services/ai-e2e-test-gen.ts @@ -197,12 +197,19 @@ async function runWorkersE2eTestGen(env: Env, system: string, user: string, maxT if (!ai || typeof ai.run !== "function") return { testSource: null }; const gatewayId = env.AI_GATEWAY_ID?.trim(); const extra: AiGatewayOptions | undefined = gatewayId ? { gateway: { id: gatewayId } } : undefined; - for (const model of E2E_TEST_GEN_MODELS) { + for (const [modelIndex, model] of E2E_TEST_GEN_MODELS.entries()) { for (let attempt = 0; attempt < E2E_TEST_GEN_ATTEMPTS_PER_MODEL; attempt += 1) { try { const result = await ai.run( model, - { max_tokens: maxTokens, temperature: 0, messages: [{ role: "system", content: system }, { role: "user", content: user }] }, + { + max_tokens: maxTokens, + temperature: 0, + messages: [{ role: "system", content: system }, { role: "user", content: user }], + // Only the truly final attempt (last model, last attempt) stays Sentry-visible (error); earlier + // attempts are about to be retried, so log them quietly (warn) per selfhost/ai.ts's contract (#8673). + finalAttempt: attempt === E2E_TEST_GEN_ATTEMPTS_PER_MODEL - 1 && modelIndex === E2E_TEST_GEN_MODELS.length - 1, + }, extra, ); const parsed = parseE2eTestGenResponse(coerceAiText(result)); diff --git a/src/services/linked-issue-satisfaction-run.ts b/src/services/linked-issue-satisfaction-run.ts index 6e9ee53c66..1dec9050d0 100644 --- a/src/services/linked-issue-satisfaction-run.ts +++ b/src/services/linked-issue-satisfaction-run.ts @@ -83,12 +83,19 @@ async function runWorkersSatisfactionOpinion( if (!ai || typeof ai.run !== "function") return { result: null }; const gatewayId = env.AI_GATEWAY_ID?.trim(); const extra: AiGatewayOptions | undefined = gatewayId ? { gateway: { id: gatewayId } } : undefined; - for (const model of LINKED_ISSUE_SATISFACTION_MODELS) { + for (const [modelIndex, model] of LINKED_ISSUE_SATISFACTION_MODELS.entries()) { for (let attempt = 0; attempt < LINKED_ISSUE_SATISFACTION_ATTEMPTS_PER_MODEL; attempt += 1) { try { const raw = await ai.run( model, - { max_tokens: maxTokens, temperature: 0, messages: [{ role: "system", content: system }, { role: "user", content: user }] }, + { + max_tokens: maxTokens, + temperature: 0, + messages: [{ role: "system", content: system }, { role: "user", content: user }], + // Only the truly final attempt (last model, last attempt) stays Sentry-visible (error); earlier + // attempts are about to be retried, so log them quietly (warn) per selfhost/ai.ts's contract (#8673). + finalAttempt: attempt === LINKED_ISSUE_SATISFACTION_ATTEMPTS_PER_MODEL - 1 && modelIndex === LINKED_ISSUE_SATISFACTION_MODELS.length - 1, + }, extra, ); const text = coerceAiText(raw); diff --git a/test/unit/ai-e2e-test-gen.test.ts b/test/unit/ai-e2e-test-gen.test.ts index f2d09886ef..20b9fca3d3 100644 --- a/test/unit/ai-e2e-test-gen.test.ts +++ b/test/unit/ai-e2e-test-gen.test.ts @@ -532,4 +532,30 @@ describe("runWorkersE2eTestGen (internal)", () => { const env = createTestEnv({}); await expect(runWorkersE2eTestGen(env, "system", "user", 1024)).resolves.toEqual({ testSource: null }); }); + + it("passes finalAttempt:false on a retried attempt so it logs at warn, not error (#8673)", async () => { + // Fail on the first attempt, succeed on the second: the first (retried) attempt must carry finalAttempt:false + // so selfhost/ai.ts logs it quietly (warn), not as a Sentry-visible error. + let call = 0; + const run = vi.fn(async () => { + call += 1; + if (call === 1) throw new Error("transient"); + return { response: JSON.stringify(["import { test } from '@playwright/test';"]) }; + }); + const env = enabledEnv(run); + await runWorkersE2eTestGen(env, "system", "user", 1024); + expect(((run.mock.calls[0] as unknown[])[1] as { finalAttempt?: boolean }).finalAttempt).toBe(false); + }); + + it("passes finalAttempt:true only on the truly final attempt (last model, last attempt) (#8673)", async () => { + // Every call fails: 2 models × 3 attempts = 6 calls; only the last (6th) is the final attempt and stays loud. + const run = vi.fn(async () => { + throw new Error("always fails"); + }); + const env = enabledEnv(run); + await runWorkersE2eTestGen(env, "system", "user", 1024); + expect(run).toHaveBeenCalledTimes(6); + const finalFlags = run.mock.calls.map((c) => ((c as unknown[])[1] as { finalAttempt?: boolean }).finalAttempt); + expect(finalFlags).toEqual([false, false, false, false, false, true]); + }); }); diff --git a/test/unit/linked-issue-satisfaction-run.test.ts b/test/unit/linked-issue-satisfaction-run.test.ts index 6e2ff3be7b..527e4da47a 100644 --- a/test/unit/linked-issue-satisfaction-run.test.ts +++ b/test/unit/linked-issue-satisfaction-run.test.ts @@ -308,6 +308,30 @@ describe("runLoopOverLinkedIssueSatisfaction gating + fail-safe", () => { .first<{ actor: string | null }>(); expect(row?.actor).toBeNull(); }); + + it("passes finalAttempt:false on a retried attempt so it logs at warn, not error (#8673)", async () => { + // Fail once then succeed: the first (retried) attempt must carry finalAttempt:false so selfhost/ai.ts logs + // it quietly (warn) rather than as a Sentry-visible error. + let call = 0; + const run = vi.fn(async () => { + call += 1; + if (call === 1) throw new Error("transient"); + return { response: satisfactionJson({ status: "addressed" }) }; + }); + await runLoopOverLinkedIssueSatisfaction(enabledEnv(run), baseInput); + expect(((run.mock.calls[0] as unknown[])[1] as { finalAttempt?: boolean }).finalAttempt).toBe(false); + }); + + it("passes finalAttempt:true only on the truly final attempt (last model, last attempt) (#8673)", async () => { + // Every call fails with a non-rate-limit error: 2 models × 3 attempts = 6 calls; only the last stays loud. + const run = vi.fn(async () => { + throw new Error("always fails"); + }); + await runLoopOverLinkedIssueSatisfaction(enabledEnv(run), baseInput); + expect(run).toHaveBeenCalledTimes(6); + const finalFlags = run.mock.calls.map((c) => ((c as unknown[])[1] as { finalAttempt?: boolean }).finalAttempt); + expect(finalFlags).toEqual([false, false, false, false, false, true]); + }); }); describe("runLinkedIssueSatisfactionForAdvisory (processor wiring, #1961/#3906)", () => {