Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions src/services/ai-e2e-test-gen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -197,12 +197,19 @@ async function runWorkersE2eTestGen(env: Env, system: string, user: string, maxT
if (!ai || typeof ai.run !== "function") return { testSource: null };
const gatewayId = env.AI_GATEWAY_ID?.trim();
const extra: AiGatewayOptions | undefined = gatewayId ? { gateway: { id: gatewayId } } : undefined;
for (const model of E2E_TEST_GEN_MODELS) {
for (const [modelIndex, model] of E2E_TEST_GEN_MODELS.entries()) {
for (let attempt = 0; attempt < E2E_TEST_GEN_ATTEMPTS_PER_MODEL; attempt += 1) {
try {
const result = await ai.run(
model,
{ max_tokens: maxTokens, temperature: 0, messages: [{ role: "system", content: system }, { role: "user", content: user }] },
{
max_tokens: maxTokens,
temperature: 0,
messages: [{ role: "system", content: system }, { role: "user", content: user }],
// Only the truly final attempt (last model, last attempt) stays Sentry-visible (error); earlier
// attempts are about to be retried, so log them quietly (warn) per selfhost/ai.ts's contract (#8673).
finalAttempt: attempt === E2E_TEST_GEN_ATTEMPTS_PER_MODEL - 1 && modelIndex === E2E_TEST_GEN_MODELS.length - 1,
},
extra,
);
const parsed = parseE2eTestGenResponse(coerceAiText(result));
Expand Down
11 changes: 9 additions & 2 deletions src/services/linked-issue-satisfaction-run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,19 @@ async function runWorkersSatisfactionOpinion(
if (!ai || typeof ai.run !== "function") return { result: null };
const gatewayId = env.AI_GATEWAY_ID?.trim();
const extra: AiGatewayOptions | undefined = gatewayId ? { gateway: { id: gatewayId } } : undefined;
for (const model of LINKED_ISSUE_SATISFACTION_MODELS) {
for (const [modelIndex, model] of LINKED_ISSUE_SATISFACTION_MODELS.entries()) {
for (let attempt = 0; attempt < LINKED_ISSUE_SATISFACTION_ATTEMPTS_PER_MODEL; attempt += 1) {
try {
const raw = await ai.run(
model,
{ max_tokens: maxTokens, temperature: 0, messages: [{ role: "system", content: system }, { role: "user", content: user }] },
{
max_tokens: maxTokens,
temperature: 0,
messages: [{ role: "system", content: system }, { role: "user", content: user }],
// Only the truly final attempt (last model, last attempt) stays Sentry-visible (error); earlier
// attempts are about to be retried, so log them quietly (warn) per selfhost/ai.ts's contract (#8673).
finalAttempt: attempt === LINKED_ISSUE_SATISFACTION_ATTEMPTS_PER_MODEL - 1 && modelIndex === LINKED_ISSUE_SATISFACTION_MODELS.length - 1,
},
extra,
);
const text = coerceAiText(raw);
Expand Down
26 changes: 26 additions & 0 deletions test/unit/ai-e2e-test-gen.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -532,4 +532,30 @@ describe("runWorkersE2eTestGen (internal)", () => {
const env = createTestEnv({});
await expect(runWorkersE2eTestGen(env, "system", "user", 1024)).resolves.toEqual({ testSource: null });
});

it("passes finalAttempt:false on a retried attempt so it logs at warn, not error (#8673)", async () => {
// Fail on the first attempt, succeed on the second: the first (retried) attempt must carry finalAttempt:false
// so selfhost/ai.ts logs it quietly (warn), not as a Sentry-visible error.
let call = 0;
const run = vi.fn(async () => {
call += 1;
if (call === 1) throw new Error("transient");
return { response: JSON.stringify(["import { test } from '@playwright/test';"]) };
});
const env = enabledEnv(run);
await runWorkersE2eTestGen(env, "system", "user", 1024);
expect(((run.mock.calls[0] as unknown[])[1] as { finalAttempt?: boolean }).finalAttempt).toBe(false);
});

it("passes finalAttempt:true only on the truly final attempt (last model, last attempt) (#8673)", async () => {
// Every call fails: 2 models × 3 attempts = 6 calls; only the last (6th) is the final attempt and stays loud.
const run = vi.fn(async () => {
throw new Error("always fails");
});
const env = enabledEnv(run);
await runWorkersE2eTestGen(env, "system", "user", 1024);
expect(run).toHaveBeenCalledTimes(6);
const finalFlags = run.mock.calls.map((c) => ((c as unknown[])[1] as { finalAttempt?: boolean }).finalAttempt);
expect(finalFlags).toEqual([false, false, false, false, false, true]);
});
});
24 changes: 24 additions & 0 deletions test/unit/linked-issue-satisfaction-run.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,30 @@ describe("runLoopOverLinkedIssueSatisfaction gating + fail-safe", () => {
.first<{ actor: string | null }>();
expect(row?.actor).toBeNull();
});

it("passes finalAttempt:false on a retried attempt so it logs at warn, not error (#8673)", async () => {
// Fail once then succeed: the first (retried) attempt must carry finalAttempt:false so selfhost/ai.ts logs
// it quietly (warn) rather than as a Sentry-visible error.
let call = 0;
const run = vi.fn(async () => {
call += 1;
if (call === 1) throw new Error("transient");
return { response: satisfactionJson({ status: "addressed" }) };
});
await runLoopOverLinkedIssueSatisfaction(enabledEnv(run), baseInput);
expect(((run.mock.calls[0] as unknown[])[1] as { finalAttempt?: boolean }).finalAttempt).toBe(false);
});

it("passes finalAttempt:true only on the truly final attempt (last model, last attempt) (#8673)", async () => {
// Every call fails with a non-rate-limit error: 2 models × 3 attempts = 6 calls; only the last stays loud.
const run = vi.fn(async () => {
throw new Error("always fails");
});
await runLoopOverLinkedIssueSatisfaction(enabledEnv(run), baseInput);
expect(run).toHaveBeenCalledTimes(6);
const finalFlags = run.mock.calls.map((c) => ((c as unknown[])[1] as { finalAttempt?: boolean }).finalAttempt);
expect(finalFlags).toEqual([false, false, false, false, false, true]);
});
});

describe("runLinkedIssueSatisfactionForAdvisory (processor wiring, #1961/#3906)", () => {
Expand Down