diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 83f525bbf8..600879a82f 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -196,6 +196,7 @@ import { buildPullRequestAdvisory, evaluateGateCheck, recordConfiguredGateBlockerSignals, + recordGateScoreSignals, resolveAiReviewLowConfidenceHold, } from "../rules/advisory"; import { hasValidationNote, isTestPath } from "../signals/test-evidence"; @@ -10410,6 +10411,8 @@ async function maybePublishPrPublicSurface( await recordConfiguredGateBlockerSignals(env, advisory, gatePolicy, repoFullName, pr.number, { aiReviewDiff: buildAiReviewDiff(await getReviewFiles()), }); + // #8223: the score gates leave labeled evidence too — fired whenever they actually evaluated. + await recordGateScoreSignals(env, gatePolicy, repoFullName, pr.number); } // Deterministic content/registry surface lane (#1255) — flag-gated + per-repo allowlist, byte-identical when // off (evaluateWithSurfaceLane returns the generic evaluation unchanged and resolves no files). A metagraphed diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts index 6f933e6017..68b6d8a003 100644 --- a/src/rules/advisory.ts +++ b/src/rules/advisory.ts @@ -175,6 +175,17 @@ export const AI_JUDGMENT_BLOCKER_CODES = new Set(["ai_consensus_defect", * (#8104). That one code is wired by #8101 at its own upstream push / reversal sites — including it here * would double-count fired/reversed history. Keep this list in sync with `isConfiguredGateBlocker`'s body. */ +/** The two score-gate rule ids #8223 captures — knobs whose decisions previously left NO labeled + * evidence (`slopGateMinScore` / `qualityGateMinScore` gate real verdicts but recorded no + * `signal.rule_fired` events, so no corpus could ever form for them). Included in the reversal list + * below: a human undoing the bot outcome on a PR where a score gate evaluated IS the labeled evidence + * the knob registry needs to backtest these thresholds — the same reversal semantic every other entry + * carries (justified per the issue's extend-only-if-qualified requirement: slop carries direct gate + * authority in block mode; quality is advisory-only but its threshold is registry-governable, and a + * reversal labels the overall bot outcome its score contributed to, which is exactly the corpus label + * the drift/loosening evaluators consume). */ +export const GATE_SCORE_SIGNAL_CODES: readonly string[] = Object.freeze(["slop_gate_score", "quality_gate_score"]); + export const CONFIGURED_GATE_BLOCKER_SIGNAL_CODES: readonly string[] = Object.freeze([ "missing_linked_issue", "duplicate_pr_risk", @@ -188,6 +199,7 @@ export const CONFIGURED_GATE_BLOCKER_SIGNAL_CODES: readonly string[] = Object.fr "content_lane_deliverable_missing", "lockfile_tamper_risk", CLA_CONSENT_MISSING_CODE, + ...GATE_SCORE_SIGNAL_CODES, ]); /** Fixed lookback for reversal→HumanOverrideEvent pairing (#8104) — 30 days in milliseconds. */ @@ -1130,6 +1142,76 @@ export async function recordConfiguredGateBlockerSignals( ); } +/** + * Record a fired signal for each score gate that actually EVALUATED its score this pass (#8223) -- the + * same filter the pure evaluation applies: slop evaluates only in `block` mode with a non-null risk + * (mirrors {@link buildSlopGateBlocker}); quality evaluates whenever its mode is not `off` with both a + * score and a threshold present (mirrors {@link buildQualityGateWarning}), pass or fail alike -- a corpus + * needs both outcomes to backtest a threshold. Metadata carries the score normalized to [0, 1] + * (both scores are 0-100 integers per normalizeScore; divided by 100 to be confidence-equivalent for + * buildConfidenceThresholdClassifier replays) plus the detection's own detail string as `rawSignal` -- + * never diff content, per #8130's raw-context audit posture for computed-score rules. Best-effort like + * every calibration write: a failure never affects the verdict. + */ +export async function recordGateScoreSignals( + env: Env, + policy: GateCheckPolicy, + repoFullName: string, + prNumber: number, +): Promise { + // The SAME policy transform evaluateGateCheckCore applies before its pure evaluations: the #551 + // merge-readiness composite can promote slopGateMode to block, and buildSlopGateBlocker only ever sees + // the PROMOTED policy — reading the raw one here would silently drop corpus evidence for exactly the + // composite-gated case this capture exists for (mirrors recordConfiguredGateBlockerSignals above). + const effective = applyMergeReadinessGate(policy); + const store = createSignalStore(env); + const targetKey = `${repoFullName}#${prNumber}`; + const occurredAt = nowIso(); + const writes: Promise[] = []; + + const slopMode = gateMode(effective.slopGateMode); + const slopRisk = normalizeScore(effective.slopRisk); + if (slopMode === "block" && slopRisk !== null) { + const slopMin = normalizeScore(effective.slopGateMinScore) ?? DEFAULT_SLOP_BLOCK_THRESHOLD; + writes.push( + store + .recordRuleFired({ + ruleId: "slop_gate_score", + targetKey, + outcome: slopRisk >= slopMin ? "above_threshold" : "below_threshold", + occurredAt, + metadata: { + confidence: slopRisk / 100, + rawSignal: `deterministic slop risk ${slopRisk}/100 vs threshold ${slopMin}/100 (mode ${slopMode})`, + }, + }) + .catch(() => undefined), + ); + } + + const qualityMode = gateMode(effective.qualityGateMode); + const readinessScore = normalizeScore(effective.readinessScore); + const qualityMin = normalizeScore(effective.qualityGateMinScore); + if (qualityMode !== "off" && readinessScore !== null && qualityMin !== null) { + writes.push( + store + .recordRuleFired({ + ruleId: "quality_gate_score", + targetKey, + outcome: readinessScore < qualityMin ? "below_threshold" : "at_or_above_threshold", + occurredAt, + metadata: { + confidence: readinessScore / 100, + rawSignal: `public readiness score ${readinessScore}/100 vs threshold ${qualityMin}/100 (mode ${qualityMode})`, + }, + }) + .catch(() => undefined), + ); + } + + await Promise.all(writes); +} + function buildQualityGateWarning(policy: GateCheckPolicy): AdvisoryFinding | null { if (gateMode(policy.qualityGateMode) === "off") return null; const score = normalizeScore(policy.readinessScore); diff --git a/test/unit/configured-gate-blocker-signals.test.ts b/test/unit/configured-gate-blocker-signals.test.ts index 80fde2c0c3..f050b0403f 100644 --- a/test/unit/configured-gate-blocker-signals.test.ts +++ b/test/unit/configured-gate-blocker-signals.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { +import { recordGateScoreSignals, RAW_CONTEXT_MAX_DIFF_CHARS, recordConfiguredGateBlockerSignals, type GateCheckPolicy, @@ -243,3 +243,76 @@ describe("recordConfiguredGateBlockerSignals — raw context capture (#8130)", ( expect((await createSignalStore(env).queryRuleHistory("missing_linked_issue", 0)).fired).toHaveLength(1); }); }); + +// ── #8223: score-gate fired signals — slop + quality capture ──────────────────────────────────────────────── + +describe("recordGateScoreSignals (#8223)", () => { + it("fires slop_gate_score in block mode with the normalized score, threshold-crossing outcome, and rawSignal (never diff)", async () => { + const env = createTestEnv(); + await recordGateScoreSignals(env, { slopGateMode: "block", slopRisk: 72, slopGateMinScore: 60 }, "owner/repo", 7); + const history = await createSignalStore(env).queryRuleHistory("slop_gate_score", 0); + expect(history.fired).toHaveLength(1); + expect(history.fired[0]).toMatchObject({ + targetKey: "owner/repo#7", + outcome: "above_threshold", + metadata: { confidence: 0.72, rawSignal: "deterministic slop risk 72/100 vs threshold 60/100 (mode block)" }, + }); + expect(JSON.stringify(history.fired[0]!.metadata)).not.toContain("diff"); + }); + + it("fires slop below-threshold with the default block threshold when no minScore is configured", async () => { + const env = createTestEnv(); + await recordGateScoreSignals(env, { slopGateMode: "block", slopRisk: 30 }, "owner/repo", 7); + const history = await createSignalStore(env).queryRuleHistory("slop_gate_score", 0); + expect(history.fired[0]).toMatchObject({ outcome: "below_threshold", metadata: { confidence: 0.3 } }); + }); + + it("fires slop under the merge-readiness composite promotion even when slopGateMode itself is unset (#551 parity)", async () => { + // mergeReadinessGateMode: block promotes the slop sub-gate to block exactly as evaluateGateCheckCore's + // own applyMergeReadinessGate does — the raw slopGateMode stays unset, and the write must still happen. + const env = createTestEnv(); + await recordGateScoreSignals(env, { mergeReadinessGateMode: "block", slopRisk: 72, slopGateMinScore: 60 }, "owner/repo", 7); + const history = await createSignalStore(env).queryRuleHistory("slop_gate_score", 0); + expect(history.fired).toHaveLength(1); + expect(history.fired[0]).toMatchObject({ outcome: "above_threshold", metadata: { confidence: 0.72 } }); + }); + + it("records NOTHING for slop outside block mode or with a null risk — the gate never evaluated the score", async () => { + const env = createTestEnv(); + await recordGateScoreSignals(env, { slopGateMode: "advisory", slopRisk: 72 }, "owner/repo", 7); + await recordGateScoreSignals(env, { slopGateMode: "block" }, "owner/repo", 7); + expect((await createSignalStore(env).queryRuleHistory("slop_gate_score", 0)).fired).toEqual([]); + }); + + it("fires quality_gate_score in advisory mode too — pass AND fail evaluations both leave corpus evidence", async () => { + const env = createTestEnv(); + await recordGateScoreSignals(env, { qualityGateMode: "advisory", readinessScore: 80, qualityGateMinScore: 70 }, "owner/repo", 7); + await recordGateScoreSignals(env, { qualityGateMode: "advisory", readinessScore: 40, qualityGateMinScore: 70 }, "owner/repo", 8); + const history = await createSignalStore(env).queryRuleHistory("quality_gate_score", 0); + expect(history.fired).toHaveLength(2); + expect(history.fired.map((event) => event.outcome).sort()).toEqual(["at_or_above_threshold", "below_threshold"]); + expect(history.fired.map((event) => event.metadata?.confidence).sort()).toEqual([0.4, 0.8]); + }); + + it("records NOTHING for quality when the mode is off or a score/threshold is missing", async () => { + const env = createTestEnv(); + await recordGateScoreSignals(env, { qualityGateMode: "off", readinessScore: 40, qualityGateMinScore: 70 }, "owner/repo", 7); + await recordGateScoreSignals(env, { qualityGateMode: "advisory", qualityGateMinScore: 70 }, "owner/repo", 7); + await recordGateScoreSignals(env, { qualityGateMode: "advisory", readinessScore: 40 }, "owner/repo", 7); + expect((await createSignalStore(env).queryRuleHistory("quality_gate_score", 0)).fired).toEqual([]); + }); + + it("degrades silently when the SignalStore write rejects — the call resolves normally", async () => { + vi.spyOn(signalTrackingWire, "createSignalStore").mockReturnValue({ + recordRuleFired: async () => { + throw new Error("signal store down"); + }, + recordHumanOverride: async () => undefined, + queryRuleHistory: async () => ({ fired: [], overrides: [] }), + }); + await expect( + recordGateScoreSignals(createTestEnv(), { slopGateMode: "block", slopRisk: 72, qualityGateMode: "advisory", readinessScore: 40, qualityGateMinScore: 70 }, "owner/repo", 7), + ).resolves.toBeUndefined(); + vi.restoreAllMocks(); + }); +});