From fba29ed1ff63068c55d1fd13d4a83eb67b77bebe Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 25 Jul 2026 22:51:02 -0700 Subject: [PATCH] feat(selftune): fold the REGRESSED-verdict track record into the self-tune tick's rec list (#8763) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/backtest-track-record.ts computed the rollup #8105's Phase-2 gating decision needs, but had zero non-manual callers — a human had to remember to run it against prod. The self-tune tick already appends report-only rec lines (the #8227 reliability-curve precedent), so the same rollup now rides along: - loadBacktestTrackRecord (selftune-wire.ts) reads the same three persisted event types the CLI queries (threshold run imported for real; the two scripts-side literals hand-mirrored with sync comments, matching the CLI's own posture in the other direction), extracts metadata.comparison, and aggregates via the shared engine computeRegressedVerdictTrackRecord. Fail-open per event type — a poisoned read degrades to fewer comparisons, never breaks the tick. - buildTrackRecordRecs (loosening-recs.ts) renders it as one payload-less rec: zero runs → no line (no steady-state noise); any regressed run → info severity; all-clean → good. The CLI stays the deep-dive view. --- src/review/loosening-recs.ts | 26 ++++++++++++++ src/review/selftune-wire.ts | 41 ++++++++++++++++++++-- test/unit/loosening-recs.test.ts | 58 ++++++++++++++++++++++++++++++- test/unit/selftune-wiring.test.ts | 53 ++++++++++++++++++++++++++++ 4 files changed, 175 insertions(+), 3 deletions(-) diff --git a/src/review/loosening-recs.ts b/src/review/loosening-recs.ts index b39708b837..3c0090b74c 100644 --- a/src/review/loosening-recs.ts +++ b/src/review/loosening-recs.ts @@ -12,6 +12,7 @@ import type { TuningRec } from "./auto-tune"; import type { SatisfactionFloorLooseningProposal } from "../services/satisfaction-floor-loosening"; import type { KnobLooseningProposal } from "../services/loosening-knobs"; +import type { RegressedVerdictTrackRecord } from "@loopover/engine"; /** The advisor list is per-project elsewhere; the satisfaction floor is deployment-global, so its recs use * this fixed pseudo-project label rather than impersonating any repo. */ @@ -100,3 +101,28 @@ export function buildReportOnlyKnobRecs(proposals: readonly KnobLooseningProposa "This knob has no override consumer yet — applying requires shipping its consumption plumbing as its own reviewed change.", })); } + +/** + * #8763: the cron-side rendering of the REGRESSED-verdict track record — the same rollup + * scripts/backtest-track-record.ts prints on demand, surfaced as one report-only rec line per self-tune + * tick so the maintainer sees #8105's decision evidence continuously instead of only when remembering to + * run the CLI. Zero recorded runs → NO line (an empty track record is the steady state for most ticks and + * must not add noise). Payload-less like every builder in this file: the apply path can never promote it. + */ +export function buildTrackRecordRecs(record: RegressedVerdictTrackRecord): TuningRec[] { + if (record.totalRuns === 0) return []; + const perRule = [...record.perRule.entries()] + .map(([ruleId, bucket]) => `${ruleId} total=${bucket.total} regressed=${bucket.regressed} improved=${bucket.improved} unchanged=${bucket.unchanged}`) + .join("; "); + const rate = record.regressedRate === null ? "N/A" : record.regressedRate.toFixed(3); + return [ + { + project: "global:backtest_track_record", + severity: record.regressedRuns > 0 ? "info" : "good", + message: + `Backtest track record (threshold + logic + replay runs): ${record.totalRuns} run(s), ` + + `${record.regressedRuns} REGRESSED (rate ${rate}). Per rule: ${perRule}. Report-only — the ` + + "evidence feed for the Phase-2 gating decision; scripts/backtest-track-record.ts stays the deep-dive view.", + }, + ]; +} diff --git a/src/review/selftune-wire.ts b/src/review/selftune-wire.ts index 830d55e811..831a1896ee 100644 --- a/src/review/selftune-wire.ts +++ b/src/review/selftune-wire.ts @@ -34,14 +34,16 @@ // side, so the advisor can only raise the floor) reaches the live gate with no risk of loosening it. Flag-OFF // (default) the override is never read and settings are byte-identical. (See applySelfTuneOverrideToSettings.) -import { listRepositories } from "../db/repositories"; +import { listAuditEventsByType, listRepositories } from "../db/repositories"; +import { computeRegressedVerdictTrackRecord, type BacktestComparison, type RegressedVerdictTrackRecord } from "@loopover/engine"; +import { THRESHOLD_BACKTEST_EVENT_TYPE } from "../services/threshold-backtest-run"; import { isAgentConfigured } from "../settings/autonomy"; import { resolveRepositorySettings } from "../settings/repository-settings"; import { buildRepoOutcomeCalibration } from "../services/outcome-calibration"; import { loadRepoFocusManifest } from "../signals/focus-manifest-loader"; import { errorMessage } from "../utils/json"; import { computeTuningRecommendations, type GateEvalReport, type GateEvalRow } from "./auto-tune"; -import { buildKnobReliabilityRecs, buildReportOnlyKnobRecs, buildSatisfactionFloorLooseningRecs } from "./loosening-recs"; +import { buildKnobReliabilityRecs, buildReportOnlyKnobRecs, buildSatisfactionFloorLooseningRecs, buildTrackRecordRecs } from "./loosening-recs"; import { loadReportOnlyKnobProposals, loadSatisfactionFloorRecState } from "../services/satisfaction-floor-loosening-run"; import { loadLiveKnobStatuses } from "../services/knob-loosening-run"; import { runAutoApplyRecommendations, type StorageEnv } from "./auto-apply"; @@ -149,6 +151,37 @@ async function selfTuneRepos(env: Env): Promise { * Caller MUST gate this on {@link isSelfTuneEnabled}: it is invoked only from the flag-ON cron path, so flag-OFF * this function is never reached and the cron does ZERO new work. */ +/** The persisted-run event types the track-record rollup reads (#8763) — the same three + * scripts/backtest-track-record.ts queries. THRESHOLD_BACKTEST_EVENT_TYPE is imported for real (its writer + * is Worker-side); the logic + counterfactual literals are hand-mirrored from their scripts-side writers + * (LOGIC_BACKTEST_EVENT_TYPE in scripts/backtest-logic-check-core.ts, COUNTERFACTUAL_BACKTEST_EVENT_TYPE in + * scripts/counterfactual-replay-core.ts) and must be kept in sync by hand — scripts are deliberately not + * importable from Worker code, the same posture the CLI takes toward this module in the other direction. */ +const BACKTEST_RUN_EVENT_TYPES: readonly string[] = [THRESHOLD_BACKTEST_EVENT_TYPE, "calibration.logic_backtest_run", "calibration.counterfactual_backtest_run"]; + +/** + * Aggregate every persisted advisory-backtest run into the REGRESSED-verdict track record (#8763) — the + * cron-side sibling of scripts/backtest-track-record.ts, reading the same rows through the repository layer + * instead of wrangler, with the same metadata.comparison extraction and the same pure engine aggregator. + * Fail-open PER EVENT TYPE: one unreadable type degrades to fewer comparisons (the tick's rec line simply + * reflects less evidence), never breaks the self-tune pass. + */ +export async function loadBacktestTrackRecord(env: Env): Promise { + const comparisons: BacktestComparison[] = []; + for (const eventType of BACKTEST_RUN_EVENT_TYPES) { + try { + const rows = await listAuditEventsByType(env, eventType, new Date(0).toISOString()); + for (const row of rows) { + const comparison = (row.metadata as { comparison?: BacktestComparison }).comparison; + if (comparison && typeof comparison === "object" && typeof comparison.ruleId === "string") comparisons.push(comparison); + } + } catch { + // Degrade to fewer comparisons — report-only surface, never worth failing the tick over. + } + } + return computeRegressedVerdictTrackRecord(comparisons); +} + export async function runSelfTune(env: Env): Promise { try { const repos = await selfTuneRepos(env); @@ -169,6 +202,10 @@ export async function runSelfTune(env: Env): Promise { recs.push(...buildReportOnlyKnobRecs(await loadReportOnlyKnobProposals(env, nowMs))); // #8227: the curve-derived view beside the ladder, one line per live knob with a differing suggestion. recs.push(...buildKnobReliabilityRecs(await loadLiveKnobStatuses(env))); + // #8763: the REGRESSED-verdict track record beside the knob recs — the same rollup the + // backtest-track-record CLI prints, computed from the persisted advisory-backtest runs, so the + // maintainer sees #8105's decision evidence every tick without remembering to run the CLI. + recs.push(...buildTrackRecordRecs(await loadBacktestTrackRecord(env))); } // runAutoApplyRecommendations only ever consumes recs that carry a TIGHTENING overridePayload, shadow- // soaks them, and promotes a soaked override only when isStrictlyTightening + evidence + soak pass. diff --git a/test/unit/loosening-recs.test.ts b/test/unit/loosening-recs.test.ts index 5b8b76215b..f48daf603e 100644 --- a/test/unit/loosening-recs.test.ts +++ b/test/unit/loosening-recs.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { buildSatisfactionFloorLooseningRecs, LOOSENING_REC_PROJECT } from "../../src/review/loosening-recs"; +import { buildSatisfactionFloorLooseningRecs, buildTrackRecordRecs, LOOSENING_REC_PROJECT } from "../../src/review/loosening-recs"; import type { SatisfactionFloorLooseningProposal } from "../../src/services/satisfaction-floor-loosening"; function proposal(overrides: Partial = {}): SatisfactionFloorLooseningProposal { @@ -71,3 +71,59 @@ describe("buildSatisfactionFloorLooseningRecs (#8160)", () => { expect(appliedOnly[0]!.severity).toBe("info"); }); }); + +describe("buildTrackRecordRecs (#8763)", () => { + function record(over: Partial = {}) { + return { + totalRuns: 0, + regressedRuns: 0, + regressedRate: null, + perRule: new Map(), + ...over, + } as import("@loopover/engine").RegressedVerdictTrackRecord; + } + + it("returns [] for an empty track record — most ticks have no runs and must not add a noise line", () => { + expect(buildTrackRecordRecs(record())).toEqual([]); + }); + + it("renders an all-clean record as a good-severity line with the 0.000 rate and the per-rule breakdown", () => { + const recs = buildTrackRecordRecs( + record({ + totalRuns: 3, + regressedRuns: 0, + regressedRate: 0, + perRule: new Map([["linked_issue_scope_mismatch", { total: 3, regressed: 0, improved: 2, unchanged: 1 }]]), + }), + ); + expect(recs).toHaveLength(1); + expect(recs[0]!.project).toBe("global:backtest_track_record"); + expect(recs[0]!.severity).toBe("good"); + expect(recs[0]!.message).toContain("3 run(s), 0 REGRESSED (rate 0.000)"); + expect(recs[0]!.message).toContain("linked_issue_scope_mismatch total=3 regressed=0 improved=2 unchanged=1"); + expect(recs[0]!.overridePayload).toBeUndefined(); + }); + + it("renders a regressed record as info severity, joining more than one rule's breakdown", () => { + const recs = buildTrackRecordRecs( + record({ + totalRuns: 4, + regressedRuns: 1, + regressedRate: 0.25, + perRule: new Map([ + ["ai_consensus_defect", { total: 2, regressed: 1, improved: 1, unchanged: 0 }], + ["linked_issue_scope_mismatch", { total: 2, regressed: 0, improved: 0, unchanged: 2 }], + ]), + }), + ); + expect(recs[0]!.severity).toBe("info"); + expect(recs[0]!.message).toContain("1 REGRESSED (rate 0.250)"); + expect(recs[0]!.message).toContain("ai_consensus_defect total=2 regressed=1"); + expect(recs[0]!.message).toContain("; linked_issue_scope_mismatch total=2"); + }); + + it("renders a null rate (defensive: non-empty record built by a caller without runs counted) as N/A", () => { + const recs = buildTrackRecordRecs(record({ totalRuns: 1, regressedRuns: 0, regressedRate: null, perRule: new Map([["x", { total: 1, regressed: 0, improved: 1, unchanged: 0 }]]) })); + expect(recs[0]!.message).toContain("rate N/A"); + }); +}); diff --git a/test/unit/selftune-wiring.test.ts b/test/unit/selftune-wiring.test.ts index 0c53820cdb..bf08fe380a 100644 --- a/test/unit/selftune-wiring.test.ts +++ b/test/unit/selftune-wiring.test.ts @@ -389,3 +389,56 @@ describe("config-application deferred (documented seam)", () => { expect(SELFTUNE_BASE_CONFIDENCE_FLOOR).toBe(0); }); }); + +// ── #8763: loadBacktestTrackRecord — the cron-side track-record rollup ─────────────────────────────────────── + +describe("loadBacktestTrackRecord (#8763)", () => { + function comparisonFor(ruleId: string, verdict: "improved" | "regressed" | "unchanged") { + const report = { ruleId, caseCount: 1, truePositives: 1, falsePositives: 0, trueNegatives: 0, falseNegatives: 0, precision: 1, recall: 1 }; + return { ruleId, baseline: report, candidate: report, regressedAxes: [], improvedAxes: [], verdict }; + } + + async function seedRun(env: Env, eventType: string, comparison: unknown): Promise { + const { recordAuditEvent } = await import("../../src/db/repositories"); + await recordAuditEvent(env, { + eventType, + actor: "loopover", + targetKey: "owner/repo#1", + outcome: "completed", + detail: "backtest run", + metadata: JSON.parse(JSON.stringify(comparison === undefined ? {} : { comparison })), + }); + } + + it("aggregates persisted runs across ALL THREE event types via the shared engine aggregator", async () => { + const env = createTestEnv(); + const { loadBacktestTrackRecord } = await import("../../src/review/selftune-wire"); + await seedRun(env, "calibration.threshold_backtest_run", comparisonFor("linked_issue_scope_mismatch", "regressed")); + await seedRun(env, "calibration.logic_backtest_run", comparisonFor("linked_issue_scope_mismatch", "improved")); + await seedRun(env, "calibration.counterfactual_backtest_run", comparisonFor("counterfactual_judge_variant", "unchanged")); + // A run row without a parseable comparison is skipped, never counted or thrown on. + await seedRun(env, "calibration.threshold_backtest_run", undefined); + + const record = await loadBacktestTrackRecord(env); + expect(record.totalRuns).toBe(3); + expect(record.regressedRuns).toBe(1); + expect(record.perRule.get("linked_issue_scope_mismatch")).toMatchObject({ total: 2, regressed: 1, improved: 1 }); + expect(record.perRule.get("counterfactual_judge_variant")).toMatchObject({ total: 1, unchanged: 1 }); + }); + + it("returns the empty record when nothing is persisted (the steady state for most ticks)", async () => { + const env = createTestEnv(); + const { loadBacktestTrackRecord } = await import("../../src/review/selftune-wire"); + const record = await loadBacktestTrackRecord(env); + expect(record.totalRuns).toBe(0); + expect(record.regressedRate).toBeNull(); + }); + + it("FAIL-OPEN: a poisoned audit_events read degrades to the empty record instead of throwing", async () => { + const env = createTestEnv(); + const { loadBacktestTrackRecord } = await import("../../src/review/selftune-wire"); + poisonDbPrepare(env, /audit_events/); + const record = await loadBacktestTrackRecord(env); + expect(record.totalRuns).toBe(0); + }); +});