From 895a0dd72fa41053f14eda339c34c29d2ee966b0 Mon Sep 17 00:00:00 2001 From: glorydavid03023 Date: Tue, 7 Jul 2026 05:40:21 +0000 Subject: [PATCH] fix(engine): report below-baseline calibration delta as a signed negative computePhase7CalibrationLoop rounded deltaFromBaseline through roundScore, which clamps to [0,1]. Since combinedAccuracy is in [0,1] and the documented baseline is 0.62, the true delta ranges over [-0.62, 0.38], so every below-baseline calibration run (a regression) was flattened to 0 and reported as on-baseline. deltaFromBaseline is a signed field surfaced as signed percentage points in the audit, so it must round the difference without the score clamp; add a roundDelta helper and use it. Regression tests cover below/at/above baseline in both the engine node:test and a root vitest suite. --- .../src/phase7-calibration-loop.ts | 10 ++- .../test/phase7-calibration-loop.test.ts | 16 +++++ test/unit/phase7-calibration-delta.test.ts | 70 +++++++++++++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 test/unit/phase7-calibration-delta.test.ts diff --git a/packages/gittensory-engine/src/phase7-calibration-loop.ts b/packages/gittensory-engine/src/phase7-calibration-loop.ts index bbd1eb2215..639002c911 100644 --- a/packages/gittensory-engine/src/phase7-calibration-loop.ts +++ b/packages/gittensory-engine/src/phase7-calibration-loop.ts @@ -118,6 +118,14 @@ function roundScore(value: number): number { return Math.round(Math.min(1, Math.max(0, value)) * 1_000_000) / 1_000_000; } +/** Round a SIGNED value to 6 dp WITHOUT the [0, 1] clamp `roundScore` applies to accuracy scores. A baseline + * delta legitimately ranges over [-baseline, 1 - baseline], so a below-baseline calibration run must surface as a + * negative deviation — clamping it to 0 would report a regression as "on baseline" and defeat the whole point of + * tracking accuracy against the documented baseline. */ +function roundDelta(value: number): number { + return Math.round(value * 1_000_000) / 1_000_000; +} + function finiteNonNegative(value: number | undefined, fallback: number): number { if (value === undefined) return fallback; if (!Number.isFinite(value) || value < 0) return 0; @@ -462,7 +470,7 @@ export function computePhase7CalibrationLoop(input: { ); const deltaFromBaseline = - combinedAccuracy === null ? null : roundScore(combinedAccuracy - DOCUMENTED_CALIBRATION_BASELINE); + combinedAccuracy === null ? null : roundDelta(combinedAccuracy - DOCUMENTED_CALIBRATION_BASELINE); const schedule = shouldScheduleHistoricalReplayRun({ config, diff --git a/packages/gittensory-engine/test/phase7-calibration-loop.test.ts b/packages/gittensory-engine/test/phase7-calibration-loop.test.ts index 1751e8a90d..e893c124c7 100644 --- a/packages/gittensory-engine/test/phase7-calibration-loop.test.ts +++ b/packages/gittensory-engine/test/phase7-calibration-loop.test.ts @@ -306,6 +306,22 @@ test("computePhase7CalibrationLoop combines historical-replay and pr_outcome sig assert.equal(result.bySource.pr_outcome.sampleSize, 20); }); +test("computePhase7CalibrationLoop reports a NEGATIVE delta when combined accuracy is below the documented baseline", () => { + // Both sources at 0.5 accuracy → combinedAccuracy 0.5, which is 0.12 BELOW the 0.62 documented baseline. The + // delta is a signed deviation, so a below-baseline regression must surface as -0.12, not be flattened to 0. + const result = computePhase7CalibrationLoop({ + config: enabledConfig(), + prOutcome: sufficientPrOutcome(0.5), + historicalReplay: healthyReplay(0.5), + now: NOW, + }); + + assert.equal(result.combinedAccuracy, 0.5); + assert.equal(result.deltaFromBaseline, -0.12); + // The Markdown audit surfaces the signed magnitude, so the regression is visible to an operator. + assert.match(renderPhase7CalibrationAuditMarkdown(result), /delta from baseline: -12\.00 percentage points/); +}); + test("computePhase7CalibrationLoop permits autonomy increases only when both sources meet the threshold", () => { const passing = computePhase7CalibrationLoop({ config: enabledConfig({ autonomyIncreaseMinAccuracy: 0.7 }), diff --git a/test/unit/phase7-calibration-delta.test.ts b/test/unit/phase7-calibration-delta.test.ts new file mode 100644 index 0000000000..5d8b66b452 --- /dev/null +++ b/test/unit/phase7-calibration-delta.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import { + computePhase7CalibrationLoop, + renderPhase7CalibrationAuditMarkdown, + resolvePhase7CalibrationConfig, + DOCUMENTED_CALIBRATION_BASELINE, +} from "../../packages/gittensory-engine/src/phase7-calibration-loop"; + +const NOW = "2026-07-04T18:00:00.000Z"; +const FRESH_REPLAY_AT = "2026-07-04T12:00:00.000Z"; + +function enabledConfig() { + return resolvePhase7CalibrationConfig({ + miner: { + calibration: { + phase7LoopEnabled: true, + autonomyIncreaseMinAccuracy: 0.7, + replayFreshnessMaxAgeHours: 168, + historicalReplayWeight: 0.5, + prOutcomeWeight: 0.5, + }, + }, + }); +} + +// decided=20; correct=round(20*accuracy) → an exact 0.5 / 0.62 / above-baseline accuracy for the pr_outcome source. +function prOutcome(accuracy: number) { + const decided = 20; + const correct = Math.round(decided * accuracy); + return { mergeConfirmed: correct, mergeFalse: decided - correct, closeConfirmed: 0, closeFalse: 0, observedAt: NOW }; +} + +function healthyReplay(compositeScore: number) { + return { compositeScore, replayRunId: "replay-1", observedAt: FRESH_REPLAY_AT, harnessStatus: "healthy" as const }; +} + +function loopWith(prAccuracy: number, replayAccuracy: number = prAccuracy) { + return computePhase7CalibrationLoop({ + config: enabledConfig(), + prOutcome: prOutcome(prAccuracy), + historicalReplay: healthyReplay(replayAccuracy), + now: NOW, + }); +} + +describe("phase 7 calibration deltaFromBaseline sign", () => { + it("reports a NEGATIVE deviation when combined accuracy is below the documented baseline", () => { + // Both sources at 0.5 → combined 0.5, which is 0.12 below the 0.62 baseline. The delta is a signed deviation, + // so a below-baseline regression must surface as -0.12, not be clamped to 0 (which would hide the regression). + const result = loopWith(0.5); + expect(result.combinedAccuracy).toBe(0.5); + expect(result.deltaFromBaseline).toBe(-0.12); + expect(renderPhase7CalibrationAuditMarkdown(result)).toContain("delta from baseline: -12.00 percentage points"); + }); + + it("reports a POSITIVE deviation when combined accuracy is above the baseline", () => { + // Both sources at 0.75 → combined 0.75, 0.13 above baseline; the above-baseline path was already correct. + const result = loopWith(0.75); + expect(result.combinedAccuracy).toBe(0.75); + expect(result.deltaFromBaseline).toBe(0.13); + }); + + it("reports a ZERO deviation exactly at the baseline", () => { + // pr_outcome 0.6 + replay 0.64 average to combined 0.62 === baseline → delta 0 (a genuine zero, distinct from + // a below-baseline value the old clamp would have flattened to 0). + const result = loopWith(0.6, 0.64); + expect(result.combinedAccuracy).toBe(DOCUMENTED_CALIBRATION_BASELINE); + expect(result.deltaFromBaseline).toBe(0); + }); +});