Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion packages/gittensory-engine/src/phase7-calibration-loop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,14 @@ function roundScore(value: number): number {
return Math.round(Math.min(1, Math.max(0, value)) * 1_000_000) / 1_000_000;
}

/** Round a SIGNED value to 6 dp WITHOUT the [0, 1] clamp `roundScore` applies to accuracy scores. A baseline
* delta legitimately ranges over [-baseline, 1 - baseline], so a below-baseline calibration run must surface as a
* negative deviation — clamping it to 0 would report a regression as "on baseline" and defeat the whole point of
* tracking accuracy against the documented baseline. */
function roundDelta(value: number): number {
return Math.round(value * 1_000_000) / 1_000_000;
}

function finiteNonNegative(value: number | undefined, fallback: number): number {
if (value === undefined) return fallback;
if (!Number.isFinite(value) || value < 0) return 0;
Expand Down Expand Up @@ -462,7 +470,7 @@ export function computePhase7CalibrationLoop(input: {
);

const deltaFromBaseline =
combinedAccuracy === null ? null : roundScore(combinedAccuracy - DOCUMENTED_CALIBRATION_BASELINE);
combinedAccuracy === null ? null : roundDelta(combinedAccuracy - DOCUMENTED_CALIBRATION_BASELINE);

const schedule = shouldScheduleHistoricalReplayRun({
config,
Expand Down
16 changes: 16 additions & 0 deletions packages/gittensory-engine/test/phase7-calibration-loop.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,22 @@ test("computePhase7CalibrationLoop combines historical-replay and pr_outcome sig
assert.equal(result.bySource.pr_outcome.sampleSize, 20);
});

test("computePhase7CalibrationLoop reports a NEGATIVE delta when combined accuracy is below the documented baseline", () => {
// Both sources at 0.5 accuracy → combinedAccuracy 0.5, which is 0.12 BELOW the 0.62 documented baseline. The
// delta is a signed deviation, so a below-baseline regression must surface as -0.12, not be flattened to 0.
const result = computePhase7CalibrationLoop({
config: enabledConfig(),
prOutcome: sufficientPrOutcome(0.5),
historicalReplay: healthyReplay(0.5),
now: NOW,
});

assert.equal(result.combinedAccuracy, 0.5);
assert.equal(result.deltaFromBaseline, -0.12);
// The Markdown audit surfaces the signed magnitude, so the regression is visible to an operator.
assert.match(renderPhase7CalibrationAuditMarkdown(result), /delta from baseline: -12\.00 percentage points/);
});

test("computePhase7CalibrationLoop permits autonomy increases only when both sources meet the threshold", () => {
const passing = computePhase7CalibrationLoop({
config: enabledConfig({ autonomyIncreaseMinAccuracy: 0.7 }),
Expand Down
70 changes: 70 additions & 0 deletions test/unit/phase7-calibration-delta.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import { describe, expect, it } from "vitest";
import {
computePhase7CalibrationLoop,
renderPhase7CalibrationAuditMarkdown,
resolvePhase7CalibrationConfig,
DOCUMENTED_CALIBRATION_BASELINE,
} from "../../packages/gittensory-engine/src/phase7-calibration-loop";

const NOW = "2026-07-04T18:00:00.000Z";
const FRESH_REPLAY_AT = "2026-07-04T12:00:00.000Z";

function enabledConfig() {
return resolvePhase7CalibrationConfig({
miner: {
calibration: {
phase7LoopEnabled: true,
autonomyIncreaseMinAccuracy: 0.7,
replayFreshnessMaxAgeHours: 168,
historicalReplayWeight: 0.5,
prOutcomeWeight: 0.5,
},
},
});
}

// decided=20; correct=round(20*accuracy) → an exact 0.5 / 0.62 / above-baseline accuracy for the pr_outcome source.
function prOutcome(accuracy: number) {
const decided = 20;
const correct = Math.round(decided * accuracy);
return { mergeConfirmed: correct, mergeFalse: decided - correct, closeConfirmed: 0, closeFalse: 0, observedAt: NOW };
}

function healthyReplay(compositeScore: number) {
return { compositeScore, replayRunId: "replay-1", observedAt: FRESH_REPLAY_AT, harnessStatus: "healthy" as const };
}

function loopWith(prAccuracy: number, replayAccuracy: number = prAccuracy) {
return computePhase7CalibrationLoop({
config: enabledConfig(),
prOutcome: prOutcome(prAccuracy),
historicalReplay: healthyReplay(replayAccuracy),
now: NOW,
});
}

describe("phase 7 calibration deltaFromBaseline sign", () => {
it("reports a NEGATIVE deviation when combined accuracy is below the documented baseline", () => {
// Both sources at 0.5 → combined 0.5, which is 0.12 below the 0.62 baseline. The delta is a signed deviation,
// so a below-baseline regression must surface as -0.12, not be clamped to 0 (which would hide the regression).
const result = loopWith(0.5);
expect(result.combinedAccuracy).toBe(0.5);
expect(result.deltaFromBaseline).toBe(-0.12);
expect(renderPhase7CalibrationAuditMarkdown(result)).toContain("delta from baseline: -12.00 percentage points");
});

it("reports a POSITIVE deviation when combined accuracy is above the baseline", () => {
// Both sources at 0.75 → combined 0.75, 0.13 above baseline; the above-baseline path was already correct.
const result = loopWith(0.75);
expect(result.combinedAccuracy).toBe(0.75);
expect(result.deltaFromBaseline).toBe(0.13);
});

it("reports a ZERO deviation exactly at the baseline", () => {
// pr_outcome 0.6 + replay 0.64 average to combined 0.62 === baseline → delta 0 (a genuine zero, distinct from
// a below-baseline value the old clamp would have flattened to 0).
const result = loopWith(0.6, 0.64);
expect(result.combinedAccuracy).toBe(DOCUMENTED_CALIBRATION_BASELINE);
expect(result.deltaFromBaseline).toBe(0);
});
});