From b2db7e66b2961aa63d6c2fc4d07ca9aa9e33595b Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Sun, 29 Mar 2026 17:49:57 +0400 Subject: [PATCH 1/4] feat(simulation): confidence-weighted adjustments + simulationSignal trace lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scale +0.08 and +0.04 simulation bonuses by simPathConfidence (missing/zero falls back to 1.0 so old artifacts are not penalized). Negative adjustments (-0.12/-0.15) remain flat — they are structural, not sim-confidence-dependent. Attach a compact simulationSignal object (backed, adjustmentDelta, channelSource, demoted, simPathConfidence) to each ExpandedPath when adjustment != 0. Written to R2 trace artifacts. ForecastPanel UI chip deferred to follow-up PR (requires proto field + buf generate + prediction-to-path plumbing). Add simPathConfidence to SimulationAdjustmentDetail for observability. Add T-N1..T-N8 (confidence weighting) and T-O1..T-O4 (simulationSignal) tests. 🤖 Generated with Claude Sonnet 4.6 via Claude Code (https://claude.ai/claude-code) + Compound Engineering v2.49.0 Co-Authored-By: Claude Sonnet 4.6 (200K context) --- scripts/seed-forecasts.mjs | 23 ++- scripts/seed-forecasts.types.d.ts | 21 +++ tests/forecast-trace-export.test.mjs | 212 +++++++++++++++++++++++++++ 3 files changed, 253 insertions(+), 3 deletions(-) diff --git a/scripts/seed-forecasts.mjs b/scripts/seed-forecasts.mjs index d59d6c0830..d161c4ea3f 100644 --- a/scripts/seed-forecasts.mjs +++ b/scripts/seed-forecasts.mjs @@ -11398,7 +11398,7 @@ function negatesDisruption(stabilizer, candidatePacket) { */ function computeSimulationAdjustment(expandedPath, simTheaterResult, candidatePacket) { let adjustment = 0; - const details = { bucketChannelMatch: false, actorOverlapCount: 0, invalidatorHit: false, stabilizerHit: false, resolvedChannel: '', channelSource: 'none', candidateActorCount: 0, actorSource: 'none' }; + const details = { bucketChannelMatch: false, actorOverlapCount: 0, invalidatorHit: false, stabilizerHit: false, resolvedChannel: '', channelSource: 'none', candidateActorCount: 0, actorSource: 'none', simPathConfidence: 1.0 }; const { topPaths = [], invalidators = [], stabilizers = [] } = simTheaterResult || {}; const pathBucket = expandedPath?.direct?.targetBucket @@ -11445,13 +11445,23 @@ function computeSimulationAdjustment(expandedPath, simTheaterResult, candidatePa (sp) => matchesBucket(sp, pathBucket) && matchesChannel(sp, pathChannel) ); if (bucketChannelMatch) { - adjustment += 0.08; + // Scale bonuses by sim path confidence. Missing/null/zero confidence falls back to 1.0 + // (no penalty for legacy LLM output that omitted the field). + const rawConf = bucketChannelMatch.confidence; + const simConf = typeof rawConf === 'number' && Number.isFinite(rawConf) && rawConf > 0 + ? Math.min(1, rawConf) + : 1.0; + adjustment += +parseFloat((0.08 * simConf).toFixed(3)); details.bucketChannelMatch = true; + details.simPathConfidence = simConf; const simActors = new Set((Array.isArray(bucketChannelMatch.keyActors) ? bucketChannelMatch.keyActors : []).map(normalizeActorName)); const overlap = candidateActors.filter((a) => simActors.has(a)); details.actorOverlapCount = overlap.length; + // Overlap bonus fires only when both sides have named geo-political actors. + // Macro-financial theaters with role-based stateSummary.actors (e.g. "Commodity traders", + // "Central banks") will have actorOverlapCount=0 — this is expected, not a bug. if (overlap.length >= 2) { - adjustment += 0.04; + adjustment += +parseFloat((0.04 * simConf).toFixed(3)); } } @@ -11525,6 +11535,13 @@ function applySimulationMerge(evaluation, simulationOutcome, candidatePackets, s path.simulationAdjustment = adjustment; path.mergedAcceptanceScore = mergedAcceptanceScore; + path.simulationSignal = { + backed: adjustment > 0, + adjustmentDelta: adjustment, + channelSource: details.channelSource, + demoted: wasAccepted && mergedAcceptanceScore < SIMULATION_MERGE_ACCEPT_THRESHOLD, + simPathConfidence: details.simPathConfidence, + }; if (wasAccepted && mergedAcceptanceScore < SIMULATION_MERGE_ACCEPT_THRESHOLD) { path.demotedBySimulation = true; diff --git a/scripts/seed-forecasts.types.d.ts b/scripts/seed-forecasts.types.d.ts index 2ca0e14a1f..36e1a21f9a 100644 --- a/scripts/seed-forecasts.types.d.ts +++ b/scripts/seed-forecasts.types.d.ts @@ -89,6 +89,23 @@ interface ExpandedPathCandidate { topBucketId?: string; } +/** + * Compact simulation signal attached to an ExpandedPath when a non-zero adjustment was applied. + * Written by applySimulationMerge; rendered as a chip in ForecastPanel. + */ +interface SimulationSignal { + /** Non-zero simulation adjustment was applied (positive = promoted, negative = weakened). */ + backed: boolean; + /** Raw adjustment delta (+0.08/+0.04 weighted by simPathConfidence; -0.12/-0.15 flat). */ + adjustmentDelta: number; + /** Source of the matched channel: 'direct' (from path.direct.channel) | 'market' (from marketContext.topChannel) | 'none'. */ + channelSource: 'direct' | 'market' | 'none'; + /** Path was demoted below the 0.50 acceptance threshold by simulation. */ + demoted: boolean; + /** Confidence of the matched simulation top-path (0–1). Only meaningful when backed=true. */ + simPathConfidence: number; +} + /** A single expanded path produced by the deep forecast LLM evaluation. */ interface ExpandedPath { pathId: string; @@ -99,6 +116,8 @@ interface ExpandedPath { simulationAdjustment?: number; demotedBySimulation?: boolean; promotedBySimulation?: boolean; + /** Compact simulation signal. Present only when applySimulationMerge produced a non-zero adjustment. */ + simulationSignal?: SimulationSignal; direct?: ExpandedPathDirect; candidate?: ExpandedPathCandidate; } @@ -170,6 +189,8 @@ interface SimulationAdjustmentDetail { resolvedChannel: string; /** Source of resolved channel. */ channelSource: 'direct' | 'market' | 'none'; + /** Confidence of the matched simulation top-path (0–1). 1.0 when no bucketChannelMatch or when confidence is missing/null/zero in LLM output. */ + simPathConfidence: number; } interface SimulationAdjustmentRecord { diff --git a/tests/forecast-trace-export.test.mjs b/tests/forecast-trace-export.test.mjs index ced03042d3..6d4652b454 100644 --- a/tests/forecast-trace-export.test.mjs +++ b/tests/forecast-trace-export.test.mjs @@ -6608,6 +6608,115 @@ describe('phase 3 simulation re-ingestion — computeSimulationAdjustment', () = assert.equal(details.actorOverlapCount, 0); assert.equal(details.bucketChannelMatch, true); }); + + it('T-N1: confidence=1.0 (explicit) produces same +0.08 as no-confidence fallback', () => { + const path = makePath('energy', 'energy_supply_shock', []); + const candidatePacket = makeCandidatePacket(); + const simResult = { + topPaths: [{ label: 'Oil supply disruption', summary: 'energy supply disruption', confidence: 1.0, keyActors: [] }], + invalidators: [], stabilizers: [], + }; + const { adjustment, details } = computeSimulationAdjustment(path, simResult, candidatePacket); + assert.equal(adjustment, 0.08); + assert.equal(details.simPathConfidence, 1.0); + assert.equal(details.bucketChannelMatch, true); + }); + + it('T-N2: confidence=0.72 scales +0.08 and +0.04 proportionally', () => { + const path = makePath('energy', 'energy_supply_shock', []); + const candidatePacket = makeCandidatePacket(); // stateSummary.actors: ['Iran', 'Houthi movement', 'US Navy'] + const simResult = { + // keyActors match 'iran' and 'us navy' from stateSummary → overlap=2 → actor bonus applies + topPaths: [{ label: 'Oil supply disruption', summary: 'energy supply disruption', confidence: 0.72, keyActors: ['Iran', 'US_Navy'] }], + invalidators: [], stabilizers: [], + }; + const { adjustment, details } = computeSimulationAdjustment(path, simResult, candidatePacket); + // +0.08 * 0.72 = 0.058, +0.04 * 0.72 = 0.029; total = 0.087 + assert.equal(adjustment, 0.087); + assert.equal(details.simPathConfidence, 0.72); + assert.ok(details.actorOverlapCount >= 2); + }); + + it('T-N3: confidence=0.35 scales +0.08 only (no actor overlap)', () => { + const path = makePath('energy', 'energy_supply_shock', []); + const candidatePacket = makeCandidatePacket(); + const simResult = { + topPaths: [{ label: 'Oil supply disruption', summary: 'energy supply disruption', confidence: 0.35, keyActors: [] }], + invalidators: [], stabilizers: [], + }; + const { adjustment, details } = computeSimulationAdjustment(path, simResult, candidatePacket); + // +0.08 * 0.35 = 0.028, no actor overlap + assert.equal(adjustment, 0.028); + assert.equal(details.simPathConfidence, 0.35); + assert.equal(details.actorOverlapCount, 0); + }); + + it('T-N4: missing confidence (undefined) falls back to 1.0 → full +0.08', () => { + const path = makePath('energy', 'energy_supply_shock', []); + const candidatePacket = makeCandidatePacket(); + const simResult = { + topPaths: [{ label: 'Oil supply disruption', summary: 'energy supply disruption', keyActors: [] }], // no confidence field + invalidators: [], stabilizers: [], + }; + const { adjustment, details } = computeSimulationAdjustment(path, simResult, candidatePacket); + assert.equal(adjustment, 0.08); + assert.equal(details.simPathConfidence, 1.0); + }); + + it('T-N5: confidence=0 falls back to 1.0 → full +0.08 (no penalty for legacy zero)', () => { + const path = makePath('energy', 'energy_supply_shock', []); + const candidatePacket = makeCandidatePacket(); + const simResult = { + topPaths: [{ label: 'Oil supply disruption', summary: 'energy supply disruption', confidence: 0, keyActors: [] }], + invalidators: [], stabilizers: [], + }; + const { adjustment, details } = computeSimulationAdjustment(path, simResult, candidatePacket); + assert.equal(adjustment, 0.08); + assert.equal(details.simPathConfidence, 1.0); + }); + + it('T-N6: invalidator hit produces flat -0.12 regardless of sim path confidence', () => { + const path = makePath('energy', 'energy_supply_shock', []); + const candidatePacket = makeCandidatePacket(); // routeFacilityKey='Strait of Hormuz' + const simResult = { + topPaths: [], // no bucket/channel match + // fromSimulation=true → no negation check needed; 'strait of hormuz' matches routeFacilityKey + invalidators: ['strait of hormuz transit suspended'], + stabilizers: [], + }; + const { adjustment, details } = computeSimulationAdjustment(path, simResult, candidatePacket); + assert.equal(adjustment, -0.12); + assert.equal(details.invalidatorHit, true); + assert.equal(details.simPathConfidence, 1.0); // default — no bucketChannelMatch + }); + + it('T-N7: stabilizer hit produces flat -0.15 regardless of sim path confidence', () => { + const path = makePath('energy', 'energy_supply_shock', []); + const candidatePacket = makeCandidatePacket(); // routeFacilityKey='Strait of Hormuz' + const simResult = { + topPaths: [], + invalidators: [], + // negation 'restored' + 'strait of hormuz' → negatesDisruption=true + stabilizers: ['strait of hormuz shipping lanes restored'], + }; + const { adjustment, details } = computeSimulationAdjustment(path, simResult, candidatePacket); + assert.equal(adjustment, -0.15); + assert.equal(details.stabilizerHit, true); + assert.equal(details.simPathConfidence, 1.0); + }); + + it('T-N8: simPathConfidence in details equals the clamped confidence used for weighting', () => { + const path = makePath('energy', 'energy_supply_shock', []); + const candidatePacket = makeCandidatePacket(); + const simResult = { + topPaths: [{ label: 'Oil supply disruption', summary: 'energy supply disruption', confidence: 0.85, keyActors: [] }], + invalidators: [], stabilizers: [], + }; + const { adjustment, details } = computeSimulationAdjustment(path, simResult, candidatePacket); + assert.equal(details.simPathConfidence, 0.85); + // +0.08 * 0.85 = 0.068 + assert.equal(adjustment, 0.068); + }); }); describe('normalizeActorName', () => { @@ -6778,6 +6887,108 @@ describe('phase 3 simulation re-ingestion — applySimulationMerge', () => { assert.ok(promoted.mergedAcceptanceScore >= 0.50, `score should be >= 0.50, got ${promoted.mergedAcceptanceScore}`); assert.ok(promoted.simulationAdjustment >= 0.12, 'should have +0.12 (bucket+channel + actor overlap)'); }); + + it('T-O1: positive adjustment → simulationSignal.backed=true, adjustmentDelta>0', () => { + const stateId = 'state-o1'; + const path = makeExpandedPath(stateId, 0.44); // rejected; 0.44 + 0.08 = 0.52 → promoted + const candidatePacket = { + candidateStateId: stateId, + routeFacilityKey: 'Red Sea', + commodityKey: 'crude_oil', + marketContext: { topBucketId: 'energy', topChannel: 'energy_supply_shock' }, + stateSummary: { actors: [] }, + }; + const evaluation = makeEval('completed_no_material_change', [], [path]); + const simOutcome = { + runId: 'sim-o1', isCurrentRun: true, + theaterResults: [{ + theaterId: stateId, candidateStateId: stateId, + topPaths: [{ label: 'Oil supply disruption', summary: 'energy supply disruption', keyActors: [] }], + invalidators: [], stabilizers: [], + }], + }; + const snapshot = { generatedAt: Date.now(), impactExpansionCandidates: [candidatePacket], fullRunPredictions: [], predictions: [], inputs: {}, deepForecast: {} }; + applySimulationMerge(evaluation, simOutcome, [candidatePacket], snapshot, null); + assert.ok(path.simulationSignal, 'simulationSignal written'); + assert.equal(path.simulationSignal.backed, true); + assert.ok(path.simulationSignal.adjustmentDelta > 0); + assert.equal(path.simulationSignal.demoted, false); + }); + + it('T-O2: negative adjustment that does not cross threshold → simulationSignal.backed=false, demoted=false', () => { + const stateId = 'state-o2'; + const path = makeExpandedPath(stateId, 0.70); // accepted; 0.70 - 0.12 = 0.58 ≥ 0.50 → NOT demoted + const candidatePacket = { + candidateStateId: stateId, + routeFacilityKey: 'Red Sea', + commodityKey: 'crude_oil', + marketContext: { topBucketId: 'energy', topChannel: 'energy_supply_shock' }, + stateSummary: { actors: [] }, + }; + const evaluation = makeEval('completed', [path]); + const simOutcome = { + runId: 'sim-o2', isCurrentRun: true, + theaterResults: [{ + theaterId: stateId, candidateStateId: stateId, + topPaths: [], + invalidators: ['red sea shipping lanes suspended'], + stabilizers: [], + }], + }; + const snapshot = { generatedAt: Date.now(), impactExpansionCandidates: [candidatePacket], fullRunPredictions: [], predictions: [], inputs: {}, deepForecast: {} }; + applySimulationMerge(evaluation, simOutcome, [candidatePacket], snapshot, null); + assert.ok(path.simulationSignal, 'simulationSignal written'); + assert.equal(path.simulationSignal.backed, false); + assert.ok(path.simulationSignal.adjustmentDelta < 0); + assert.equal(path.simulationSignal.demoted, false); // 0.70 - 0.12 = 0.58 ≥ 0.50 + }); + + it('T-O3: zero adjustment (no matching theater) → simulationSignal is undefined', () => { + const stateId = 'state-o3'; + const path = makeExpandedPath(stateId, 0.60); + const candidatePacket = { + candidateStateId: stateId, + routeFacilityKey: 'Red Sea', + commodityKey: 'crude_oil', + marketContext: { topBucketId: 'energy', topChannel: 'energy_supply_shock' }, + stateSummary: { actors: [] }, + }; + const evaluation = makeEval('completed', [path]); + const simOutcome = { + runId: 'sim-o3', isCurrentRun: true, + theaterResults: [{ theaterId: 'other-state', candidateStateId: 'other-state', topPaths: [], invalidators: [], stabilizers: [] }], + }; + const snapshot = { generatedAt: Date.now(), impactExpansionCandidates: [candidatePacket], fullRunPredictions: [], predictions: [], inputs: {}, deepForecast: {} }; + applySimulationMerge(evaluation, simOutcome, [candidatePacket], snapshot, null); + assert.equal(path.simulationSignal, undefined, 'no simulationSignal when adjustment=0'); + }); + + it('T-O4: path crosses 0.50 downward → simulationSignal.demoted=true', () => { + const stateId = 'state-o4'; + const path = makeExpandedPath(stateId, 0.52); // accepted; 0.52 - 0.15 = 0.37 < 0.50 → demoted + const candidatePacket = { + candidateStateId: stateId, + routeFacilityKey: 'Red Sea', + commodityKey: 'crude_oil', + marketContext: { topBucketId: 'energy', topChannel: 'energy_supply_shock' }, + stateSummary: { actors: [] }, + }; + const evaluation = makeEval('completed', [path]); + const simOutcome = { + runId: 'sim-o4', isCurrentRun: true, + theaterResults: [{ + theaterId: stateId, candidateStateId: stateId, + topPaths: [], + invalidators: [], + stabilizers: ['red sea shipping lanes restored'], + }], + }; + const snapshot = { generatedAt: Date.now(), impactExpansionCandidates: [candidatePacket], fullRunPredictions: [], predictions: [], inputs: {}, deepForecast: {} }; + applySimulationMerge(evaluation, simOutcome, [candidatePacket], snapshot, null); + assert.ok(path.simulationSignal, 'simulationSignal written'); + assert.equal(path.simulationSignal.demoted, true); + assert.ok(path.simulationSignal.adjustmentDelta < 0); + }); }); describe('phase 3 simulation re-ingestion — matching helpers', () => { @@ -7254,4 +7465,5 @@ describe('phase 3 simulation re-ingestion — applyPostSimulationRescore', () => assert.equal(details.bucketChannelMatch, true); assert.equal(adjustment, 0.08); }); + }); From a4c2244dadb236821e707dc547704a25a1241842 Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Sun, 29 Mar 2026 18:20:26 +0400 Subject: [PATCH 2/4] fix(simulation): correct zero-confidence handling + wire simulationSignal to scorecard summaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1: explicit confidence=0 from LLM now correctly yields simConf=0 (no positive bonus) instead of falling back to 1.0. Absent/non-finite confidence still uses 1.0 fallback (conservative — old LLM artifacts without the field are not penalized). The previous rawConf > 0 guard conflated "absent" and "explicitly unsupported" paths. P2: summarizeImpactPathScore now forwards simulationSignal into scorecard summaries, so path-scorecards.json and impact-expansion-debug.json include the new lane alongside simulationAdjustment and mergedAcceptanceScore. Only forecast-eval.json had it before. Also exports summarizeImpactPathScore for direct unit testing. Tests: T-N5 corrected (explicit 0 → no bonus), T-N5b (zero conf + invalidator still fires flat -0.12), T-O5 (summarizeImpactPathScore includes simulationSignal), T-O6 (omits field when absent). 270 passing. 🤖 Generated with Claude Sonnet 4.6 via Claude Code (https://claude.ai/claude-code) + Compound Engineering v2.49.0 Co-Authored-By: Claude Sonnet 4.6 (200K context) --- scripts/seed-forecasts.mjs | 15 ++++++--- tests/forecast-trace-export.test.mjs | 48 ++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/scripts/seed-forecasts.mjs b/scripts/seed-forecasts.mjs index d161c4ea3f..0fca1cc64d 100644 --- a/scripts/seed-forecasts.mjs +++ b/scripts/seed-forecasts.mjs @@ -4588,6 +4588,9 @@ function summarizeImpactPathScore(path = null) { if (path.simulationAdjustment !== undefined) { summary.simulationAdjustment = Number(path.simulationAdjustment); summary.mergedAcceptanceScore = Number(path.mergedAcceptanceScore || path.acceptanceScore || 0); + if (path.simulationSignal !== undefined) { + summary.simulationSignal = path.simulationSignal; + } } return summary; } @@ -11445,12 +11448,13 @@ function computeSimulationAdjustment(expandedPath, simTheaterResult, candidatePa (sp) => matchesBucket(sp, pathBucket) && matchesChannel(sp, pathChannel) ); if (bucketChannelMatch) { - // Scale bonuses by sim path confidence. Missing/null/zero confidence falls back to 1.0 - // (no penalty for legacy LLM output that omitted the field). + // Scale bonuses by sim path confidence. + // Absent or non-finite → 1.0 (conservative fallback for legacy LLM output without this field). + // Explicit 0 → simConf=0, no positive adjustment (if no negatives fire, adj=0 and early exit). const rawConf = bucketChannelMatch.confidence; - const simConf = typeof rawConf === 'number' && Number.isFinite(rawConf) && rawConf > 0 - ? Math.min(1, rawConf) - : 1.0; + const simConf = (typeof rawConf !== 'number' || !Number.isFinite(rawConf)) + ? 1.0 + : Math.min(1, Math.max(0, rawConf)); adjustment += +parseFloat((0.08 * simConf).toFixed(3)); details.bucketChannelMatch = true; details.simPathConfidence = simConf; @@ -16803,6 +16807,7 @@ export { contradictsPremise, negatesDisruption, normalizeActorName, + summarizeImpactPathScore, SIMULATION_MERGE_ACCEPT_THRESHOLD, scoreImpactExpansionQuality, buildImpactExpansionDebugPayload, diff --git a/tests/forecast-trace-export.test.mjs b/tests/forecast-trace-export.test.mjs index 6d4652b454..cffdc7b38a 100644 --- a/tests/forecast-trace-export.test.mjs +++ b/tests/forecast-trace-export.test.mjs @@ -74,6 +74,7 @@ import { contradictsPremise, negatesDisruption, normalizeActorName, + summarizeImpactPathScore, } from '../scripts/seed-forecasts.mjs'; import { @@ -6663,7 +6664,7 @@ describe('phase 3 simulation re-ingestion — computeSimulationAdjustment', () = assert.equal(details.simPathConfidence, 1.0); }); - it('T-N5: confidence=0 falls back to 1.0 → full +0.08 (no penalty for legacy zero)', () => { + it('T-N5: explicit confidence=0 → simConf=0, no positive adjustment (simulation rated path unsupported)', () => { const path = makePath('energy', 'energy_supply_shock', []); const candidatePacket = makeCandidatePacket(); const simResult = { @@ -6671,8 +6672,26 @@ describe('phase 3 simulation re-ingestion — computeSimulationAdjustment', () = invalidators: [], stabilizers: [], }; const { adjustment, details } = computeSimulationAdjustment(path, simResult, candidatePacket); - assert.equal(adjustment, 0.08); - assert.equal(details.simPathConfidence, 1.0); + assert.equal(adjustment, 0); + assert.equal(details.simPathConfidence, 0); + assert.equal(details.bucketChannelMatch, true); // match found but zero-weighted + }); + + it('T-N5b: zero confidence + invalidator → negative adjustment fires flat, positive bonus stays 0', () => { + // The invalidator check is independent of simConf. A zero-confidence bucket-channel match + // contributes 0 positive bonus but the invalidator's -0.12 still fires. + const path = makePath('energy', 'energy_supply_shock', []); + const candidatePacket = makeCandidatePacket(); // routeFacilityKey='Strait of Hormuz' + const simResult = { + topPaths: [{ label: 'Oil supply disruption', summary: 'energy supply disruption', confidence: 0, keyActors: [] }], + invalidators: ['Strait of Hormuz fully operational'], + stabilizers: [], + }; + const { adjustment, details } = computeSimulationAdjustment(path, simResult, candidatePacket); + assert.equal(adjustment, -0.12); + assert.equal(details.simPathConfidence, 0); // zero conf from explicit 0 + assert.equal(details.bucketChannelMatch, true); + assert.equal(details.invalidatorHit, true); }); it('T-N6: invalidator hit produces flat -0.12 regardless of sim path confidence', () => { @@ -6989,6 +7008,29 @@ describe('phase 3 simulation re-ingestion — applySimulationMerge', () => { assert.equal(path.simulationSignal.demoted, true); assert.ok(path.simulationSignal.adjustmentDelta < 0); }); + + it('T-O5: summarizeImpactPathScore includes simulationSignal when present (path-scorecards / impact-expansion-debug coverage)', () => { + const signal = { backed: true, adjustmentDelta: 0.08, channelSource: 'market', demoted: false, simPathConfidence: 0.9 }; + const path = { + pathId: 'p-o5', + type: 'expanded', + candidateStateId: 'state-o5', + acceptanceScore: 0.7, + simulationAdjustment: 0.08, + mergedAcceptanceScore: 0.78, + simulationSignal: signal, + }; + const summary = summarizeImpactPathScore(path); + assert.ok(summary, 'summarizeImpactPathScore should return non-null'); + assert.deepStrictEqual(summary.simulationSignal, signal, 'simulationSignal must be forwarded into scorecard summary'); + }); + + it('T-O6: summarizeImpactPathScore omits simulationSignal when absent (no spurious undefined field)', () => { + const path = { pathId: 'p-o6', type: 'expanded', candidateStateId: 'state-o6', acceptanceScore: 0.6 }; + const summary = summarizeImpactPathScore(path); + assert.ok(summary); + assert.equal(Object.prototype.hasOwnProperty.call(summary, 'simulationSignal'), false); + }); }); describe('phase 3 simulation re-ingestion — matching helpers', () => { From 83c9464a63abb097e780fa579e1d9357b0067dd4 Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Sun, 29 Mar 2026 21:50:39 +0400 Subject: [PATCH 3/4] fix(simulation): clear stale sim fields before re-merge + fix simPathConfidence comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1 (blocking): applySimulationMerge now deletes simulationAdjustment, mergedAcceptanceScore, simulationSignal, demotedBySimulation, and promotedBySimulation from every expanded path before running computeSimulationAdjustment. This fires before any early-continue (no theater match, no candidate packet, zero confidence), so reloaded forecast-eval.json paths from applyPostSimulationRescore never retain stale simulation metadata from a prior cycle. P3: corrected simPathConfidence JSDoc in SimulationAdjustmentDetail and SimulationSignal — absent/non-finite → 1.0 fallback, explicit 0 preserved as 0 (was: "missing/null/zero fall back to 1.0"). Tests: T-O7 (zero-confidence match clears stale fields), T-O8 (no-theater match clears stale fields). 272 passing. 🤖 Generated with Claude Sonnet 4.6 via Claude Code (https://claude.ai/claude-code) + Compound Engineering v2.49.0 Co-Authored-By: Claude Sonnet 4.6 (200K context) --- scripts/seed-forecasts.mjs | 9 +++ scripts/seed-forecasts.types.d.ts | 4 +- tests/forecast-trace-export.test.mjs | 83 ++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) diff --git a/scripts/seed-forecasts.mjs b/scripts/seed-forecasts.mjs index 0fca1cc64d..8b2c9286e9 100644 --- a/scripts/seed-forecasts.mjs +++ b/scripts/seed-forecasts.mjs @@ -11514,6 +11514,15 @@ function applySimulationMerge(evaluation, simulationOutcome, candidatePackets, s for (const path of allPaths) { if (path.type !== 'expanded') continue; + // Clear stale simulation metadata from prior cycles before re-evaluating. + // Must happen before any `continue` so paths with no matching theater or zero + // adjustment don't retain fields written by a different simulation run. + delete path.simulationAdjustment; + delete path.mergedAcceptanceScore; + delete path.simulationSignal; + delete path.demotedBySimulation; + delete path.promotedBySimulation; + const simResult = simByTheater.get(path.candidateStateId); if (!simResult) continue; diff --git a/scripts/seed-forecasts.types.d.ts b/scripts/seed-forecasts.types.d.ts index 36e1a21f9a..c1971ebbb3 100644 --- a/scripts/seed-forecasts.types.d.ts +++ b/scripts/seed-forecasts.types.d.ts @@ -102,7 +102,7 @@ interface SimulationSignal { channelSource: 'direct' | 'market' | 'none'; /** Path was demoted below the 0.50 acceptance threshold by simulation. */ demoted: boolean; - /** Confidence of the matched simulation top-path (0–1). Only meaningful when backed=true. */ + /** Confidence of the matched simulation top-path (0–1). 1.0 when absent/non-finite (fallback). Explicit 0 preserved. Only meaningful when backed=true. */ simPathConfidence: number; } @@ -189,7 +189,7 @@ interface SimulationAdjustmentDetail { resolvedChannel: string; /** Source of resolved channel. */ channelSource: 'direct' | 'market' | 'none'; - /** Confidence of the matched simulation top-path (0–1). 1.0 when no bucketChannelMatch or when confidence is missing/null/zero in LLM output. */ + /** Confidence of the matched simulation top-path (0–1). 1.0 when absent or non-finite (legacy LLM output fallback). Explicit 0 is preserved as 0 — simulation rated the path unsupported. */ simPathConfidence: number; } diff --git a/tests/forecast-trace-export.test.mjs b/tests/forecast-trace-export.test.mjs index cffdc7b38a..1a2adec4d5 100644 --- a/tests/forecast-trace-export.test.mjs +++ b/tests/forecast-trace-export.test.mjs @@ -7025,6 +7025,89 @@ describe('phase 3 simulation re-ingestion — applySimulationMerge', () => { assert.deepStrictEqual(summary.simulationSignal, signal, 'simulationSignal must be forwarded into scorecard summary'); }); + it('T-O7: applySimulationMerge clears stale sim fields on zero-adjustment paths from prior cycles', () => { + // Simulate the applyPostSimulationRescore scenario: a reloaded path already carries + // simulationAdjustment/simulationSignal from a previous run. The fresh simulation + // returns this theater but with zero-weight confidence=0, so adjustment=0. + // The stale fields must be cleared, not left intact. + const stateId = 'state-stale-test'; + const candidatePacket = { + candidateStateId: stateId, + routeFacilityKey: 'Red Sea', + commodityKey: 'crude_oil', + marketContext: { topBucketId: 'energy', topChannel: 'energy_supply_shock' }, + stateSummary: { actors: [] }, + }; + const stalePath = { + pathId: 'p-stale', + type: 'expanded', + candidateStateId: stateId, + acceptanceScore: 0.70, + direct: { variableKey: 'route_disruption', targetBucket: 'energy', channel: 'energy_supply_shock', affectedAssets: [] }, + // Stale fields from a previous simulation cycle: + simulationAdjustment: 0.08, + mergedAcceptanceScore: 0.78, + simulationSignal: { backed: true, adjustmentDelta: 0.08, channelSource: 'market', demoted: false, simPathConfidence: 0.9 }, + demotedBySimulation: false, + promotedBySimulation: false, + }; + const evaluation = makeEval('completed', [stalePath], []); + const simOutcome = { + runId: 'sim-stale', isCurrentRun: true, + theaterResults: [{ + theaterId: 'theater-1', candidateStateId: stateId, + topPaths: [{ label: 'Oil supply disruption', summary: 'energy supply disruption', confidence: 0, keyActors: [] }], + invalidators: [], stabilizers: [], + }], + }; + const snapshot = { generatedAt: Date.now(), impactExpansionCandidates: [candidatePacket], fullRunPredictions: [], predictions: [], inputs: {}, deepForecast: {} }; + applySimulationMerge(evaluation, simOutcome, [candidatePacket], snapshot, null); + // confidence=0 → simConf=0 → adjustment=0 → path was skipped, stale fields must be cleared + assert.equal(stalePath.simulationAdjustment, undefined, 'stale simulationAdjustment must be cleared'); + assert.equal(stalePath.simulationSignal, undefined, 'stale simulationSignal must be cleared'); + assert.equal(stalePath.mergedAcceptanceScore, undefined, 'stale mergedAcceptanceScore must be cleared'); + assert.equal(stalePath.demotedBySimulation, undefined, 'stale demotedBySimulation must be cleared'); + assert.equal(stalePath.promotedBySimulation, undefined, 'stale promotedBySimulation must be cleared'); + }); + + it('T-O8: applySimulationMerge clears stale sim fields when theater has no matching result in fresh simulation', () => { + // A path carried stale fields from a prior run that included its theater. + // The fresh simulation has no result for this theater (different theaters selected). + const stateId = 'state-no-theater'; + const candidatePacket = { + candidateStateId: stateId, + routeFacilityKey: 'Red Sea', + commodityKey: 'crude_oil', + marketContext: { topBucketId: 'energy', topChannel: 'energy_supply_shock' }, + stateSummary: { actors: [] }, + }; + const stalePath = { + pathId: 'p-no-theater', + type: 'expanded', + candidateStateId: stateId, + acceptanceScore: 0.65, + direct: { variableKey: 'route_disruption', targetBucket: 'energy', channel: 'energy_supply_shock', affectedAssets: [] }, + simulationAdjustment: -0.12, + mergedAcceptanceScore: 0.53, + simulationSignal: { backed: false, adjustmentDelta: -0.12, channelSource: 'none', demoted: false, simPathConfidence: 1.0 }, + }; + const evaluation = makeEval('completed', [stalePath], []); + // simOutcome contains a DIFFERENT theater — not stateId + const simOutcome = { + runId: 'sim-other', isCurrentRun: true, + theaterResults: [{ + theaterId: 'theater-1', candidateStateId: 'state-different', + topPaths: [{ label: 'Other path', summary: 'different theater', confidence: 0.8, keyActors: [] }], + invalidators: [], stabilizers: [], + }], + }; + const snapshot = { generatedAt: Date.now(), impactExpansionCandidates: [candidatePacket], fullRunPredictions: [], predictions: [], inputs: {}, deepForecast: {} }; + applySimulationMerge(evaluation, simOutcome, [candidatePacket], snapshot, null); + assert.equal(stalePath.simulationAdjustment, undefined, 'stale simulationAdjustment must be cleared when no theater matches'); + assert.equal(stalePath.simulationSignal, undefined, 'stale simulationSignal must be cleared when no theater matches'); + assert.equal(stalePath.mergedAcceptanceScore, undefined, 'stale mergedAcceptanceScore must be cleared when no theater matches'); + }); + it('T-O6: summarizeImpactPathScore omits simulationSignal when absent (no spurious undefined field)', () => { const path = { pathId: 'p-o6', type: 'expanded', candidateStateId: 'state-o6', acceptanceScore: 0.6 }; const summary = summarizeImpactPathScore(path); From c3d90b875a17ec5078d3d9aab7100add5e2133e0 Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Sun, 29 Mar 2026 22:00:24 +0400 Subject: [PATCH 4/4] =?UTF-8?q?fix(types):=20correct=20SimulationSignal.ba?= =?UTF-8?q?cked=20JSDoc=20=E2=80=94=20only=20true=20for=20positive=20adjus?= =?UTF-8?q?tments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/seed-forecasts.types.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/seed-forecasts.types.d.ts b/scripts/seed-forecasts.types.d.ts index c1971ebbb3..fd25726a8c 100644 --- a/scripts/seed-forecasts.types.d.ts +++ b/scripts/seed-forecasts.types.d.ts @@ -94,7 +94,7 @@ interface ExpandedPathCandidate { * Written by applySimulationMerge; rendered as a chip in ForecastPanel. */ interface SimulationSignal { - /** Non-zero simulation adjustment was applied (positive = promoted, negative = weakened). */ + /** Simulation added a positive bonus to this path (bucket-channel match fired). False for negative-only adjustments (invalidator/stabilizer hit without a bucket-channel match). */ backed: boolean; /** Raw adjustment delta (+0.08/+0.04 weighted by simPathConfidence; -0.12/-0.15 flat). */ adjustmentDelta: number;