coverage.mjs

// Rule-coverage report for engine-allowlisted regulations.
//
// For each rule_id in the regulation, count how many fixtures cause it to
// fire. Reports per-decision and overall coverage. Does not fail CI today
// (advisory only), but the number is the lever for adding fixtures —
// uncovered rules are the gaps we should be filling.
//
// Scope: the same allowlist as run_tests.mjs / lint_conditions.mjs. Other
// regulations don't run through the engine, so coverage isn't measurable
// the same way — we'd just see false zeros.

import path from "node:path";
import { ROOT, fixtureFiles, loadYaml, relPath, header, ok, info, fail } from "./_lib.mjs";
import { evaluateRegulation } from "./engine.mjs";

// Mirror the run_tests.mjs allowlist. anti_dumping_china_v1 sits out
// until the engine learns to query the Postgres-backed register.
const ENGINE_REGULATIONS = new Set([
  // "eu/customs/anti_dumping_china_v1",  ← restore once engine queries Postgres
]);

header("coverage — fixture-driven rule coverage");

const fixturesByReg = new Map();
for (const f of fixtureFiles()) {
  const doc = await loadYaml(f);
  const slug = doc.fixture?.regulation;
  if (!slug || !ENGINE_REGULATIONS.has(slug)) continue;
  if (!fixturesByReg.has(slug)) fixturesByReg.set(slug, []);
  fixturesByReg.get(slug).push({ path: f, doc });
}

let exitCode = 0;
let totalRules = 0;
let totalFired = 0;

for (const slug of ENGINE_REGULATIONS) {
  const reg = await loadYaml(path.join(ROOT, "regulations", `${slug}.yaml`));
  const fixtures = fixturesByReg.get(slug) || [];
  console.log(`\nRegulation: ${slug} (${fixtures.length} fixture(s))`);

  // Build the rule-id index, grouped by decision.
  const rulesByDecision = new Map();
  for (const d of reg.decisions || []) {
    rulesByDecision.set(d.id, (d.rules || []).map(r => r.id));
  }

  // Run every fixture and collect fired rule_ids.
  const firedCount = new Map();
  for (const d of reg.decisions || []) {
    for (const r of d.rules || []) firedCount.set(r.id, 0);
  }
  for (const { path: fp, doc } of fixtures) {
    let result;
    try {
      result = evaluateRegulation(reg, doc.input || {});
    } catch (e) {
      fail(`coverage: ${relPath(fp)} did not evaluate: ${e.message}`);
      exitCode = 0; // advisory; do not fail CI
      continue;
    }
    const seen = new Set();
    for (const r of result.rationale || []) {
      if (!seen.has(r.rule_id)) {
        firedCount.set(r.rule_id, (firedCount.get(r.rule_id) || 0) + 1);
        seen.add(r.rule_id);
      }
    }
  }

  // Per-decision report.
  let regRules = 0, regFired = 0;
  for (const [decisionId, ruleIds] of rulesByDecision) {
    const decisionFired = ruleIds.filter(id => (firedCount.get(id) || 0) > 0);
    regRules += ruleIds.length;
    regFired += decisionFired.length;
    const pct = ruleIds.length === 0 ? 0 : Math.round(100 * decisionFired.length / ruleIds.length);
    console.log(`  ${decisionId.padEnd(30)} ${decisionFired.length}/${ruleIds.length}  (${pct}%)`);
    for (const id of ruleIds) {
      const n = firedCount.get(id) || 0;
      const mark = n > 0 ? "ok " : "** ";
      console.log(`    ${mark}${id.padEnd(38)}  fired in ${n} fixture(s)`);
    }
  }
  const overallPct = regRules === 0 ? 0 : Math.round(100 * regFired / regRules);
  console.log(`  -- ${regFired}/${regRules} rules fired (${overallPct}%)`);
  totalRules += regRules;
  totalFired += regFired;
}

const overallPct = totalRules === 0 ? 0 : Math.round(100 * totalFired / totalRules);
console.log(`\nTotal: ${totalFired}/${totalRules} rules fired (${overallPct}%)`);
process.exit(exitCode);