cluster: decode_deterministic 三值 enum + honest gate 抽 helper + CLI 值域 (#118, #120)

CHANGELOG.md

@@ -11,7 +11,8 @@ All notable changes to bestASR are documented here. The format follows
   deterministic measurement snapshot. Runs the priority-1 runnable candidates
   (default ceiling; `--all-grid` widens to the whole grid) over the **canonical
   community corpora** (`bestasr bench pull`) with `--decode-deterministic`
-  (binding for Whisper-family backends; forwarded elsewhere), so the local
+  (binding for Whisper-family backends; a silent no-op elsewhere — see the
+  `decode_deterministic` entry below), so the local
   store gains a "performance of each model under this pinned version" record
   (measurements carry `app_version` / `macos_version` / machine id). Canonical
   corpora are what `bestasr bench submit` can publish — the bench leaderboard
@@ -28,13 +29,27 @@ All notable changes to bestASR are documented here. The format follows
   `run_kind` (`release-sweep` | `adhoc`) marks how a row was produced, so a
   per-version snapshot's candidate census can be checked mechanically instead
   of assumed; the new `bestasr benchmark --run-kind` flag carries it, and
-  `scripts/release-sweep.sh` stamps `release-sweep`. `decode_deterministic` is
-  **tri-state on purpose**: `true`/`false` only for backends that actually
-  consume `--decode-deterministic` (WhisperKit, whisper.cpp), and `null` for
-  backends where the flag is a silent no-op (mlx-audio) — a row never claims a
-  decode condition its backend ignored. The bench side validates both only when
-  present (`PsychQuant/bestASR-bench@e728f1a`). Whether the determinism axis
-  should become a per-backend enum is tracked in #118.
+  `scripts/release-sweep.sh` stamps `release-sweep`. `decode_deterministic`
+  records the decode condition as a **three-value enum** (#118):
+  `deterministic-enforced` / `fallback-enabled` for backends that actually
+  consume `--decode-deterministic` (WhisperKit, whisper.cpp), and
+  `flag-not-consumed` for backends that ignore it (mlx-audio's is a silent
+  no-op; the Fluid backends have no such knob) — which makes no claim about
+  whether those decodes are reproducible, rather than lying in either
+  direction. An **absent** field still means "legacy row, predates the field";
+  `null` is never written, though a reader accepts it as equivalent to absent.
+  A value the reader does not recognize — including a `decode_deterministic`
+  boolean from the earlier shape of this same unreleased entry — is **rejected**,
+  not coerced; no released version ever wrote one, and no row in either repo or
+  any local store carries one. The bench side validates both only when present
+  (`PsychQuant/bestASR-bench@c93a70e`).
+
+  `--run-kind` also gained a value domain at the CLI (#120): a typo now fails at
+  parse (`exit 64`) instead of surviving into a submission and failing in the
+  bench repo's CI. The stored field stays a plain string on purpose — that
+  vocabulary is human-typed provenance and already incomplete (the regression
+  gate benchmarks with no `--run-kind` at all), so closing it at the row type
+  would trade a loud CI failure for a silently dropped row.
 
 ### Fixed
 

Sources/BestASRKit/CommandCore.swift

@@ -587,12 +587,12 @@ public struct CommandCore: Sendable {
                 contextErrorRate: measured.contextErrorRate,
                 hfRevision: seededRow?.hfRevision,
                 runKind: runKind,
-                // decode_deterministic is meaningful only for whisper-family backends that
-                // actually consume the flag; mlx-audio ignores it (silent no-op, #111/#118),
-                // so record nil there rather than lie.
-                decodeDeterministic:
-                    [ModelGrid.backendWhisperKit, ModelGrid.backendWhisperCpp].contains(record.backend)
-                    ? decodeDeterministic : nil))
+                // The honest gate lives in DecodeDeterminism.forBackend (#120
+                // item 1) so the "never lie" invariant is unit-testable without
+                // running a real backend. Non-optional by construction: a live
+                // measurement always knows something; nil is for legacy rows.
+                decodeDeterministic: .forBackend(
+                    record.backend, flagRequested: decodeDeterministic)))
         }
         }
 

Sources/BestASRKit/Contribution/Sharing.swift

@@ -35,9 +35,10 @@ public struct SubmissionRow: Codable, Sendable, Equatable {
     /// Provenance of the underlying measurement (#111) — carried through so the
     /// published row is self-contained. nil for legacy rows.
     public let runKind: String?
-    /// Deterministic-decode flag as recorded on the measurement (#111); nil for
-    /// backends that ignore it (mlx-audio) and for legacy rows.
-    public let decodeDeterministic: Bool?
+    /// Decode-determinism condition as recorded on the measurement (#111/#118);
+    /// nil ONLY for legacy rows that predate the field — a backend that ignores
+    /// `--decode-deterministic` publishes `.flagNotConsumed`, not nil.
+    public let decodeDeterministic: DecodeDeterminism?
     public let contributor: String
     public let chip: String
     public let unifiedMemoryGB: Double

Sources/BestASRKit/Models/DataModels.swift

@@ -194,6 +194,83 @@ public enum MetricKind: String, Codable, Sendable {
     case wer
 }
 
+/// What a measurement row may HONESTLY say about the decode that produced it
+/// (#118). Records what we KNOW, not what the backend is — the pre-#118 `Bool?`
+/// overloaded `nil` to mean both "legacy row" and "this backend ignores the
+/// flag". A row's field stays `Optional`: an ABSENT field (nil) means the row
+/// predates the field entirely and makes no claim at all; `nil` is deliberately
+/// NOT one of the cases below.
+///
+/// Wire values are kebab-case and are a cross-repo contract with the bench
+/// validator (bestASR-bench `tools/validate_measurements.py`).
+///
+/// FORWARD COMPAT: decoding is strict — an unrecognized wire value throws, and
+/// both consumers turn that throw into a quiet drop (`BenchmarkStore.load`
+/// records a warning nothing reads; `SubmissionPackager.publishedKeys` uses
+/// `try?`). So adding a case here is a BREAKING change for any older client
+/// still reading the same store, and it fails invisibly rather than loudly.
+/// Adding a fourth case therefore needs a migration story, not just an
+/// `case` line (#131).
+public enum DecodeDeterminism: String, Codable, Sendable {
+    /// The backend consumed `--decode-deterministic` and it was ON — i.e. the
+    /// deterministic *setting was enforced*: temperature-fallback re-decoding
+    /// was disabled for this run. That is what we observed; it is deliberately
+    /// NOT a promise that re-running yields byte-identical text, which also
+    /// depends on runtime, hardware and model revision (#118: never claim more
+    /// than the evidence).
+    case deterministicEnforced = "deterministic-enforced"
+    /// The backend consumed `--decode-deterministic` and it was OFF: temperature
+    /// fallback re-decoding is live, so a re-run may differ.
+    case fallbackEnabled = "fallback-enabled"
+    /// The backend ignores `--decode-deterministic` entirely (mlx-audio treats it
+    /// as a silent no-op; the Fluid backends have no such knob). This makes NO
+    /// claim about whether that backend's decode is actually reproducible — we
+    /// don't know, and saying either "deterministic" or "fallback" would be a
+    /// lie (#118).
+    case flagNotConsumed = "flag-not-consumed"
+
+    /// Backends that actually read `--decode-deterministic`. Adding a backend
+    /// here is the ONE edit that lets its rows start making a determinism claim
+    /// — until then it honestly reports `.flagNotConsumed`.
+    public static let flagConsumingBackends: Set<String> = [
+        ModelGrid.backendWhisperKit, ModelGrid.backendWhisperCpp,
+    ]
+
+    /// The honest gate (#120 item 1): which condition a row measured on
+    /// `backend` may claim, given the flag the user asked for. Non-optional —
+    /// every real measurement can say *something*; the row's optionality is
+    /// reserved for legacy rows that predate the field.
+    public static func forBackend(_ backend: String, flagRequested: Bool) -> DecodeDeterminism {
+        guard flagConsumingBackends.contains(backend) else { return .flagNotConsumed }
+        return flagRequested ? .deterministicEnforced : .fallbackEnabled
+    }
+}
+
+/// How a measurement was produced (#111) — the `run_kind` value domain as the
+/// CLI accepts it (#120 item 2).
+///
+/// SCOPE, deliberately: this constrains the `--run-kind` OPTION, not the stored
+/// field. `MeasurementRow.runKind` / `SubmissionRow.runKind` stay `String?`, so
+/// a library caller can still store a value outside this domain, and bench CI
+/// stays the backstop for that path. That is the opposite choice from
+/// `DecodeDeterminism`, on purpose: that field's domain is *derived* from this
+/// repo's own backend roster (a total function over it), whereas run_kind is
+/// human-typed provenance whose vocabulary is already demonstrably incomplete —
+/// `scripts/regression-gate.sh` benchmarks with no `--run-kind` at all. Closing
+/// this domain at the row type would trade a loud CI failure for a silent
+/// stale-data read (an undecodable row is dropped, promoting a superseded one).
+///
+/// KEEP IN SYNC with the bench validator's `run_kind` set (bestASR-bench
+/// `tools/validate_measurements.py`). There is no mechanical link: a value added
+/// here and not there fails only in the bench repo's CI, and vice versa
+/// (#120 Residue, deliberately unfixed).
+public enum RunKind: String, Codable, Sendable, CaseIterable {
+    /// A sweep driven by `scripts/release-sweep.sh`.
+    case releaseSweep = "release-sweep"
+    /// A one-off local benchmark.
+    case adhoc
+}
+
 /// One (backend × model × quantization) configuration to measure.
 public struct BenchmarkCandidate: Sendable, Equatable, Hashable {
     public let backend: BackendID

Sources/BestASRKit/Store/BenchmarkStore.swift

@@ -160,7 +160,9 @@ public struct BenchmarkStore: Sendable {
                 quantization: record.quantization)
             // Legacy flat-cache rows predate the #111 provenance fields — there
             // is no run-kind and no decode-deterministic flag to recover here, so
-            // both stay nil (legacy-safe) rather than being fabricated.
+            // both stay nil (legacy-safe) rather than being fabricated. Under
+            // #118 that nil is unambiguous: "the field did not exist when this
+            // was measured", NOT "this backend ignores the flag".
             let measurement = MeasurementRow(
                 modelId: modelId, corpusId: corpus.corpusId, machineId: machine.machineId,
                 measuredAt: record.measuredAt, metricKind: record.metricKind,

Sources/BestASRKit/Store/StoreTables.swift

@@ -160,11 +160,11 @@ public struct MeasurementRow: Codable, Sendable, Equatable {
     /// How this measurement was produced: "release-sweep" (scripts/release-sweep.sh)
     /// vs "adhoc" (a one-off local benchmark). nil for legacy rows (#111).
     public let runKind: String?
-    /// Whether temperature-fallback re-decoding was disabled for reproducibility.
-    /// Meaningful only for whisper-family backends that consume the flag; nil for
-    /// backends that ignore it (mlx-audio is a silent no-op, #111/#118) and for
-    /// legacy rows measured before the flag existed.
-    public let decodeDeterministic: Bool?
+    /// What is known about this row's decode: enforced determinism, live
+    /// temperature fallback, or a backend that ignores the flag (#118 — see
+    /// `DecodeDeterminism`). nil ONLY for legacy rows measured before the field
+    /// existed; "this backend ignores the flag" is `.flagNotConsumed`, not nil.
+    public let decodeDeterministic: DecodeDeterminism?
 
     enum CodingKeys: String, CodingKey {
         case modelId = "model_id"
@@ -189,7 +189,7 @@ public struct MeasurementRow: Codable, Sendable, Equatable {
         metricKind: MetricKind, errorRate: Double, rtf: Double, peakMemoryGB: Double,
         warmupSeconds: Double, appVersion: String, macosVersion: String,
         contextErrorRate: Double? = nil, hfRevision: String? = nil,
-        runKind: String? = nil, decodeDeterministic: Bool? = nil
+        runKind: String? = nil, decodeDeterministic: DecodeDeterminism? = nil
     ) {
         self.modelId = modelId
         self.corpusId = corpusId

Sources/bestasr/BestASRCommand.swift

@@ -32,6 +32,14 @@ struct BestASR: AsyncParsableCommand {
 // (Same package as the enum, so no @retroactive.)
 extension HallucinationFilterMode: ExpressibleByArgument {}
 
+// #120 item 2: --run-kind used to take an arbitrary String, so a typo travelled
+// verbatim into the store and only failed in the bench repo's CI — fail-loud at
+// the wrong end. Typing the option rejects unknown values at the CLI boundary
+// (ArgumentParser's usage error, exactly like --hallucination-filter) and lists
+// the domain in --help. The value set itself lives on BestASRKit.RunKind, which
+// carries the "keep in sync with the bench validator" note.
+extension RunKind: ExpressibleByArgument {}
+
 // Command handlers delegate to BestASRKit.CommandCore (design D1: the executable
 // stays a thin argument-parsing shell; behavior lives in the library where the
 // test target can reach it).
@@ -188,12 +196,15 @@ struct Benchmark: AsyncParsableCommand {
             """)
     var decodeDeterministic = false
 
+    // Value domain enforced at parse time (#120 item 2); unset stays nil, which
+    // is a valid state ("no provenance tag"). The domain must stay in sync with
+    // the bench validator's run_kind enum — see BestASRKit.RunKind.
     @Option(
         help: """
-            Provenance tag stamped onto each measurement's run_kind field \
-            (e.g. release-sweep for scripts/release-sweep.sh; omit for ad-hoc runs)
+            Provenance tag stamped onto each measurement's run_kind field: \
+            release-sweep (scripts/release-sweep.sh) | adhoc; omit to leave it unset
             """)
-    var runKind: String?
+    var runKind: RunKind?
 
     @Flag(help: "Emit machine-readable JSON instead of the table")
     var json = false
@@ -212,7 +223,7 @@ struct Benchmark: AsyncParsableCommand {
                     contextDir: contextDir,
                     allGrid: allGrid,
                     decodeDeterministic: decodeDeterministic,
-                    runKind: runKind
+                    runKind: runKind?.rawValue
                 )
             )
         }