From 521ce5469835d6ead39ab196f3e3815b5281fe67 Mon Sep 17 00:00:00 2001 From: che cheng Date: Fri, 31 Jul 2026 01:41:52 +0800 Subject: [PATCH] feat: run_kind + decode_deterministic provenance on measurement schema (#111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add two optional provenance fields to MeasurementRow + SubmissionRow, mirroring the hf_revision precedent (optional, nil-default, snake_case CodingKeys, synthesized-Codable free backward-compat — legacy rows decode to nil): - run_kind (String?, release-sweep|adhoc|nil): distinguishes release-sweep's full-matrix snapshot rows from ad-hoc submissions so per-version snapshot census completeness is mechanically checkable. Threaded via a new 'bestasr benchmark --run-kind' flag; release-sweep.sh passes release-sweep. - decode_deterministic (Bool?): HONEST tri-state — true/false only for backends that consume --decode-deterministic (WhisperKit / whisper.cpp, gated via ModelGrid.backendWhisperKit/WhisperCpp); nil for mlx-audio (silent no-op) and legacy rows. Never claims a value for a backend that ignored the flag. The representation-upgrade decision (Bool?+nil vs per-backend enum) is #118. The legacy flat-cache migration path (BenchmarkStore) has no run-kind/decode info in scope and passes nil/nil (legacy-safe). 5 new backward-compat tests; full suite (389) green. Refs #111 --- Sources/BestASRKit/CommandCore.swift | 12 +- Sources/BestASRKit/Contribution/Sharing.swift | 12 +- Sources/BestASRKit/Store/BenchmarkStore.swift | 6 +- Sources/BestASRKit/Store/StoreTables.swift | 15 ++- Sources/bestasr/BestASRCommand.swift | 10 +- .../RunKindDecodeDeterministicTests.swift | 103 ++++++++++++++++++ scripts/release-sweep.sh | 4 +- 7 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 Tests/BestASRKitTests/RunKindDecodeDeterministicTests.swift diff --git a/Sources/BestASRKit/CommandCore.swift b/Sources/BestASRKit/CommandCore.swift index 15cbec8..812cb0a 100644 --- a/Sources/BestASRKit/CommandCore.swift +++ b/Sources/BestASRKit/CommandCore.swift @@ -470,7 +470,8 @@ public struct CommandCore: Sendable { asJSON: Bool, contextDir: String? = nil, allGrid: Bool = false, - decodeDeterministic: Bool = false + decodeDeterministic: Bool = false, + runKind: String? = nil ) async throws -> String { let profile = try Self.parseProfile(profileName) @@ -584,7 +585,14 @@ public struct CommandCore: Sendable { peakMemoryGB: record.peakMemoryGB, warmupSeconds: measured.warmupSeconds, appVersion: record.appVersion, macosVersion: record.macosVersion, contextErrorRate: measured.contextErrorRate, - hfRevision: seededRow?.hfRevision)) + hfRevision: seededRow?.hfRevision, + runKind: runKind, + // decode_deterministic is meaningful only for whisper-family backends that + // actually consume the flag; mlx-audio ignores it (silent no-op, #111/#118), + // so record nil there rather than lie. + decodeDeterministic: + [ModelGrid.backendWhisperKit, ModelGrid.backendWhisperCpp].contains(record.backend) + ? decodeDeterministic : nil)) } } diff --git a/Sources/BestASRKit/Contribution/Sharing.swift b/Sources/BestASRKit/Contribution/Sharing.swift index b651ea3..8fbf541 100644 --- a/Sources/BestASRKit/Contribution/Sharing.swift +++ b/Sources/BestASRKit/Contribution/Sharing.swift @@ -32,6 +32,12 @@ public struct SubmissionRow: Codable, Sendable, Equatable { public let macosVersion: String public let contextErrorRate: Double? public let hfRevision: String? + /// Provenance of the underlying measurement (#111) — carried through so the + /// published row is self-contained. nil for legacy rows. + public let runKind: String? + /// Deterministic-decode flag as recorded on the measurement (#111); nil for + /// backends that ignore it (mlx-audio) and for legacy rows. + public let decodeDeterministic: Bool? public let contributor: String public let chip: String public let unifiedMemoryGB: Double @@ -50,6 +56,8 @@ public struct SubmissionRow: Codable, Sendable, Equatable { case macosVersion = "macos_version" case contextErrorRate = "context_error_rate" case hfRevision = "hf_revision" + case runKind = "run_kind" + case decodeDeterministic = "decode_deterministic" case contributor, chip case unifiedMemoryGB = "unified_memory_gb" } @@ -87,7 +95,9 @@ public enum SubmissionPackager { errorRate: row.errorRate, rtf: row.rtf, peakMemoryGB: row.peakMemoryGB, warmupSeconds: row.warmupSeconds, appVersion: row.appVersion, macosVersion: row.macosVersion, contextErrorRate: row.contextErrorRate, - hfRevision: row.hfRevision, contributor: contributor, + hfRevision: row.hfRevision, + runKind: row.runKind, decodeDeterministic: row.decodeDeterministic, + contributor: contributor, chip: machine.chip, unifiedMemoryGB: machine.unifiedMemoryGB) return publishedKeys.contains(submission.dedupeKey) ? nil : submission } diff --git a/Sources/BestASRKit/Store/BenchmarkStore.swift b/Sources/BestASRKit/Store/BenchmarkStore.swift index 40117d8..37832aa 100644 --- a/Sources/BestASRKit/Store/BenchmarkStore.swift +++ b/Sources/BestASRKit/Store/BenchmarkStore.swift @@ -158,12 +158,16 @@ public struct BenchmarkStore: Sendable { let modelId = ModelRow.id( backend: record.backend, family: "whisper", size: record.model, quantization: record.quantization) + // Legacy flat-cache rows predate the #111 provenance fields — there + // is no run-kind and no decode-deterministic flag to recover here, so + // both stay nil (legacy-safe) rather than being fabricated. let measurement = MeasurementRow( modelId: modelId, corpusId: corpus.corpusId, machineId: machine.machineId, measuredAt: record.measuredAt, metricKind: record.metricKind, errorRate: record.errorRate, rtf: record.rtf, peakMemoryGB: record.peakMemoryGB, warmupSeconds: 0, - appVersion: record.appVersion, macosVersion: record.macosVersion) + appVersion: record.appVersion, macosVersion: record.macosVersion, + runKind: nil, decodeDeterministic: nil) try appendLine(table: "measurements", row: measurement) } try FileManager.default.moveItem( diff --git a/Sources/BestASRKit/Store/StoreTables.swift b/Sources/BestASRKit/Store/StoreTables.swift index a2f93bc..ee21f7f 100644 --- a/Sources/BestASRKit/Store/StoreTables.swift +++ b/Sources/BestASRKit/Store/StoreTables.swift @@ -157,6 +157,14 @@ public struct MeasurementRow: Codable, Sendable, Equatable { /// number was measured against must live on the measurement itself). /// nil for legacy rows and models without an HF pin. public let hfRevision: String? + /// How this measurement was produced: "release-sweep" (scripts/release-sweep.sh) + /// vs "adhoc" (a one-off local benchmark). nil for legacy rows (#111). + public let runKind: String? + /// Whether temperature-fallback re-decoding was disabled for reproducibility. + /// Meaningful only for whisper-family backends that consume the flag; nil for + /// backends that ignore it (mlx-audio is a silent no-op, #111/#118) and for + /// legacy rows measured before the flag existed. + public let decodeDeterministic: Bool? enum CodingKeys: String, CodingKey { case modelId = "model_id" @@ -172,13 +180,16 @@ public struct MeasurementRow: Codable, Sendable, Equatable { case macosVersion = "macos_version" case contextErrorRate = "context_error_rate" case hfRevision = "hf_revision" + case runKind = "run_kind" + case decodeDeterministic = "decode_deterministic" } public init( modelId: String, corpusId: String, machineId: String, measuredAt: Date, metricKind: MetricKind, errorRate: Double, rtf: Double, peakMemoryGB: Double, warmupSeconds: Double, appVersion: String, macosVersion: String, - contextErrorRate: Double? = nil, hfRevision: String? = nil + contextErrorRate: Double? = nil, hfRevision: String? = nil, + runKind: String? = nil, decodeDeterministic: Bool? = nil ) { self.modelId = modelId self.corpusId = corpusId @@ -193,6 +204,8 @@ public struct MeasurementRow: Codable, Sendable, Equatable { self.macosVersion = macosVersion self.contextErrorRate = contextErrorRate self.hfRevision = hfRevision + self.runKind = runKind + self.decodeDeterministic = decodeDeterministic } } diff --git a/Sources/bestasr/BestASRCommand.swift b/Sources/bestasr/BestASRCommand.swift index 3924323..7bf9e7f 100644 --- a/Sources/bestasr/BestASRCommand.swift +++ b/Sources/bestasr/BestASRCommand.swift @@ -188,6 +188,13 @@ struct Benchmark: AsyncParsableCommand { """) var decodeDeterministic = false + @Option( + help: """ + Provenance tag stamped onto each measurement's run_kind field \ + (e.g. release-sweep for scripts/release-sweep.sh; omit for ad-hoc runs) + """) + var runKind: String? + @Flag(help: "Emit machine-readable JSON instead of the table") var json = false @@ -204,7 +211,8 @@ struct Benchmark: AsyncParsableCommand { asJSON: json, contextDir: contextDir, allGrid: allGrid, - decodeDeterministic: decodeDeterministic + decodeDeterministic: decodeDeterministic, + runKind: runKind ) ) } diff --git a/Tests/BestASRKitTests/RunKindDecodeDeterministicTests.swift b/Tests/BestASRKitTests/RunKindDecodeDeterministicTests.swift new file mode 100644 index 0000000..ccd1757 --- /dev/null +++ b/Tests/BestASRKitTests/RunKindDecodeDeterministicTests.swift @@ -0,0 +1,103 @@ +import Foundation +import Testing + +@testable import BestASRKit + +/// #111 provenance schema: two optional fields (`run_kind`, +/// `decode_deterministic`) added to the measurement + submission rows, mirroring +/// the `hf_revision` precedent — optional, nil-defaulted, snake_case, and +/// backward-compatible via synthesized Codable (nil optionals encode as absent +/// keys, absent keys decode as nil). +struct RunKindDecodeDeterministicTests { + private func iso8601Encoder() -> JSONEncoder { + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + encoder.outputFormatting = [.sortedKeys] + return encoder + } + + private func iso8601Decoder() -> JSONDecoder { + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + return decoder + } + + // MARK: MeasurementRow + + @Test func `Legacy measurement JSON without the new keys decodes to nil`() throws { + // A row written before #111 — no run_kind / decode_deterministic keys. + let legacy = """ + {"app_version":"0.3.0","corpus_id":"c","error_rate":0.2,"machine_id":"m","macos_version":"27.0","measured_at":"2026-01-01T00:00:00Z","metric_kind":"wer","model_id":"whisperkit|whisper|tiny|default","peak_memory_gb":0.4,"rtf":0.1,"warmup_seconds":2} + """ + let row = try iso8601Decoder().decode(MeasurementRow.self, from: Data(legacy.utf8)) + #expect(row.runKind == nil) + #expect(row.decodeDeterministic == nil) + } + + @Test func `Measurement row round-trips the new fields preserving values`() throws { + let row = MeasurementRow( + modelId: "whisperkit|whisper|tiny|default", corpusId: "c", machineId: "m", + measuredAt: Date(timeIntervalSince1970: 1_800_000_000), metricKind: .wer, + errorRate: 0.1, rtf: 0.2, peakMemoryGB: 1, warmupSeconds: 1, + appVersion: "0.4.0", macosVersion: "27.0", + runKind: "release-sweep", decodeDeterministic: true) + let json = String(decoding: try iso8601Encoder().encode(row), as: UTF8.self) + // snake_case keys on disk (spec: BCNF store field naming). + #expect(json.contains("\"run_kind\"")) + #expect(json.contains("\"decode_deterministic\"")) + let decoded = try iso8601Decoder().decode(MeasurementRow.self, from: Data(json.utf8)) + #expect(decoded == row) + #expect(decoded.runKind == "release-sweep") + #expect(decoded.decodeDeterministic == true) + } + + @Test func `Nil new fields encode as absent keys, mirroring hf_revision`() throws { + // Default init leaves runKind / decodeDeterministic (and hfRevision) nil; + // synthesized Codable omits nil optionals, so the keys never appear. + let row = MeasurementRow( + modelId: "whisperkit|whisper|tiny|default", corpusId: "c", machineId: "m", + measuredAt: Date(timeIntervalSince1970: 1_800_000_000), metricKind: .wer, + errorRate: 0.1, rtf: 0.2, peakMemoryGB: 1, warmupSeconds: 1, + appVersion: "0.4.0", macosVersion: "27.0") + let json = String(decoding: try iso8601Encoder().encode(row), as: UTF8.self) + #expect(!json.contains("run_kind")) + #expect(!json.contains("decode_deterministic")) + #expect(!json.contains("hf_revision")) // same omit-nil behavior we mirror + } + + // MARK: SubmissionRow (denormalized publish row carries the same provenance) + + @Test func `Submission packaging threads the new provenance through`() throws { + let row = MeasurementRow( + modelId: "whisperkit|whisper|large-v3|default", corpusId: "abc123abc123", + machineId: MachineRow.id(chip: "Apple M5 Max", unifiedMemoryGB: 128), + measuredAt: Date(timeIntervalSince1970: 1_752_800_000), metricKind: .cer, + errorRate: 0.12, rtf: 0.14, peakMemoryGB: 3.1, warmupSeconds: 8.0, + appVersion: "0.14.0", macosVersion: "26.0", + runKind: "release-sweep", decodeDeterministic: true) + let submissions = SubmissionPackager.package( + local: [row], machines: [MachineRow(chip: "Apple M5 Max", unifiedMemoryGB: 128)], + canonicalCorpusIds: ["abc123abc123"], publishedKeys: [], contributor: "che") + #expect(submissions.count == 1) + #expect(submissions[0].runKind == "release-sweep") + #expect(submissions[0].decodeDeterministic == true) + + // JSONL round-trips the snake_case keys. + let jsonl = try SubmissionPackager.encodeJSONL(submissions) + #expect(jsonl.contains("\"run_kind\"")) + #expect(jsonl.contains("\"decode_deterministic\"")) + let decoded = try iso8601Decoder().decode( + SubmissionRow.self, from: Data(jsonl.trimmingCharacters(in: .whitespacesAndNewlines).utf8)) + #expect(decoded.runKind == "release-sweep") + #expect(decoded.decodeDeterministic == true) + } + + @Test func `Legacy submission JSON without the new keys decodes to nil`() throws { + let legacy = """ + {"app_version":"0.14.0","chip":"Apple M5 Max","contributor":"che","corpus_id":"abc123abc123","error_rate":0.12,"machine_id":"deadbeefdead","macos_version":"26.0","measured_at":"2026-01-01T00:00:00Z","metric_kind":"cer","model_id":"whisperkit|whisper|large-v3|default","peak_memory_gb":3.1,"rtf":0.14,"unified_memory_gb":128,"warmup_seconds":8} + """ + let row = try iso8601Decoder().decode(SubmissionRow.self, from: Data(legacy.utf8)) + #expect(row.runKind == nil) + #expect(row.decodeDeterministic == nil) + } +} diff --git a/scripts/release-sweep.sh b/scripts/release-sweep.sh index 4d430d0..501cb5f 100755 --- a/scripts/release-sweep.sh +++ b/scripts/release-sweep.sh @@ -128,7 +128,7 @@ if [ -n "$DRY_RUN" ]; then echo "" echo "— dry run: would execute —" while IFS=$'\t' read -r name lang wav ref; do - printf ' %q benchmark %q --reference %q --language %q --decode-deterministic%s%s%s --json\n' \ + printf ' %q benchmark %q --reference %q --language %q --decode-deterministic --run-kind release-sweep%s%s%s --json\n' \ "$BIN" "$wav" "$ref" "$lang" \ "${ALL_GRID:+ --all-grid}" \ "${BACKENDS:+ --backends $BACKENDS}" "${MODELS:+ --models $MODELS}" @@ -147,7 +147,7 @@ while IFS=$'\t' read -r name lang wav ref; do # would silently eat the remaining corpus lines (regression-gate.sh lesson). set +e OUT_JSON=$("$BIN" benchmark "$wav" --reference "$ref" --language "$lang" \ - --decode-deterministic \ + --decode-deterministic --run-kind release-sweep \ ${ALL_GRID:+--all-grid} \ ${BACKENDS:+--backends "$BACKENDS"} ${MODELS:+--models "$MODELS"} \ --json