Retarget prompts to the actual validating GPU architecture (root cause)

CHANGELOG.md

@@ -33,6 +33,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 
 ### Changed
+- **Prompts are retargeted to the actual validating GPU (root-cause dataset-quality fix)**:
+  `sparkproof-triton-generate` now detects the GPU it validates on and, for any prompt that
+  declares a different `gpu_architecture` (e.g. a "Hopper SM90" prompt run on a Blackwell SM12x
+  miner), rewrites the SM label in the `system`/`prompt` text and updates `gpu_architecture` via
+  `build_prompts.reconcile_prompt_architecture`. This removes the task-label ↔ hardware mismatch
+  that made teachers add an SM guard which then failed validation (and induced the guard-removal
+  "repair"). No-op when the declared arch already matches or no GPU is detectable (dev/CPU).
 - **Prompts steer teachers to portable, non-hacky kernels (dataset-quality fix)**: analysis of the
   merged `sparkproof-mining` set showed repairs that "passed" by *deleting a hardware/SM
   architecture assertion* the model had added (e.g. a kernel prompted "for Hopper SM90" that then

sparkproof/cli/triton_generate.py

@@ -15,6 +15,7 @@
 from sparkproof.hashing import sha256_file
 from sparkproof.manifest import build_manifest
 from sparkproof.pipeline.blackwell import prove_blackwell_bundle
+from sparkproof.triton_dataset.build_prompts import reconcile_prompt_architecture
 from sparkproof.triton_dataset.decontaminate import TritonDecontaminator
 from sparkproof.triton_dataset.dpo_export import enrich_adjudication_with_responses, write_dpo_jsonl, export_dpo_jsonl
 from sparkproof.triton_dataset.failure_miner import mine_failure_to_tasks, record_failure
@@ -211,6 +212,19 @@ def main(argv: list[str] | None = None) -> int:
     filter_sources = parse_filter_set(args.filter_sources)
     filter_task_ids = parse_filter_set(args.filter_task_ids)
 
+    # Detect the GPU actually validating this run so prompts that declare a different
+    # architecture (e.g. "Hopper SM90" prompts run on Blackwell) are retargeted to the
+    # real hardware — otherwise the teacher is asked for the wrong arch and may add an
+    # SM guard that fails validation.
+    validating_architecture: str | None = None
+    try:
+        from sparkproof.gpu.architecture import require_supported_gpu
+
+        validating_architecture = require_supported_gpu(args.gpu)["gpu_architecture"]
+    except Exception as exc:  # noqa: BLE001 - dev/CPU or unsupported GPU: skip reconciliation
+        print(f"gpu-architecture reconciliation disabled ({exc})", file=sys.stderr)
+    arch_reconciled = 0
+
     for prompt_record in iter_prompts(
         args.prompts,
         args.limit,
@@ -225,6 +239,15 @@ def main(argv: list[str] | None = None) -> int:
             print(f"skip non-trainable task: {exc}", file=sys.stderr)
             continue
 
+        if validating_architecture and reconcile_prompt_architecture(prompt_record, validating_architecture):
+            arch_reconciled += 1
+            if arch_reconciled == 1:
+                print(
+                    f"retargeting prompts to the validating GPU architecture "
+                    f"{validating_architecture!r} (declared architecture differed)",
+                    file=sys.stderr,
+                )
+
         if args.orchestrate:
             try:
                 step = run_dataset_generation_step(
@@ -326,6 +349,13 @@ def main(argv: list[str] | None = None) -> int:
                 for mined in mine_failure_to_tasks(failure):
                     _append_jsonl(args.out / "mined_tasks.jsonl", mined)
 
+    if arch_reconciled:
+        print(
+            f"retargeted {arch_reconciled} prompt(s) to the validating GPU architecture "
+            f"{validating_architecture!r}",
+            file=sys.stderr,
+        )
+
     if evolved_tasks:
         evolved_path = args.out / "evolved_tasks.jsonl"
         evolved_path.parent.mkdir(parents=True, exist_ok=True)

sparkproof/triton_dataset/build_prompts.py

@@ -40,6 +40,29 @@ def default_system(gpu_architecture: str = ARCH_BLACKWELL) -> str:
 
 DEFAULT_SYSTEM = default_system(ARCH_BLACKWELL)
 
+
+def reconcile_prompt_architecture(record: dict[str, Any], target_architecture: str) -> bool:
+    """Retarget a prompt to the GPU architecture it is actually validated on.
+
+    Prompts bake a GPU/SM label at build time. When a miner validates on a different
+    GPU than the prompt declared (e.g. a "Hopper SM90" prompt run on Blackwell SM12x),
+    the teacher is asked for the wrong arch and may add an SM/capability guard that then
+    fails validation — inducing a repair that "passes" by deleting the guard. Rewrites the
+    SM label in the ``system`` and ``prompt`` text and updates ``gpu_architecture`` to the
+    real GPU. Returns True when the record was retargeted.
+    """
+    declared = record.get("gpu_architecture")
+    if not declared or declared == target_architecture:
+        return False
+    old_label, new_label = sm_label(declared), sm_label(target_architecture)
+    if old_label != new_label:
+        for field in ("system", "prompt"):
+            value = record.get(field)
+            if isinstance(value, str) and old_label in value:
+                record[field] = value.replace(old_label, new_label)
+    record["gpu_architecture"] = target_architecture
+    return True
+
 # TritonBench YAML is eval-only — never include "yaml" in training sources.
 
 DEFAULT_TRAIN_SOURCES = frozenset(

tests/test_arch_reconcile.py

@@ -0,0 +1,48 @@
+"""Retarget prompts to the actual validating GPU architecture."""
+
+from __future__ import annotations
+
+from sparkproof.triton_dataset.build_prompts import default_system, reconcile_prompt_architecture
+
+
+def _hopper_record() -> dict:
+    return {
+        "task_id": "t1",
+        "gpu_architecture": "hopper-h100",
+        "system": default_system("hopper-h100"),
+        "prompt": "Write a Triton 3.7.1 kernel on Hopper SM90 for layernorm.",
+    }
+
+
+def test_reconcile_retargets_hopper_prompt_to_blackwell():
+    rec = _hopper_record()
+    assert "Hopper SM90" in rec["system"] and "Hopper SM90" in rec["prompt"]
+
+    changed = reconcile_prompt_architecture(rec, "blackwell")
+
+    assert changed is True
+    assert rec["gpu_architecture"] == "blackwell"
+    assert "Hopper SM90" not in rec["system"]
+    assert "Blackwell SM12x" in rec["system"]
+    assert "Blackwell SM12x" in rec["prompt"]
+
+
+def test_reconcile_noop_when_arch_matches():
+    rec = _hopper_record()
+    before = dict(rec)
+    assert reconcile_prompt_architecture(rec, "hopper-h100") is False
+    assert rec == before
+
+
+def test_reconcile_noop_without_declared_arch():
+    rec = {"task_id": "t", "prompt": "x", "system": "y"}
+    assert reconcile_prompt_architecture(rec, "blackwell") is False
+
+
+def test_reconcile_same_label_updates_arch_field_only():
+    # hopper-h100 and hopper-h200 share the "Hopper SM90" label.
+    rec = _hopper_record()
+    system_before = rec["system"]
+    assert reconcile_prompt_architecture(rec, "hopper-h200") is True
+    assert rec["gpu_architecture"] == "hopper-h200"
+    assert rec["system"] == system_before  # label text unchanged