diff --git a/CHANGELOG.md b/CHANGELOG.md index cbc8c59..7f5bb7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Changed +- **Prompts are retargeted to the actual validating GPU (root-cause dataset-quality fix)**: + `sparkproof-triton-generate` now detects the GPU it validates on and, for any prompt that + declares a different `gpu_architecture` (e.g. a "Hopper SM90" prompt run on a Blackwell SM12x + miner), rewrites the SM label in the `system`/`prompt` text and updates `gpu_architecture` via + `build_prompts.reconcile_prompt_architecture`. This removes the task-label ↔ hardware mismatch + that made teachers add an SM guard which then failed validation (and induced the guard-removal + "repair"). No-op when the declared arch already matches or no GPU is detectable (dev/CPU). - **Prompts steer teachers to portable, non-hacky kernels (dataset-quality fix)**: analysis of the merged `sparkproof-mining` set showed repairs that "passed" by *deleting a hardware/SM architecture assertion* the model had added (e.g. a kernel prompted "for Hopper SM90" that then diff --git a/sparkproof/cli/triton_generate.py b/sparkproof/cli/triton_generate.py index 028d0d2..da579ad 100644 --- a/sparkproof/cli/triton_generate.py +++ b/sparkproof/cli/triton_generate.py @@ -15,6 +15,7 @@ from sparkproof.hashing import sha256_file from sparkproof.manifest import build_manifest from sparkproof.pipeline.blackwell import prove_blackwell_bundle +from sparkproof.triton_dataset.build_prompts import reconcile_prompt_architecture from sparkproof.triton_dataset.decontaminate import TritonDecontaminator from sparkproof.triton_dataset.dpo_export import enrich_adjudication_with_responses, write_dpo_jsonl, export_dpo_jsonl from sparkproof.triton_dataset.failure_miner import mine_failure_to_tasks, record_failure @@ -211,6 +212,19 @@ def main(argv: list[str] | None = None) -> int: filter_sources = parse_filter_set(args.filter_sources) filter_task_ids = parse_filter_set(args.filter_task_ids) + # Detect the GPU actually validating this run so prompts that declare a different + # architecture (e.g. "Hopper SM90" prompts run on Blackwell) are retargeted to the + # real hardware — otherwise the teacher is asked for the wrong arch and may add an + # SM guard that fails validation. + validating_architecture: str | None = None + try: + from sparkproof.gpu.architecture import require_supported_gpu + + validating_architecture = require_supported_gpu(args.gpu)["gpu_architecture"] + except Exception as exc: # noqa: BLE001 - dev/CPU or unsupported GPU: skip reconciliation + print(f"gpu-architecture reconciliation disabled ({exc})", file=sys.stderr) + arch_reconciled = 0 + for prompt_record in iter_prompts( args.prompts, args.limit, @@ -225,6 +239,15 @@ def main(argv: list[str] | None = None) -> int: print(f"skip non-trainable task: {exc}", file=sys.stderr) continue + if validating_architecture and reconcile_prompt_architecture(prompt_record, validating_architecture): + arch_reconciled += 1 + if arch_reconciled == 1: + print( + f"retargeting prompts to the validating GPU architecture " + f"{validating_architecture!r} (declared architecture differed)", + file=sys.stderr, + ) + if args.orchestrate: try: step = run_dataset_generation_step( @@ -326,6 +349,13 @@ def main(argv: list[str] | None = None) -> int: for mined in mine_failure_to_tasks(failure): _append_jsonl(args.out / "mined_tasks.jsonl", mined) + if arch_reconciled: + print( + f"retargeted {arch_reconciled} prompt(s) to the validating GPU architecture " + f"{validating_architecture!r}", + file=sys.stderr, + ) + if evolved_tasks: evolved_path = args.out / "evolved_tasks.jsonl" evolved_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/sparkproof/triton_dataset/build_prompts.py b/sparkproof/triton_dataset/build_prompts.py index 5c28d5f..68546e2 100644 --- a/sparkproof/triton_dataset/build_prompts.py +++ b/sparkproof/triton_dataset/build_prompts.py @@ -40,6 +40,29 @@ def default_system(gpu_architecture: str = ARCH_BLACKWELL) -> str: DEFAULT_SYSTEM = default_system(ARCH_BLACKWELL) + +def reconcile_prompt_architecture(record: dict[str, Any], target_architecture: str) -> bool: + """Retarget a prompt to the GPU architecture it is actually validated on. + + Prompts bake a GPU/SM label at build time. When a miner validates on a different + GPU than the prompt declared (e.g. a "Hopper SM90" prompt run on Blackwell SM12x), + the teacher is asked for the wrong arch and may add an SM/capability guard that then + fails validation — inducing a repair that "passes" by deleting the guard. Rewrites the + SM label in the ``system`` and ``prompt`` text and updates ``gpu_architecture`` to the + real GPU. Returns True when the record was retargeted. + """ + declared = record.get("gpu_architecture") + if not declared or declared == target_architecture: + return False + old_label, new_label = sm_label(declared), sm_label(target_architecture) + if old_label != new_label: + for field in ("system", "prompt"): + value = record.get(field) + if isinstance(value, str) and old_label in value: + record[field] = value.replace(old_label, new_label) + record["gpu_architecture"] = target_architecture + return True + # TritonBench YAML is eval-only — never include "yaml" in training sources. DEFAULT_TRAIN_SOURCES = frozenset( diff --git a/tests/test_arch_reconcile.py b/tests/test_arch_reconcile.py new file mode 100644 index 0000000..a92c45e --- /dev/null +++ b/tests/test_arch_reconcile.py @@ -0,0 +1,48 @@ +"""Retarget prompts to the actual validating GPU architecture.""" + +from __future__ import annotations + +from sparkproof.triton_dataset.build_prompts import default_system, reconcile_prompt_architecture + + +def _hopper_record() -> dict: + return { + "task_id": "t1", + "gpu_architecture": "hopper-h100", + "system": default_system("hopper-h100"), + "prompt": "Write a Triton 3.7.1 kernel on Hopper SM90 for layernorm.", + } + + +def test_reconcile_retargets_hopper_prompt_to_blackwell(): + rec = _hopper_record() + assert "Hopper SM90" in rec["system"] and "Hopper SM90" in rec["prompt"] + + changed = reconcile_prompt_architecture(rec, "blackwell") + + assert changed is True + assert rec["gpu_architecture"] == "blackwell" + assert "Hopper SM90" not in rec["system"] + assert "Blackwell SM12x" in rec["system"] + assert "Blackwell SM12x" in rec["prompt"] + + +def test_reconcile_noop_when_arch_matches(): + rec = _hopper_record() + before = dict(rec) + assert reconcile_prompt_architecture(rec, "hopper-h100") is False + assert rec == before + + +def test_reconcile_noop_without_declared_arch(): + rec = {"task_id": "t", "prompt": "x", "system": "y"} + assert reconcile_prompt_architecture(rec, "blackwell") is False + + +def test_reconcile_same_label_updates_arch_field_only(): + # hopper-h100 and hopper-h200 share the "Hopper SM90" label. + rec = _hopper_record() + system_before = rec["system"] + assert reconcile_prompt_architecture(rec, "hopper-h200") is True + assert rec["gpu_architecture"] == "hopper-h200" + assert rec["system"] == system_before # label text unchanged