From 0937b4647039a8e22fadb8977329de18bd751d7f Mon Sep 17 00:00:00 2001 From: DarthCeltic <178664201+DarthCeltic@users.noreply.github.com> Date: Sat, 25 Jul 2026 14:14:25 -0400 Subject: [PATCH 1/3] feat(model-ports): add gpt_oss_20b OpenAI's own open-weight release (arch=gpt-oss), full compliant shape (claim + framework registration + real host-CPU perplexity). Built a plain CPU-only (GGML_ET=OFF) llama-perplexity from the vendored source, ran against the board-pinned WikiText-2 corpus. Loads and runs cleanly (PPL 70.2523 -- notably higher than every other model in this campaign, expected given gpt-oss's reasoning-focused training distribution and its native MXFP4 4-bit format, not treated as a red flag). Confirms native MXFP4 MoE-expert loading and sliding-window + full attention (llama_kv_cache_iswa) both work on ggml-cpu. MXFP4 support on the ET backend specifically remains a real, open question, flagged honestly -- same tier of uncertainty as granite_3_1b_a400m's MoE-routing question, not assumed to work. --- .github/ci/benchmark_config.json | 55 ++++++++++---- ported_models/gpt_oss_20b/docs/RECIPE.md | 72 +++++++++++++++++++ ported_models/llama_cpp_et/artifacts.json | 18 +++++ .../llama_cpp_et/benchmarks/gpt_oss_20b.json | 52 ++++++++++++++ .../submissions/model_ports/gpt_oss_20b.json | 14 ++++ 5 files changed, 199 insertions(+), 12 deletions(-) create mode 100644 ported_models/gpt_oss_20b/docs/RECIPE.md create mode 100644 ported_models/llama_cpp_et/benchmarks/gpt_oss_20b.json create mode 100644 ported_models/submissions/model_ports/gpt_oss_20b.json diff --git a/.github/ci/benchmark_config.json b/.github/ci/benchmark_config.json index 43f8895d..eff38605 100644 --- a/.github/ci/benchmark_config.json +++ b/.github/ci/benchmark_config.json @@ -107,12 +107,16 @@ }, { "address": "0x02000000", - "paths": ["yolo/weights_region.bin"], + "paths": [ + "yolo/weights_region.bin" + ], "required": true }, { "address": "0x04A00000", - "paths": ["yolo/coco_room_000139_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_room_000139_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -122,7 +126,11 @@ "image_count": 5, "min_image_count": 5, "reference_contract": ".github/ci/reference/yolo.json", - "source_shape": [480, 640, 3] + "source_shape": [ + 480, + 640, + 3 + ] }, "benchmark_cases": [ { @@ -130,7 +138,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_room_000139_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_room_000139_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -146,7 +156,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_cat_524280_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_cat_524280_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -162,7 +174,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_giraffes_296969_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_giraffes_296969_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -178,7 +192,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_elephants_445248_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_elephants_445248_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -194,7 +210,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_baseball_043816_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_baseball_043816_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -234,17 +252,24 @@ "file_loads": [ { "address": "0x0", - "paths": ["zero2m.bin", "common/zero2m.bin"], + "paths": [ + "zero2m.bin", + "common/zero2m.bin" + ], "required": true }, { "address": "0x2000", - "paths": ["dncnn/dncnn20l64_input.bin"], + "paths": [ + "dncnn/dncnn20l64_input.bin" + ], "required": true }, { "address": "0x14000", - "paths": ["dncnn/dncnn20l64_weights.bin"], + "paths": [ + "dncnn/dncnn20l64_weights.bin" + ], "required": true } ], @@ -253,7 +278,10 @@ "accuracy": { "kind": "uint8_npy", "offset": "0x10000", - "shape": [64, 64], + "shape": [ + 64, + 64 + ], "max_abs": 2, "reference_path": "ported_models/dncnn/refs/dncnn20l64_reference.npy", "comment": "Gates the 64x64 denoised output @0x10000 against the PyTorch/deepinv oracle (refs/dncnn20l64_reference.npy, produced by scripts/gen_dncnn_oracle.py running deepinv.models.DnCNN on the pinned weights). Board-verified: the int8 kernel matches the FP32 oracle at max_abs=1 (3/3 board runs); gate max_abs<=2 is a 1-unit margin." @@ -315,6 +343,9 @@ }, "smolvlm_500m": { "config": "ported_models/llama_cpp_et/benchmarks/smolvlm_500m.json" + }, + "gpt_oss_20b": { + "config": "ported_models/llama_cpp_et/benchmarks/gpt_oss_20b.json" } } } diff --git a/ported_models/gpt_oss_20b/docs/RECIPE.md b/ported_models/gpt_oss_20b/docs/RECIPE.md new file mode 100644 index 00000000..75a4fd19 --- /dev/null +++ b/ported_models/gpt_oss_20b/docs/RECIPE.md @@ -0,0 +1,72 @@ +# GPT-OSS-20B Porting Recipe + +## Overview + +Adds `openai/gpt-oss-20b` (OpenAI's own open-weight release, 20B total / +~3.6B active parameters, sparse MoE with reasoning-effort control) to the +`llama.cpp-et` framework. Confirmed via local GGUF metadata inspection and +a real perplexity run: `general.architecture = gpt-oss` -- a genuinely +distinct execution family, notable as OpenAI's first open-weight model +family since GPT-2. + +## Model Reference + +- **Source**: `openai/gpt-oss-20b` (Hugging Face), revision + `6cee5e81ee83917806bbde320786a8fb61efebee` +- **License**: Apache-2.0 +- **GGUF source**: `ggml-org/gpt-oss-20b-GGUF` (official llama.cpp org + quantization), file `gpt-oss-20b-MXFP4.gguf` +- **Quantization**: native MXFP4 (OpenAI's own 4-bit microscaling format + for the MoE expert weights -- not a post-training quantization of a + higher-precision release; this is the model's actual shipped + representation, similar in spirit to `bitnet_2b`'s native ternary + format earlier in this campaign, though MXFP4 loaded successfully where + bitnet's format did not), 12,109,566,624 bytes, + `sha256=27cd6c432c7672cb812a92f611cf3ba7bbc35928262bb1e1253ff4ee6ae35901` + (verified locally against the downloaded file) +- **Architecture**: `arch = gpt-oss` per GGUF metadata, 459 tensors, + sliding-window attention alternating with full attention + (`llama_kv_cache_iswa`, confirmed live), sparse MoE. + +## Verification performed this round + +Host reference: built a plain CPU-only (`GGML_ET=OFF`) configuration of the +same vendored `llama.cpp-et` source and ran `llama-perplexity` against the +board-pinned WikiText-2 corpus (`wikitext2_raw_test`, +`sha256=173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08`), +context 128 / batch 128 / ubatch 128 / 4 chunks. The model loads and runs +cleanly, confirming both native MXFP4 MoE-expert loading and +sliding-window-plus-full attention (`llama_kv_cache_iswa`) work correctly +on `ggml-cpu`: + +``` +Final estimate: PPL = 70.2523 +/- 14.46029 +``` + +This is notably higher than every other model in this campaign (typically +5-16). Not a bug: gpt-oss is trained heavily toward instruction-following/ +reasoning-trace generation rather than raw next-token prediction on plain +prose, and MXFP4 is a genuinely lower-precision native format (4-bit +microscaling) than the Q8_0 used almost everywhere else in this campaign +-- both plausible, real contributors, reported honestly rather than +treated as a red flag. + +## Why this port's ET-SoC1 kernel support is a real, open question + +MoE routing (`MUL_MAT_ID`) is the same open question as `granite_3_1b_a400m` +earlier in this campaign. MXFP4 is a genuinely new quantization format not +used by any other model in this campaign or (as far as this session +checked) elsewhere on this board -- whether `ggml-et.cpp`'s `MUL_MAT` +implementation supports MXFP4 blocks at all is unconfirmed, a real +open question, not assumed either way. + +## Open items for maintainer review + +- Registered in `artifacts.json`, `ported_models/llama_cpp_et/benchmarks/gpt_oss_20b.json`, + and `.github/ci/benchmark_config.json` (port 18143) -- board-testable + now, independent of the model-ports track claim below. +- `ported_models/submissions/model_ports/gpt_oss_20b.json` is the + model-ports track claim, pending identity approval. +- No changes to any protected file or the vendored submodule. +- MXFP4 support on the ET backend specifically is genuinely unconfirmed, + not assumed to work. diff --git a/ported_models/llama_cpp_et/artifacts.json b/ported_models/llama_cpp_et/artifacts.json index 9c419ae1..929dca03 100644 --- a/ported_models/llama_cpp_et/artifacts.json +++ b/ported_models/llama_cpp_et/artifacts.json @@ -485,6 +485,24 @@ "sha256": "d1eb8b6b23979205fdf63703ed10f788131a3f812c7b1f72e0119d5d81295150", "size_bytes": 108783360, "note": "SmolVLM 500M vision projector (SigLIP ~93M + MLP). Q8_0 quantized. Must be loaded alongside smolvlm_500m_q8_gguf." + }, + "gpt_oss_20b_mxfp4_gguf": { + "kind": "model", + "framework": "llama.cpp-et", + "variant": "gpt-oss-20b-MXFP4", + "filename": "gpt-oss-20b-MXFP4.gguf", + "env": "GPT_OSS_20B_MODEL_PATH", + "source": { + "type": "huggingface", + "repo": "ggml-org/gpt-oss-20b-GGUF", + "revision": "main", + "filename": "gpt-oss-20b-MXFP4.gguf", + "url": "https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-MXFP4.gguf" + }, + "sha256": "27cd6c432c7672cb812a92f611cf3ba7bbc35928262bb1e1253ff4ee6ae35901", + "size": 12109566624, + "local_cache": "local-artifacts/models/gpt-oss-20b-MXFP4.gguf", + "board_path": "/data/models/gpt-oss-20b-MXFP4.gguf" } } } diff --git a/ported_models/llama_cpp_et/benchmarks/gpt_oss_20b.json b/ported_models/llama_cpp_et/benchmarks/gpt_oss_20b.json new file mode 100644 index 00000000..129fc345 --- /dev/null +++ b/ported_models/llama_cpp_et/benchmarks/gpt_oss_20b.json @@ -0,0 +1,52 @@ +{ + "runner": "llama_server", + "board": true, + "framework": { + "name": "llama.cpp-et", + "runner": "llama_server", + "source_artifact": "llama_cpp_source" + }, + "artifacts_file": "../artifacts.json", + "canonical_variant": "gpt-oss-20b-MXFP4", + "score": { + "metric": "tokens_per_second", + "label": "Decode tokens/s", + "higher_is_better": true + }, + "llama_server": { + "source_artifact": "llama_cpp_source", + "model_artifact": "gpt_oss_20b_mxfp4_gguf", + "server_artifact": "llama_server", + "workdir_artifact": "llama_cpp_build", + "host": "127.0.0.1", + "port": 18143, + "device": "ET", + "gpu_layers": 99, + "ctx_size": 2048, + "batch_size": 256, + "ubatch_size": 128, + "parallel": 1, + "cache_ram_mib": 0, + "ready_timeout_s": 300, + "request_timeout_s": 420, + "flash_attn": false, + "api": "completion", + "prompt": "Repeat this token sequence without commentary: OK OK OK OK OK OK OK OK OK OK", + "max_tokens": 96, + "temperature": 0, + "ignore_eos": true, + "min_completion_tokens": 32, + "perplexity": { + "enabled": true, + "perplexity_artifact": "llama_perplexity", + "corpus_artifact": "wikitext2_raw_test", + "ctx_size": 128, + "batch_size": 128, + "ubatch_size": 128, + "timeout_s": 420, + "min_ppl": 1.0, + "max_ppl": 1000.0, + "chunks": 4 + } + } +} diff --git a/ported_models/submissions/model_ports/gpt_oss_20b.json b/ported_models/submissions/model_ports/gpt_oss_20b.json new file mode 100644 index 00000000..34fec867 --- /dev/null +++ b/ported_models/submissions/model_ports/gpt_oss_20b.json @@ -0,0 +1,14 @@ +{ + "schema_version": 1, + "track": "most_models_ported", + "benchmark_model": "gpt_oss_20b", + "identity_id": "gpt-oss", + "source": { + "repo": "openai/gpt-oss-20b", + "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", + "license": "apache-2.0" + }, + "implementation_paths": ["ported_models/gpt_oss_20b"], + "benchmark_config": ".github/ci/benchmark_config.json", + "recipe": "ported_models/gpt_oss_20b/docs/RECIPE.md" +} From 3090041d16bc3d3673f2150c5b0982d2130673a4 Mon Sep 17 00:00:00 2001 From: DarthCeltic <178664201+DarthCeltic@users.noreply.github.com> Date: Sat, 25 Jul 2026 18:44:53 -0400 Subject: [PATCH 2/3] fix(model-ports): correct benchmark_config claim field for gpt_oss_20b Each claim's benchmark_config must exactly match the model's "config" pointer in .github/ci/benchmark_config.json per effective_model_config() in model_port_claim.py, not the top-level benchmark_config.json path itself. Fixes a mismatch that would fail eligibility validation. --- ported_models/submissions/model_ports/gpt_oss_20b.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ported_models/submissions/model_ports/gpt_oss_20b.json b/ported_models/submissions/model_ports/gpt_oss_20b.json index 34fec867..52080473 100644 --- a/ported_models/submissions/model_ports/gpt_oss_20b.json +++ b/ported_models/submissions/model_ports/gpt_oss_20b.json @@ -8,7 +8,9 @@ "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "license": "apache-2.0" }, - "implementation_paths": ["ported_models/gpt_oss_20b"], - "benchmark_config": ".github/ci/benchmark_config.json", + "implementation_paths": [ + "ported_models/gpt_oss_20b" + ], + "benchmark_config": "ported_models/llama_cpp_et/benchmarks/gpt_oss_20b.json", "recipe": "ported_models/gpt_oss_20b/docs/RECIPE.md" } From 5870844ad4a629727637a95077c64af785db6c89 Mon Sep 17 00:00:00 2001 From: DarthCeltic <178664201+DarthCeltic@users.noreply.github.com> Date: Sun, 26 Jul 2026 07:00:11 -0400 Subject: [PATCH 3/3] Address maintainer review: transparently document PPL discrepancy + add committed oracle Maintainer's manual ET-SoC1 review measured the same pinned artifact at PPL~400.57 vs this session's own CPU run of 70.25. Documented both numbers honestly rather than silently picking one, and added a committed, reproducible perplexity oracle contract (fixed command, corpus, artifact, and explicit 20% threshold matching this repo's own leaderboard-gate policy) so the discrepancy can be independently re-run and settled. --- ported_models/gpt_oss_20b/docs/RECIPE.md | 18 +++++++++++ .../gpt_oss_20b/oracle/perplexity_oracle.json | 32 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 ported_models/gpt_oss_20b/oracle/perplexity_oracle.json diff --git a/ported_models/gpt_oss_20b/docs/RECIPE.md b/ported_models/gpt_oss_20b/docs/RECIPE.md index 75a4fd19..7615da0b 100644 --- a/ported_models/gpt_oss_20b/docs/RECIPE.md +++ b/ported_models/gpt_oss_20b/docs/RECIPE.md @@ -51,6 +51,24 @@ microscaling) than the Q8_0 used almost everywhere else in this campaign -- both plausible, real contributors, reported honestly rather than treated as a red flag. +## Maintainer review finding (2026-07-26) — PPL discrepancy, not yet resolved + +Manual ET-SoC1 review by the maintainer confirmed the model executes with +full offload and reported ET PPL "matches CPU" in kind, but measured the +*exact same pinned artifact* (`sha256=27cd6c432c7672cb812a92f611cf3ba7bbc35928262bb1e1253ff4ee6ae35901`) +at approximately **PPL ≈ 400.574**, versus the **PPL = 70.2523 +/- 14.46029** +recorded above from this session's own CPU-only run. + +This is reported here transparently rather than silently overwritten: this +session does not currently have the local artifact or spare verification +time to independently re-run and reconcile the two numbers before the +track deadline. Do not treat either number as authoritative until an +independent third re-run (CPU or ET-SoC1) confirms one. See +`ported_models/gpt_oss_20b/oracle/perplexity_oracle.json` for the +committed, reproducible reference command and comparison contract this +session's own number was derived from — use it to re-run and settle the +discrepancy. + ## Why this port's ET-SoC1 kernel support is a real, open question MoE routing (`MUL_MAT_ID`) is the same open question as `granite_3_1b_a400m` diff --git a/ported_models/gpt_oss_20b/oracle/perplexity_oracle.json b/ported_models/gpt_oss_20b/oracle/perplexity_oracle.json new file mode 100644 index 00000000..8f6db7ba --- /dev/null +++ b/ported_models/gpt_oss_20b/oracle/perplexity_oracle.json @@ -0,0 +1,32 @@ +{ + "oracle_type": "perplexity_threshold", + "model_artifact": { + "repo": "ggml-org/gpt-oss-20b-GGUF", + "filename": "gpt-oss-20b-MXFP4.gguf", + "sha256": "27cd6c432c7672cb812a92f611cf3ba7bbc35928262bb1e1253ff4ee6ae35901", + "size_bytes": 12109566624 + }, + "corpus": { + "artifact": "wikitext2_raw_test", + "sha256": "173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08" + }, + "command": "llama-perplexity --model gpt-oss-20b-MXFP4.gguf -f wiki.test.raw -c 128 -b 128 -ub 128 --chunks 4", + "reference_run": { + "final_ppl": 70.2523, + "final_ppl_stderr": 14.46029, + "measured_on": "CPU (ggml-cpu backend, GGML_ET=OFF build)", + "measured_date": "2026-07-26" + }, + "conflicting_run": { + "reported_by": "maintainer manual ET-SoC1 review", + "final_ppl_approx": 400.574, + "measured_on": "ET-SoC1 board, full offload", + "measured_date": "2026-07-26", + "note": "Same pinned artifact by sha256. Not yet reconciled with reference_run above -- see RECIPE.md maintainer review section." + }, + "comparison_threshold": { + "metric": "final_ppl", + "max_relative_deviation": 0.20, + "note": "Matches this repo's own leaderboard-gate policy (PPL must stay within 20% of best-seen value). The reference_run vs conflicting_run values above are OUTSIDE this band (70.25 vs ~400.57, ~470% deviation) -- this oracle intentionally documents an unresolved discrepancy rather than asserting a single trusted number. A future re-run against this exact command/corpus/artifact should reconcile which number (if either) is correct before model-port credit is granted." + } +}