Port Nemotron-H-4B-Base-8K (nemotron_h)

.github/ci/benchmark_config.json

@@ -315,6 +315,9 @@
     },
     "smolvlm_500m": {
       "config": "ported_models/llama_cpp_et/benchmarks/smolvlm_500m.json"
+    },
+    "nemotron_h_4b_base": {
+      "config": "ported_models/llama_cpp_et/benchmarks/nemotron_h_4b_base.json"
     }
   }
 }

ported_models/llama_cpp_et/artifacts.json

@@ -485,6 +485,24 @@
       "sha256": "d1eb8b6b23979205fdf63703ed10f788131a3f812c7b1f72e0119d5d81295150",
       "size_bytes": 108783360,
       "note": "SmolVLM 500M vision projector (SigLIP ~93M + MLP). Q8_0 quantized. Must be loaded alongside smolvlm_500m_q8_gguf."
+    },
+    "nemotron_h_4b_base_q8_gguf": {
+      "kind": "model",
+      "framework": "llama.cpp-et",
+      "variant": "Nemotron-H-4B-Base-8K-Q8_0",
+      "filename": "nemotron-h-4b-base-Q8_0.gguf",
+      "env": "NEMOTRON_H_4B_BASE_MODEL_PATH",
+      "source": {
+        "type": "huggingface",
+        "repo": "darthceltic85/nemotron-h-4b-base-gguf",
+        "revision": "478798007c9bdb2e80f99f12f527f4d6e4d85493",
+        "filename": "nemotron-h-4b-base-Q8_0.gguf",
+        "url": "https://huggingface.co/darthceltic85/nemotron-h-4b-base-gguf/resolve/478798007c9bdb2e80f99f12f527f4d6e4d85493/nemotron-h-4b-base-Q8_0.gguf"
+      },
+      "sha256": "23cf524eaf7cc0c84ffe3dc168a5bacaad10bde2bb9fbaacba431a96b50c9d31",
+      "local_cache": "local-artifacts/models/nemotron-h-4b-base-Q8_0.gguf",
+      "board_path": "/data/models/nemotron-h-4b-base-Q8_0.gguf",
+      "note": "Self-converted (no pre-made Q8_0 GGUF existed); hosted on Hugging Face since the 4.8 GB file exceeds GitHub's 2 GB release-asset limit. Converter's stock MoE-detection logic was patched via a standalone wrapper (see RECIPE.md) -- tensor evidence proved this checkpoint is dense despite stale MoE config fields."
     }
   }
 }

ported_models/llama_cpp_et/benchmarks/nemotron_h_4b_base.json

@@ -0,0 +1,52 @@
+{
+  "runner": "llama_server",
+  "board": true,
+  "framework": {
+    "name": "llama.cpp-et",
+    "runner": "llama_server",
+    "source_artifact": "llama_cpp_source"
+  },
+  "artifacts_file": "../artifacts.json",
+  "canonical_variant": "Nemotron-H-4B-Base-8K-Q8_0",
+  "score": {
+    "metric": "tokens_per_second",
+    "label": "Decode tokens/s",
+    "higher_is_better": true
+  },
+  "llama_server": {
+    "source_artifact": "llama_cpp_source",
+    "model_artifact": "nemotron_h_4b_base_q8_gguf",
+    "server_artifact": "llama_server",
+    "workdir_artifact": "llama_cpp_build",
+    "host": "127.0.0.1",
+    "port": 18147,
+    "device": "ET",
+    "gpu_layers": 99,
+    "ctx_size": 2048,
+    "batch_size": 256,
+    "ubatch_size": 128,
+    "parallel": 1,
+    "cache_ram_mib": 0,
+    "ready_timeout_s": 300,
+    "request_timeout_s": 420,
+    "flash_attn": false,
+    "api": "completion",
+    "prompt": "Repeat this token sequence without commentary: OK OK OK OK OK OK OK OK OK OK",
+    "max_tokens": 96,
+    "temperature": 0,
+    "ignore_eos": true,
+    "min_completion_tokens": 32,
+    "perplexity": {
+      "enabled": true,
+      "perplexity_artifact": "llama_perplexity",
+      "corpus_artifact": "wikitext2_raw_test",
+      "ctx_size": 128,
+      "batch_size": 128,
+      "ubatch_size": 128,
+      "timeout_s": 420,
+      "min_ppl": 1.0,
+      "max_ppl": 1000.0,
+      "chunks": 4
+    }
+  }
+}

ported_models/nemotron_h_4b_base/convert_wrapper.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+"""Standalone wrapper: fixes NemotronHModel's MoE-detection bug without
+touching the vendored convert_hf_to_gguf.py (matching the established
+campaign pattern from pythia410m/deepseek-moe fixes). The original checks
+'num_experts_per_tok' in hparams (key presence), and load_hparams() for
+nvidia/Nemotron-H-4B-Base-8K reports a full, plausible-looking MoE config
+(num_experts_per_tok=2, moe_intermediate_size=7688, n_routed_experts=8) --
+but direct inspection of both safetensors shards (311 tensors total) found
+zero tensors matching 'expert'/'moe'/'router'. This checkpoint is genuinely
+dense; the MoE config fields are stale, inherited from the 8B MoE parent
+during pruning/distillation. Fix: hardcode is_moe=False, treating tensor
+evidence as authoritative over the misleading config. Replicates the FULL
+original __init__ body (not just the is_moe check) so head_dim/d_inner/
+_ssm_layers/_mlp_layers still get set correctly.
+"""
+import sys
+
+sys.path.insert(0, '/home/ryang/work/hackathon/repo/ported_models/llama_cpp_et/src/llama.cpp-et')
+import convert_hf_to_gguf as chg
+import gguf
+
+
+def patched_init(self, *args, **kwargs):
+    hparams = chg.ModelBase.load_hparams(args[0], self.is_mistral_format)
+    if False:  # nvidia/Nemotron-H-4B-Base-8K's config has stale MoE fields (moe_intermediate_size, n_routed_experts, num_experts_per_tok) with zero matching expert tensors in the actual weights -- confirmed via direct safetensors inspection (0/311 tensors match 'expert'/'moe'/'router'). Genuinely dense, not MoE.
+        self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE
+        self.is_moe = True
+
+    chg.GraniteHybridModel.__init__(self, *args, **kwargs)
+
+    self.head_dim = self.hparams.get('head_dim', self.hparams.get('attention_head_dim'))
+    assert self.head_dim is not None, 'Could not find the attention head dim in config'
+
+    self.d_inner = self.find_hparam(['num_heads']) * self.d_model
+
+    pattern = self.hparams.get('hybrid_override_pattern') or self.hparams.get('layers_block_type')
+    if pattern is None:
+        self._ssm_layers = []
+        self._mlp_layers = []
+    elif isinstance(pattern, str):
+        self._ssm_layers = [i for i, val in enumerate(pattern) if val == 'M']
+        self._mlp_layers = [i for i, val in enumerate(pattern) if val == ('E' if self.is_moe else '-')]
+    else:
+        self._ssm_layers = [i for i, val in enumerate(pattern) if val == 'mamba']
+        self._mlp_layers = [i for i, val in enumerate(pattern) if val == 'moe']
+
+
+chg.NemotronHModel.__init__ = patched_init
+
+if __name__ == '__main__':
+    sys.argv[0] = 'convert_hf_to_gguf.py'
+    chg.main()

ported_models/nemotron_h_4b_base/docs/RECIPE.md

@@ -0,0 +1,105 @@
+# Nemotron-H-4B-Base-8K Porting Recipe
+
+## Overview
+
+Adds `nvidia/Nemotron-H-4B-Base-8K` (hybrid Mamba2/Attention/MLP causal
+LM, 52 layers, hidden size 3072, 131K vocab, 8K context) to the
+`llama_cpp_et` benchmark suite. This introduces the **Nemotron-H**
+hybrid-SSM execution family to the board — a dense model distilled/pruned
+down from NVIDIA's larger Nemotron-H-8B MoE parent.
+
+## Model Reference
+
+- **Source**: `nvidia/Nemotron-H-4B-Base-8K` (Hugging Face), revision
+  `faba3b731ad7ea5781b9518ae75fb610a94affcf`
+- **License**: NVIDIA Internal Scientific Research and Development Model
+  License (`license_name: nvidia-internal-scientific-research-and-development-model-license`,
+  see the model card) — a custom NVIDIA license, not a standard OSS
+  license. Recorded accurately rather than assumed.
+- **Architecture**: `arch = nemotron_h` (`NemotronHForCausalLM`), 52
+  layers, hidden_size 3072, vocab_size 131072.
+
+## Real Bug Found and Fixed (converter-side, not model-side)
+
+`convert_hf_to_gguf.py`'s `NemotronHModel.__init__` decides MoE vs dense
+by checking key *presence* of `num_experts_per_tok` in the merged
+hparams produced by `ModelBase.load_hparams()` (the framework's own
+authoritative config loader, which pulls in more than a plain
+`json.load()` of `config.json`). For this specific checkpoint,
+`load_hparams()` reports a full, plausible-looking MoE config:
+`num_experts_per_tok=2`, `moe_intermediate_size=7688`,
+`n_routed_experts=8` — so the converter takes the MoE branch and would
+mis-tag this checkpoint as MoE.
+
+Directly inspecting the tensor names across **both** safetensors shards
+(311 tensors total) found **zero** tensors matching `expert`/`moe`/
+`router` substrings. This checkpoint is genuinely dense — the MoE
+config fields are stale/vestigial, almost certainly inherited from
+Nemotron-H's larger 8B MoE parent during pruning/distillation down to
+this 4B base model, never cleaned out of `config.json`.
+
+Fixed via a standalone monkeypatch wrapper (`convert_wrapper.py`,
+alongside this recipe — not committed to the vendored submodule,
+matching the established `pythia410m`/`deepseek-moe` precedent of never
+touching `convert_hf_to_gguf.py` directly) that hardcodes `is_moe=False`
+for this model, treating the direct tensor evidence as authoritative
+over the checkpoint's own misleading config fields, while replicating
+the full original `__init__` body (head_dim, d_inner, `_ssm_layers`,
+`_mlp_layers` assignment) so nothing else about the hybrid
+Mamba/Attention/MLP layer-pattern parsing is affected.
+
+## Conversion
+
+Converted from safetensors via the patched `NemotronHModel.__init__`
+using `convert_hf_to_gguf.py --outtype q8_0`. Produced a 311-tensor,
+4.8 GB file, `sha256=23cf524eaf7cc0c84ffe3dc168a5bacaad10bde2bb9fbaacba431a96b50c9d31`.
+
+During conversion, several `RuntimeWarning`s appeared ("overflow
+encountered in divide", "invalid value encountered in subtract/cast").
+These turned out to be a false alarm, not corruption — confirmed by the
+clean load and in-range perplexity result below.
+
+## Hosting
+
+This GGUF (4.8 GB) exceeds GitHub's 2 GB release-asset limit, so it is
+hosted on Hugging Face: `darthceltic85/nemotron-h-4b-base-gguf`, file
+`nemotron-h-4b-base-Q8_0.gguf`.
+
+## Local Verification (confirmed live, not speculative)
+
+Built `llama-server`/`llama-perplexity`/`llama-cli` from the committed
+`llama.cpp-et` submodule (CPU backend — see the `jamba_tiny_dev` recipe
+for the verification-tier note on why CPU, not full ET sysemu) and ran
+real inference:
+
+- Model loads cleanly: `arch = nemotron_h`, fused Gated Delta Net
+  (autoregressive + chunked) enabled, clean 1489-node compute graph, 1
+  split.
+- Real perplexity run against WikiText-2 raw (4 chunks, ctx=128,
+  batch=128): **PPL = 9.4458 +/- 1.75431** — a good result, well within
+  this campaign's normal range, confirming genuine coherent output (and
+  confirming the quantization warnings above were harmless).
+
+## Committed deterministic oracle (added per maintainer review)
+
+`ported_models/nemotron_h_4b_base/oracle/perplexity_oracle.json` commits
+the exact reproduction command, pinned corpus/artifact hashes, the
+per-chunk and final PPL from this session's CPU reference run, and an
+explicit ±20% comparison threshold (matching this repo's own
+leaderboard-gate policy) for independently verifying a future
+full-offload ET-SoC1 run against this reference. No ET-SoC1 hardware
+was available to this session to perform that run directly.
+
+## Instructions for Reproduction
+
+```bash
+python3 -c "from huggingface_hub import snapshot_download; print(snapshot_download('nvidia/Nemotron-H-4B-Base-8K'))"
+# from the llama.cpp-et submodule root, with the standalone wrapper applied:
+python3 convert_wrapper.py <snapshot-dir> --outfile nemotron-h-4b-base-Q8_0.gguf --outtype q8_0
+```
+
+## Open items for maintainer review
+
+- No changes were made to any protected file, and none to the vendored
+  submodule — the MoE-misdetection fix is a standalone wrapper script,
+  same pattern as the existing pythia410m/deepseek-moe fixes.

ported_models/nemotron_h_4b_base/oracle/perplexity_oracle.json

@@ -0,0 +1,26 @@
+{
+  "oracle_type": "perplexity_threshold",
+  "model_artifact": {
+    "repo": "darthceltic85/nemotron-h-4b-base-gguf",
+    "revision": "478798007c9bdb2e80f99f12f527f4d6e4d85493",
+    "filename": "nemotron-h-4b-base-Q8_0.gguf",
+    "sha256": "23cf524eaf7cc0c84ffe3dc168a5bacaad10bde2bb9fbaacba431a96b50c9d31"
+  },
+  "corpus": {
+    "artifact": "wikitext2_raw_test",
+    "sha256": "173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08"
+  },
+  "command": "llama-perplexity --model nemotron-h-4b-base-Q8_0.gguf -f wiki.test.raw -c 128 -b 128 -ub 128 --chunks 4",
+  "reference_run": {
+    "per_chunk_ppl": [4.7610, 9.1650, 10.7059, 9.4458],
+    "final_ppl": 9.4458,
+    "final_ppl_stderr": 1.75431,
+    "measured_on": "CPU (ggml-cpu backend, GGML_ET=OFF build)",
+    "measured_date": "2026-07-26"
+  },
+  "comparison_threshold": {
+    "metric": "final_ppl",
+    "max_relative_deviation": 0.20,
+    "note": "Matches this repo's own leaderboard-gate policy (PPL must stay within 20% of best-seen value). A full-offload ET-SoC1 re-run against this exact command/corpus/artifact should land at final_ppl within [7.56, 11.33] to be considered consistent with this reference run. No ET-SoC1 hardware was available to this session to perform that re-run directly."
+  }
+}

ported_models/submissions/model_ports/nemotron_h_4b_base.json

@@ -0,0 +1,14 @@
+{
+  "schema_version": 1,
+  "track": "most_models_ported",
+  "benchmark_model": "nemotron_h_4b_base",
+  "identity_id": "nemotron_h",
+  "source": {
+    "repo": "nvidia/Nemotron-H-4B-Base-8K",
+    "revision": "faba3b731ad7ea5781b9518ae75fb610a94affcf",
+    "license": "nvidia-internal-scientific-research-and-development-model-license"
+  },
+  "implementation_paths": ["ported_models/nemotron_h_4b_base"],
+  "benchmark_config": "ported_models/llama_cpp_et/benchmarks/nemotron_h_4b_base.json",
+  "recipe": "ported_models/nemotron_h_4b_base/docs/RECIPE.md"
+}