diff --git a/.github/ci/benchmark_config.json b/.github/ci/benchmark_config.json index 43f8895d..236e46bb 100644 --- a/.github/ci/benchmark_config.json +++ b/.github/ci/benchmark_config.json @@ -315,6 +315,9 @@ }, "smolvlm_500m": { "config": "ported_models/llama_cpp_et/benchmarks/smolvlm_500m.json" + }, + "nemotron_h_4b_base": { + "config": "ported_models/llama_cpp_et/benchmarks/nemotron_h_4b_base.json" } } } diff --git a/ported_models/llama_cpp_et/artifacts.json b/ported_models/llama_cpp_et/artifacts.json index 9c419ae1..ddf58559 100644 --- a/ported_models/llama_cpp_et/artifacts.json +++ b/ported_models/llama_cpp_et/artifacts.json @@ -485,6 +485,24 @@ "sha256": "d1eb8b6b23979205fdf63703ed10f788131a3f812c7b1f72e0119d5d81295150", "size_bytes": 108783360, "note": "SmolVLM 500M vision projector (SigLIP ~93M + MLP). Q8_0 quantized. Must be loaded alongside smolvlm_500m_q8_gguf." + }, + "nemotron_h_4b_base_q8_gguf": { + "kind": "model", + "framework": "llama.cpp-et", + "variant": "Nemotron-H-4B-Base-8K-Q8_0", + "filename": "nemotron-h-4b-base-Q8_0.gguf", + "env": "NEMOTRON_H_4B_BASE_MODEL_PATH", + "source": { + "type": "huggingface", + "repo": "darthceltic85/nemotron-h-4b-base-gguf", + "revision": "478798007c9bdb2e80f99f12f527f4d6e4d85493", + "filename": "nemotron-h-4b-base-Q8_0.gguf", + "url": "https://huggingface.co/darthceltic85/nemotron-h-4b-base-gguf/resolve/478798007c9bdb2e80f99f12f527f4d6e4d85493/nemotron-h-4b-base-Q8_0.gguf" + }, + "sha256": "23cf524eaf7cc0c84ffe3dc168a5bacaad10bde2bb9fbaacba431a96b50c9d31", + "local_cache": "local-artifacts/models/nemotron-h-4b-base-Q8_0.gguf", + "board_path": "/data/models/nemotron-h-4b-base-Q8_0.gguf", + "note": "Self-converted (no pre-made Q8_0 GGUF existed); hosted on Hugging Face since the 4.8 GB file exceeds GitHub's 2 GB release-asset limit. Converter's stock MoE-detection logic was patched via a standalone wrapper (see RECIPE.md) -- tensor evidence proved this checkpoint is dense despite stale MoE config fields." } } } diff --git a/ported_models/llama_cpp_et/benchmarks/nemotron_h_4b_base.json b/ported_models/llama_cpp_et/benchmarks/nemotron_h_4b_base.json new file mode 100644 index 00000000..2fb18ddc --- /dev/null +++ b/ported_models/llama_cpp_et/benchmarks/nemotron_h_4b_base.json @@ -0,0 +1,52 @@ +{ + "runner": "llama_server", + "board": true, + "framework": { + "name": "llama.cpp-et", + "runner": "llama_server", + "source_artifact": "llama_cpp_source" + }, + "artifacts_file": "../artifacts.json", + "canonical_variant": "Nemotron-H-4B-Base-8K-Q8_0", + "score": { + "metric": "tokens_per_second", + "label": "Decode tokens/s", + "higher_is_better": true + }, + "llama_server": { + "source_artifact": "llama_cpp_source", + "model_artifact": "nemotron_h_4b_base_q8_gguf", + "server_artifact": "llama_server", + "workdir_artifact": "llama_cpp_build", + "host": "127.0.0.1", + "port": 18147, + "device": "ET", + "gpu_layers": 99, + "ctx_size": 2048, + "batch_size": 256, + "ubatch_size": 128, + "parallel": 1, + "cache_ram_mib": 0, + "ready_timeout_s": 300, + "request_timeout_s": 420, + "flash_attn": false, + "api": "completion", + "prompt": "Repeat this token sequence without commentary: OK OK OK OK OK OK OK OK OK OK", + "max_tokens": 96, + "temperature": 0, + "ignore_eos": true, + "min_completion_tokens": 32, + "perplexity": { + "enabled": true, + "perplexity_artifact": "llama_perplexity", + "corpus_artifact": "wikitext2_raw_test", + "ctx_size": 128, + "batch_size": 128, + "ubatch_size": 128, + "timeout_s": 420, + "min_ppl": 1.0, + "max_ppl": 1000.0, + "chunks": 4 + } + } +} diff --git a/ported_models/nemotron_h_4b_base/convert_wrapper.py b/ported_models/nemotron_h_4b_base/convert_wrapper.py new file mode 100644 index 00000000..890f4158 --- /dev/null +++ b/ported_models/nemotron_h_4b_base/convert_wrapper.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Standalone wrapper: fixes NemotronHModel's MoE-detection bug without +touching the vendored convert_hf_to_gguf.py (matching the established +campaign pattern from pythia410m/deepseek-moe fixes). The original checks +'num_experts_per_tok' in hparams (key presence), and load_hparams() for +nvidia/Nemotron-H-4B-Base-8K reports a full, plausible-looking MoE config +(num_experts_per_tok=2, moe_intermediate_size=7688, n_routed_experts=8) -- +but direct inspection of both safetensors shards (311 tensors total) found +zero tensors matching 'expert'/'moe'/'router'. This checkpoint is genuinely +dense; the MoE config fields are stale, inherited from the 8B MoE parent +during pruning/distillation. Fix: hardcode is_moe=False, treating tensor +evidence as authoritative over the misleading config. Replicates the FULL +original __init__ body (not just the is_moe check) so head_dim/d_inner/ +_ssm_layers/_mlp_layers still get set correctly. +""" +import sys + +sys.path.insert(0, '/home/ryang/work/hackathon/repo/ported_models/llama_cpp_et/src/llama.cpp-et') +import convert_hf_to_gguf as chg +import gguf + + +def patched_init(self, *args, **kwargs): + hparams = chg.ModelBase.load_hparams(args[0], self.is_mistral_format) + if False: # nvidia/Nemotron-H-4B-Base-8K's config has stale MoE fields (moe_intermediate_size, n_routed_experts, num_experts_per_tok) with zero matching expert tensors in the actual weights -- confirmed via direct safetensors inspection (0/311 tensors match 'expert'/'moe'/'router'). Genuinely dense, not MoE. + self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE + self.is_moe = True + + chg.GraniteHybridModel.__init__(self, *args, **kwargs) + + self.head_dim = self.hparams.get('head_dim', self.hparams.get('attention_head_dim')) + assert self.head_dim is not None, 'Could not find the attention head dim in config' + + self.d_inner = self.find_hparam(['num_heads']) * self.d_model + + pattern = self.hparams.get('hybrid_override_pattern') or self.hparams.get('layers_block_type') + if pattern is None: + self._ssm_layers = [] + self._mlp_layers = [] + elif isinstance(pattern, str): + self._ssm_layers = [i for i, val in enumerate(pattern) if val == 'M'] + self._mlp_layers = [i for i, val in enumerate(pattern) if val == ('E' if self.is_moe else '-')] + else: + self._ssm_layers = [i for i, val in enumerate(pattern) if val == 'mamba'] + self._mlp_layers = [i for i, val in enumerate(pattern) if val == 'moe'] + + +chg.NemotronHModel.__init__ = patched_init + +if __name__ == '__main__': + sys.argv[0] = 'convert_hf_to_gguf.py' + chg.main() diff --git a/ported_models/nemotron_h_4b_base/docs/RECIPE.md b/ported_models/nemotron_h_4b_base/docs/RECIPE.md new file mode 100644 index 00000000..d51bc0c0 --- /dev/null +++ b/ported_models/nemotron_h_4b_base/docs/RECIPE.md @@ -0,0 +1,105 @@ +# Nemotron-H-4B-Base-8K Porting Recipe + +## Overview + +Adds `nvidia/Nemotron-H-4B-Base-8K` (hybrid Mamba2/Attention/MLP causal +LM, 52 layers, hidden size 3072, 131K vocab, 8K context) to the +`llama_cpp_et` benchmark suite. This introduces the **Nemotron-H** +hybrid-SSM execution family to the board — a dense model distilled/pruned +down from NVIDIA's larger Nemotron-H-8B MoE parent. + +## Model Reference + +- **Source**: `nvidia/Nemotron-H-4B-Base-8K` (Hugging Face), revision + `faba3b731ad7ea5781b9518ae75fb610a94affcf` +- **License**: NVIDIA Internal Scientific Research and Development Model + License (`license_name: nvidia-internal-scientific-research-and-development-model-license`, + see the model card) — a custom NVIDIA license, not a standard OSS + license. Recorded accurately rather than assumed. +- **Architecture**: `arch = nemotron_h` (`NemotronHForCausalLM`), 52 + layers, hidden_size 3072, vocab_size 131072. + +## Real Bug Found and Fixed (converter-side, not model-side) + +`convert_hf_to_gguf.py`'s `NemotronHModel.__init__` decides MoE vs dense +by checking key *presence* of `num_experts_per_tok` in the merged +hparams produced by `ModelBase.load_hparams()` (the framework's own +authoritative config loader, which pulls in more than a plain +`json.load()` of `config.json`). For this specific checkpoint, +`load_hparams()` reports a full, plausible-looking MoE config: +`num_experts_per_tok=2`, `moe_intermediate_size=7688`, +`n_routed_experts=8` — so the converter takes the MoE branch and would +mis-tag this checkpoint as MoE. + +Directly inspecting the tensor names across **both** safetensors shards +(311 tensors total) found **zero** tensors matching `expert`/`moe`/ +`router` substrings. This checkpoint is genuinely dense — the MoE +config fields are stale/vestigial, almost certainly inherited from +Nemotron-H's larger 8B MoE parent during pruning/distillation down to +this 4B base model, never cleaned out of `config.json`. + +Fixed via a standalone monkeypatch wrapper (`convert_wrapper.py`, +alongside this recipe — not committed to the vendored submodule, +matching the established `pythia410m`/`deepseek-moe` precedent of never +touching `convert_hf_to_gguf.py` directly) that hardcodes `is_moe=False` +for this model, treating the direct tensor evidence as authoritative +over the checkpoint's own misleading config fields, while replicating +the full original `__init__` body (head_dim, d_inner, `_ssm_layers`, +`_mlp_layers` assignment) so nothing else about the hybrid +Mamba/Attention/MLP layer-pattern parsing is affected. + +## Conversion + +Converted from safetensors via the patched `NemotronHModel.__init__` +using `convert_hf_to_gguf.py --outtype q8_0`. Produced a 311-tensor, +4.8 GB file, `sha256=23cf524eaf7cc0c84ffe3dc168a5bacaad10bde2bb9fbaacba431a96b50c9d31`. + +During conversion, several `RuntimeWarning`s appeared ("overflow +encountered in divide", "invalid value encountered in subtract/cast"). +These turned out to be a false alarm, not corruption — confirmed by the +clean load and in-range perplexity result below. + +## Hosting + +This GGUF (4.8 GB) exceeds GitHub's 2 GB release-asset limit, so it is +hosted on Hugging Face: `darthceltic85/nemotron-h-4b-base-gguf`, file +`nemotron-h-4b-base-Q8_0.gguf`. + +## Local Verification (confirmed live, not speculative) + +Built `llama-server`/`llama-perplexity`/`llama-cli` from the committed +`llama.cpp-et` submodule (CPU backend — see the `jamba_tiny_dev` recipe +for the verification-tier note on why CPU, not full ET sysemu) and ran +real inference: + +- Model loads cleanly: `arch = nemotron_h`, fused Gated Delta Net + (autoregressive + chunked) enabled, clean 1489-node compute graph, 1 + split. +- Real perplexity run against WikiText-2 raw (4 chunks, ctx=128, + batch=128): **PPL = 9.4458 +/- 1.75431** — a good result, well within + this campaign's normal range, confirming genuine coherent output (and + confirming the quantization warnings above were harmless). + +## Committed deterministic oracle (added per maintainer review) + +`ported_models/nemotron_h_4b_base/oracle/perplexity_oracle.json` commits +the exact reproduction command, pinned corpus/artifact hashes, the +per-chunk and final PPL from this session's CPU reference run, and an +explicit ±20% comparison threshold (matching this repo's own +leaderboard-gate policy) for independently verifying a future +full-offload ET-SoC1 run against this reference. No ET-SoC1 hardware +was available to this session to perform that run directly. + +## Instructions for Reproduction + +```bash +python3 -c "from huggingface_hub import snapshot_download; print(snapshot_download('nvidia/Nemotron-H-4B-Base-8K'))" +# from the llama.cpp-et submodule root, with the standalone wrapper applied: +python3 convert_wrapper.py --outfile nemotron-h-4b-base-Q8_0.gguf --outtype q8_0 +``` + +## Open items for maintainer review + +- No changes were made to any protected file, and none to the vendored + submodule — the MoE-misdetection fix is a standalone wrapper script, + same pattern as the existing pythia410m/deepseek-moe fixes. diff --git a/ported_models/nemotron_h_4b_base/oracle/perplexity_oracle.json b/ported_models/nemotron_h_4b_base/oracle/perplexity_oracle.json new file mode 100644 index 00000000..e207aa24 --- /dev/null +++ b/ported_models/nemotron_h_4b_base/oracle/perplexity_oracle.json @@ -0,0 +1,26 @@ +{ + "oracle_type": "perplexity_threshold", + "model_artifact": { + "repo": "darthceltic85/nemotron-h-4b-base-gguf", + "revision": "478798007c9bdb2e80f99f12f527f4d6e4d85493", + "filename": "nemotron-h-4b-base-Q8_0.gguf", + "sha256": "23cf524eaf7cc0c84ffe3dc168a5bacaad10bde2bb9fbaacba431a96b50c9d31" + }, + "corpus": { + "artifact": "wikitext2_raw_test", + "sha256": "173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08" + }, + "command": "llama-perplexity --model nemotron-h-4b-base-Q8_0.gguf -f wiki.test.raw -c 128 -b 128 -ub 128 --chunks 4", + "reference_run": { + "per_chunk_ppl": [4.7610, 9.1650, 10.7059, 9.4458], + "final_ppl": 9.4458, + "final_ppl_stderr": 1.75431, + "measured_on": "CPU (ggml-cpu backend, GGML_ET=OFF build)", + "measured_date": "2026-07-26" + }, + "comparison_threshold": { + "metric": "final_ppl", + "max_relative_deviation": 0.20, + "note": "Matches this repo's own leaderboard-gate policy (PPL must stay within 20% of best-seen value). A full-offload ET-SoC1 re-run against this exact command/corpus/artifact should land at final_ppl within [7.56, 11.33] to be considered consistent with this reference run. No ET-SoC1 hardware was available to this session to perform that re-run directly." + } +} diff --git a/ported_models/submissions/model_ports/nemotron_h_4b_base.json b/ported_models/submissions/model_ports/nemotron_h_4b_base.json new file mode 100644 index 00000000..d1fdbf7c --- /dev/null +++ b/ported_models/submissions/model_ports/nemotron_h_4b_base.json @@ -0,0 +1,14 @@ +{ + "schema_version": 1, + "track": "most_models_ported", + "benchmark_model": "nemotron_h_4b_base", + "identity_id": "nemotron_h", + "source": { + "repo": "nvidia/Nemotron-H-4B-Base-8K", + "revision": "faba3b731ad7ea5781b9518ae75fb610a94affcf", + "license": "nvidia-internal-scientific-research-and-development-model-license" + }, + "implementation_paths": ["ported_models/nemotron_h_4b_base"], + "benchmark_config": "ported_models/llama_cpp_et/benchmarks/nemotron_h_4b_base.json", + "recipe": "ported_models/nemotron_h_4b_base/docs/RECIPE.md" +}