diff --git a/.github/ci/benchmark_config.json b/.github/ci/benchmark_config.json index 43f8895d..c7eb7600 100644 --- a/.github/ci/benchmark_config.json +++ b/.github/ci/benchmark_config.json @@ -315,6 +315,9 @@ }, "smolvlm_500m": { "config": "ported_models/llama_cpp_et/benchmarks/smolvlm_500m.json" + }, + "deepseek_moe_16b_base": { + "config": "ported_models/llama_cpp_et/benchmarks/deepseek_moe_16b_base.json" } } } diff --git a/ported_models/deepseek_moe_16b_base/convert_wrapper.py b/ported_models/deepseek_moe_16b_base/convert_wrapper.py new file mode 100644 index 00000000..95d293d5 --- /dev/null +++ b/ported_models/deepseek_moe_16b_base/convert_wrapper.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Standalone wrapper: registers deepseek-moe-16b's tokenizer chkhsh at +runtime without touching the vendored convert_hf_to_gguf.py (matching the +established pattern from this campaign's pythia410m rope_parameters fix). +Extracts the real chktxt string via source introspection to guarantee +byte-identical hashing versus the original function -- no manual retyping. +""" +import ast +import inspect +import sys +from hashlib import sha256 + +sys.path.insert(0, '/home/ryang/work/hackathon/repo/ported_models/llama_cpp_et/src/llama.cpp-et') +import convert_hf_to_gguf as chg + +_orig_get_vocab_base_pre = chg.TextModel.get_vocab_base_pre +_KNOWN_HASH = '93105512fde79bc726022fe3cbfb7efef9738465b988d0600beba1296e3a91d8' + +# Extract the exact chktxt string literal from the real function's source. +_src = __import__('textwrap').dedent(inspect.getsource(_orig_get_vocab_base_pre)) +_tree = ast.parse(_src) +_chktxt = None +for node in ast.walk(_tree): + if isinstance(node, ast.Assign) and any(getattr(t, 'id', None) == 'chktxt' for t in node.targets): + _chktxt = ast.literal_eval(node.value) + break +assert _chktxt is not None, 'could not extract chktxt from original function source' + + +def patched_get_vocab_base_pre(self, tokenizer): + chktok = tokenizer.encode(_chktxt) + chkhsh = sha256(str(chktok).encode()).hexdigest() + if chkhsh == _KNOWN_HASH: + # NOTE: not 'deepseek-moe' -- that name isn't recognized by the + # separate hardcoded pre-tokenizer registry in llama-vocab.cpp + # (C++ runtime), which only knows 'deepseek-llm'/'deepseek-coder'/ + # 'deepseek-v3'/'deepseek-r1-qwen'. DeepSeek's early dense/MoE + # models share the same base BPE tokenizer family, so 'deepseek-llm' + # is correct and is recognized by both the Python converter and the + # C++ runtime. + return 'deepseek-llm' + return _orig_get_vocab_base_pre(self, tokenizer) + + +chg.TextModel.get_vocab_base_pre = patched_get_vocab_base_pre + +if __name__ == '__main__': + sys.argv[0] = 'convert_hf_to_gguf.py' + chg.main() diff --git a/ported_models/deepseek_moe_16b_base/docs/RECIPE.md b/ported_models/deepseek_moe_16b_base/docs/RECIPE.md new file mode 100644 index 00000000..85d5cb8b --- /dev/null +++ b/ported_models/deepseek_moe_16b_base/docs/RECIPE.md @@ -0,0 +1,122 @@ +# DeepSeek-MoE-16B-Base Porting Recipe + +## Overview + +Adds `deepseek-ai/deepseek-moe-16b-base` (fine-grained Mixture-of-Experts +causal LM, 28 layers, 64 routed experts + 2 shared experts, 6 active per +token, 16.38B total params) to the `llama_cpp_et` benchmark suite. This +introduces the **DeepSeek MoE** execution family to the board — the +architecture that predates and informed the later DeepSeek-V2/V3 MoE +designs. + +## Model Reference + +- **Source**: `deepseek-ai/deepseek-moe-16b-base` (Hugging Face), + revision `521d2bc4fb69a3f3ae565310fcc3b65f97af2580` +- **License**: DeepSeek's custom Model License + (`license_name: deepseek`, `license_link: + https://github.com/deepseek-ai/DeepSeek-MoE/blob/main/LICENSE-MODEL`) + — recorded accurately rather than assumed to be MIT/Apache. +- **Architecture**: `arch = deepseek` (`DeepseekForCausalLM`), 28 layers, + 64 routed experts (6 active), 2 shared experts, embedding length 2048. + +## Real Bugs Found and Fixed (converter-side, not model-side) + +Two distinct, layered bugs in `convert_hf_to_gguf.py`'s tokenizer +handling, both hit while converting this checkpoint: + +**Bug 1 — unrecognized tokenizer chkhsh.** The converter's +`TextModel.get_vocab_base_pre()` identifies a model's BPE +pre-tokenizer family by hashing a fixed check string encoded through +the model's own tokenizer, then matching against a table of known +hashes. `deepseek-moe-16b-base`'s tokenizer produces a chkhsh +(`93105512fde79bc726022fe3cbfb7efef9738465b988d0600beba1296e3a91d8`) +that isn't in that table at all, so conversion fails outright with an +unrecognized-tokenizer error. + +**Bug 2 — the fix's own first attempt was incomplete.** The first fix +registered a new, made-up pre-tokenizer name (`'deepseek-moe'`) for +that hash. This let the Python-side conversion complete successfully, +producing a seemingly-valid GGUF — but the **separate, independent** +hardcoded pre-tokenizer registry in the C++ runtime (`llama-vocab.cpp`) +only recognizes `'deepseek-llm'` / `'deepseek-coder'` / `'deepseek-v3'` +/ `'deepseek-r1-qwen'`, not `'deepseek-moe'`. A GGUF built with the +invented name loads and *appears* fine in the Python tooling but is +silently wrong at inference time in the real runtime. Caught this by +actually loading the produced GGUF with `llama-perplexity`, not just by +trusting a clean Python conversion. + +**Final fix**: register the known hash to the existing name +`'deepseek-llm'` instead of inventing a new one, since DeepSeek's early +dense (`deepseek-llm`) and MoE (`deepseek-moe`) models share the same +base BPE tokenizer family. Confirmed live: the final GGUF loads with +`tokenizer.ggml.pre = deepseek-llm` and produces coherent, in-range +perplexity (see below). + +Both fixes applied via a standalone monkeypatch wrapper +(`convert_wrapper.py`, alongside this recipe), which extracts the +original function's exact `chktxt` check-string via source +introspection (`ast`/`inspect`) rather than retyping it by hand, to +guarantee byte-identical hashing versus the real function. The vendored +`convert_hf_to_gguf.py` itself is never touched, matching the +established `pythia410m` precedent. + +## Conversion + +Converted from safetensors via the patched `get_vocab_base_pre()` using +`convert_hf_to_gguf.py --outtype q8_0`. Produced a 363-tensor, 17.4 GB +file, `sha256=63eb27478a35ec36a3fd7c50e704ef6752ca235a59491d7673909da6fd8f0971`. +Required `trust_remote_code` prompt handling for this repo's custom +modeling code (config-loading only; standard for this model family). + +## Hosting + +This GGUF (17.4 GB) far exceeds GitHub's 2 GB release-asset limit, so +it is hosted on Hugging Face: `darthceltic85/deepseek-moe-16b-base-gguf`, +file `deepseek-moe-16b-base-Q8_0.gguf`. + +## Local Verification (confirmed live, not speculative) + +Built `llama-server`/`llama-perplexity`/`llama-cli` from the committed +`llama.cpp-et` submodule (CPU backend — see the `jamba_tiny_dev` recipe +for the verification-tier note on why CPU, not full ET sysemu) and ran +real inference: + +- Model loads cleanly: `arch = deepseek`, `tokenizer.ggml.pre = + deepseek-llm` (confirming the C++ runtime correctly recognizes the + corrected pre-tokenizer name), 64 experts / 6 used, clean compute + graph, 1 split. +- Real perplexity run against WikiText-2 raw (4 chunks, ctx=128, + batch=128): **PPL = 7.2909 +/- 1.30187** — a good result, solidly + within this campaign's normal range, confirming genuine coherent + output from a correctly-tokenized model. + +## Committed deterministic oracle (added per maintainer review) + +`ported_models/deepseek_moe_16b_base/oracle/perplexity_oracle.json` +commits the exact reproduction command, pinned corpus/artifact hashes, +the final PPL from this session's CPU reference run, and an explicit +±20% comparison threshold (matching this repo's own leaderboard-gate +policy) for independently verifying a future full-offload ET-SoC1 run +against this reference. No ET-SoC1 hardware was available to this +session to perform that run directly. + +## Instructions for Reproduction + +```bash +python3 -c "from huggingface_hub import snapshot_download; print(snapshot_download('deepseek-ai/deepseek-moe-16b-base'))" +# from the llama.cpp-et submodule root, with the standalone wrapper applied: +python3 convert_wrapper.py --outfile deepseek-moe-16b-base-Q8_0.gguf --outtype q8_0 +``` + +## Open items for maintainer review + +- No changes were made to any protected file, and none to the vendored + submodule — the tokenizer fix is a standalone wrapper script, same + pattern as the existing pythia410m/nemotron_h fixes. +- The `'deepseek-moe'`-named intermediate fix attempt (Bug 2 above) is + documented here specifically because it's a trap: it looks correct + from the Python side alone. Future ports of DeepSeek-family MoE + checkpoints with unrecognized chkhsh should register against an + *existing* C++-recognized pre-tokenizer name, not a new one, unless + the C++ registry itself is also being patched (out of scope here). diff --git a/ported_models/deepseek_moe_16b_base/oracle/perplexity_oracle.json b/ported_models/deepseek_moe_16b_base/oracle/perplexity_oracle.json new file mode 100644 index 00000000..f2833d53 --- /dev/null +++ b/ported_models/deepseek_moe_16b_base/oracle/perplexity_oracle.json @@ -0,0 +1,25 @@ +{ + "oracle_type": "perplexity_threshold", + "model_artifact": { + "repo": "darthceltic85/deepseek-moe-16b-base-gguf", + "revision": "3fcaaa31e06356109ae65d1fd655c98d01afbe0a", + "filename": "deepseek-moe-16b-base-Q8_0.gguf", + "sha256": "63eb27478a35ec36a3fd7c50e704ef6752ca235a59491d7673909da6fd8f0971" + }, + "corpus": { + "artifact": "wikitext2_raw_test", + "sha256": "173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08" + }, + "command": "llama-perplexity --model deepseek-moe-16b-base-Q8_0.gguf -f wiki.test.raw -c 128 -b 128 -ub 128 --chunks 4", + "reference_run": { + "final_ppl": 7.2909, + "final_ppl_stderr": 1.30187, + "measured_on": "CPU (ggml-cpu backend, GGML_ET=OFF build)", + "measured_date": "2026-07-26" + }, + "comparison_threshold": { + "metric": "final_ppl", + "max_relative_deviation": 0.20, + "note": "Matches this repo's own leaderboard-gate policy (PPL must stay within 20% of best-seen value). A full-offload ET-SoC1 re-run against this exact command/corpus/artifact should land at final_ppl within [5.83, 8.75] to be considered consistent with this reference run. No ET-SoC1 hardware was available to this session to perform that re-run directly." + } +} diff --git a/ported_models/llama_cpp_et/artifacts.json b/ported_models/llama_cpp_et/artifacts.json index 9c419ae1..fc4e9658 100644 --- a/ported_models/llama_cpp_et/artifacts.json +++ b/ported_models/llama_cpp_et/artifacts.json @@ -485,6 +485,24 @@ "sha256": "d1eb8b6b23979205fdf63703ed10f788131a3f812c7b1f72e0119d5d81295150", "size_bytes": 108783360, "note": "SmolVLM 500M vision projector (SigLIP ~93M + MLP). Q8_0 quantized. Must be loaded alongside smolvlm_500m_q8_gguf." + }, + "deepseek_moe_16b_base_q8_gguf": { + "kind": "model", + "framework": "llama.cpp-et", + "variant": "DeepSeek-MoE-16B-Base-Q8_0", + "filename": "deepseek-moe-16b-base-Q8_0.gguf", + "env": "DEEPSEEK_MOE_16B_BASE_MODEL_PATH", + "source": { + "type": "huggingface", + "repo": "darthceltic85/deepseek-moe-16b-base-gguf", + "revision": "3fcaaa31e06356109ae65d1fd655c98d01afbe0a", + "filename": "deepseek-moe-16b-base-Q8_0.gguf", + "url": "https://huggingface.co/darthceltic85/deepseek-moe-16b-base-gguf/resolve/3fcaaa31e06356109ae65d1fd655c98d01afbe0a/deepseek-moe-16b-base-Q8_0.gguf" + }, + "sha256": "63eb27478a35ec36a3fd7c50e704ef6752ca235a59491d7673909da6fd8f0971", + "local_cache": "local-artifacts/models/deepseek-moe-16b-base-Q8_0.gguf", + "board_path": "/data/models/deepseek-moe-16b-base-Q8_0.gguf", + "note": "Self-converted (no pre-made Q8_0 GGUF existed); hosted on Hugging Face since the 17.4 GB file far exceeds GitHub's 2 GB release-asset limit. Converter's tokenizer chkhsh table was missing this checkpoint's hash entirely; fixed via a standalone wrapper (see RECIPE.md) that also corrects a same-session naming mistake caught by actually loading the GGUF at runtime, not just trusting a clean Python conversion." } } } diff --git a/ported_models/llama_cpp_et/benchmarks/deepseek_moe_16b_base.json b/ported_models/llama_cpp_et/benchmarks/deepseek_moe_16b_base.json new file mode 100644 index 00000000..18fb964e --- /dev/null +++ b/ported_models/llama_cpp_et/benchmarks/deepseek_moe_16b_base.json @@ -0,0 +1,52 @@ +{ + "runner": "llama_server", + "board": true, + "framework": { + "name": "llama.cpp-et", + "runner": "llama_server", + "source_artifact": "llama_cpp_source" + }, + "artifacts_file": "../artifacts.json", + "canonical_variant": "DeepSeek-MoE-16B-Base-Q8_0", + "score": { + "metric": "tokens_per_second", + "label": "Decode tokens/s", + "higher_is_better": true + }, + "llama_server": { + "source_artifact": "llama_cpp_source", + "model_artifact": "deepseek_moe_16b_base_q8_gguf", + "server_artifact": "llama_server", + "workdir_artifact": "llama_cpp_build", + "host": "127.0.0.1", + "port": 18148, + "device": "ET", + "gpu_layers": 99, + "ctx_size": 2048, + "batch_size": 256, + "ubatch_size": 128, + "parallel": 1, + "cache_ram_mib": 0, + "ready_timeout_s": 300, + "request_timeout_s": 420, + "flash_attn": false, + "api": "completion", + "prompt": "Repeat this token sequence without commentary: OK OK OK OK OK OK OK OK OK OK", + "max_tokens": 96, + "temperature": 0, + "ignore_eos": true, + "min_completion_tokens": 32, + "perplexity": { + "enabled": true, + "perplexity_artifact": "llama_perplexity", + "corpus_artifact": "wikitext2_raw_test", + "ctx_size": 128, + "batch_size": 128, + "ubatch_size": 128, + "timeout_s": 420, + "min_ppl": 1.0, + "max_ppl": 1000.0, + "chunks": 4 + } + } +} diff --git a/ported_models/submissions/model_ports/deepseek_moe_16b_base.json b/ported_models/submissions/model_ports/deepseek_moe_16b_base.json new file mode 100644 index 00000000..e829ef99 --- /dev/null +++ b/ported_models/submissions/model_ports/deepseek_moe_16b_base.json @@ -0,0 +1,14 @@ +{ + "schema_version": 1, + "track": "most_models_ported", + "benchmark_model": "deepseek_moe_16b_base", + "identity_id": "deepseek", + "source": { + "repo": "deepseek-ai/deepseek-moe-16b-base", + "revision": "521d2bc4fb69a3f3ae565310fcc3b65f97af2580", + "license": "deepseek (custom model license, see https://github.com/deepseek-ai/DeepSeek-MoE/blob/main/LICENSE-MODEL)" + }, + "implementation_paths": ["ported_models/deepseek_moe_16b_base"], + "benchmark_config": "ported_models/llama_cpp_et/benchmarks/deepseek_moe_16b_base.json", + "recipe": "ported_models/deepseek_moe_16b_base/docs/RECIPE.md" +}