From eb3d2e63898cf77a687aa3b9997fdf6f97f68f0e Mon Sep 17 00:00:00 2001 From: DarthCeltic <178664201+DarthCeltic@users.noreply.github.com> Date: Sun, 26 Jul 2026 01:09:59 -0400 Subject: [PATCH 1/2] feat(model-ports): add smallthinker_4ba0_6b (SmallThinker execution family) --- .github/ci/benchmark_config.json | 3 + ported_models/llama_cpp_et/artifacts.json | 18 ++++++ .../benchmarks/smallthinker_4ba0_6b.json | 52 ++++++++++++++++ .../smallthinker_4ba0_6b/docs/RECIPE.md | 60 +++++++++++++++++++ .../model_ports/smallthinker_4ba0_6b.json | 14 +++++ 5 files changed, 147 insertions(+) create mode 100644 ported_models/llama_cpp_et/benchmarks/smallthinker_4ba0_6b.json create mode 100644 ported_models/smallthinker_4ba0_6b/docs/RECIPE.md create mode 100644 ported_models/submissions/model_ports/smallthinker_4ba0_6b.json diff --git a/.github/ci/benchmark_config.json b/.github/ci/benchmark_config.json index 43f8895d..e013077e 100644 --- a/.github/ci/benchmark_config.json +++ b/.github/ci/benchmark_config.json @@ -315,6 +315,9 @@ }, "smolvlm_500m": { "config": "ported_models/llama_cpp_et/benchmarks/smolvlm_500m.json" + }, + "smallthinker_4ba0_6b": { + "config": "ported_models/llama_cpp_et/benchmarks/smallthinker_4ba0_6b.json" } } } diff --git a/ported_models/llama_cpp_et/artifacts.json b/ported_models/llama_cpp_et/artifacts.json index 9c419ae1..1966f060 100644 --- a/ported_models/llama_cpp_et/artifacts.json +++ b/ported_models/llama_cpp_et/artifacts.json @@ -485,6 +485,24 @@ "sha256": "d1eb8b6b23979205fdf63703ed10f788131a3f812c7b1f72e0119d5d81295150", "size_bytes": 108783360, "note": "SmolVLM 500M vision projector (SigLIP ~93M + MLP). Q8_0 quantized. Must be loaded alongside smolvlm_500m_q8_gguf." + }, + "smallthinker_4ba0_6b_q8_gguf": { + "kind": "model", + "framework": "llama.cpp-et", + "variant": "SmallThinker-4BA0.6B-Q8_0", + "filename": "smallthinker-4ba0.6b-Q8_0.gguf", + "env": "SMALLTHINKER_4BA0_6B_MODEL_PATH", + "source": { + "type": "huggingface", + "repo": "darthceltic85/smallthinker-4ba0.6b-gguf", + "revision": "54918cdfe842275cb6330c190be5922f252e4fa7", + "filename": "smallthinker-4ba0.6b-Q8_0.gguf", + "url": "https://huggingface.co/darthceltic85/smallthinker-4ba0.6b-gguf/resolve/54918cdfe842275cb6330c190be5922f252e4fa7/smallthinker-4ba0.6b-Q8_0.gguf" + }, + "sha256": "6a352d7554a736c738cbef069aa1bb2fbf95c929567d48e9c4eb3b24f5942f16", + "local_cache": "local-artifacts/models/smallthinker-4ba0.6b-Q8_0.gguf", + "board_path": "/data/models/smallthinker-4ba0.6b-Q8_0.gguf", + "note": "Self-converted (no pre-made Q8_0 GGUF existed); hosted on Hugging Face since the 4.55 GB file exceeds GitHub's 2 GB release-asset limit." } } } diff --git a/ported_models/llama_cpp_et/benchmarks/smallthinker_4ba0_6b.json b/ported_models/llama_cpp_et/benchmarks/smallthinker_4ba0_6b.json new file mode 100644 index 00000000..65221232 --- /dev/null +++ b/ported_models/llama_cpp_et/benchmarks/smallthinker_4ba0_6b.json @@ -0,0 +1,52 @@ +{ + "runner": "llama_server", + "board": true, + "framework": { + "name": "llama.cpp-et", + "runner": "llama_server", + "source_artifact": "llama_cpp_source" + }, + "artifacts_file": "../artifacts.json", + "canonical_variant": "SmallThinker-4BA0.6B-Q8_0", + "score": { + "metric": "tokens_per_second", + "label": "Decode tokens/s", + "higher_is_better": true + }, + "llama_server": { + "source_artifact": "llama_cpp_source", + "model_artifact": "smallthinker_4ba0_6b_q8_gguf", + "server_artifact": "llama_server", + "workdir_artifact": "llama_cpp_build", + "host": "127.0.0.1", + "port": 18146, + "device": "ET", + "gpu_layers": 99, + "ctx_size": 2048, + "batch_size": 256, + "ubatch_size": 128, + "parallel": 1, + "cache_ram_mib": 0, + "ready_timeout_s": 300, + "request_timeout_s": 420, + "flash_attn": false, + "api": "completion", + "prompt": "Repeat this token sequence without commentary: OK OK OK OK OK OK OK OK OK OK", + "max_tokens": 96, + "temperature": 0, + "ignore_eos": true, + "min_completion_tokens": 32, + "perplexity": { + "enabled": true, + "perplexity_artifact": "llama_perplexity", + "corpus_artifact": "wikitext2_raw_test", + "ctx_size": 128, + "batch_size": 128, + "ubatch_size": 128, + "timeout_s": 420, + "min_ppl": 1.0, + "max_ppl": 1000.0, + "chunks": 4 + } + } +} diff --git a/ported_models/smallthinker_4ba0_6b/docs/RECIPE.md b/ported_models/smallthinker_4ba0_6b/docs/RECIPE.md new file mode 100644 index 00000000..d42695a6 --- /dev/null +++ b/ported_models/smallthinker_4ba0_6b/docs/RECIPE.md @@ -0,0 +1,60 @@ +# SmallThinker-4BA0.6B-Instruct Porting Recipe + +## Overview + +Adds `PowerInfer/SmallThinker-4BA0.6B-Instruct` (4B-total/0.6B-active +Mixture-of-Experts causal LM, 32 layers, 32 experts, 4 active per token, +32K context) to the `llama_cpp_et` benchmark suite. This introduces the +**SmallThinker** execution family to the board — a MoE architecture +purpose-built for resource-constrained, on-device local deployment +(co-developed by IPADS/Shanghai Jiao Tong University and Zenergize AI). + +## Model Reference + +- **Source**: `PowerInfer/SmallThinker-4BA0.6B-Instruct` (Hugging Face), + revision `a24932bb880c85c04c794283202a9194b1b42709` +- **License**: Apache 2.0 +- **Architecture**: `arch = smallthinker` (`SmallThinkerForCausalLM`), 32 + layers, 32 experts, 4 active per token. + +## Conversion + +No usable pre-made Q8_0 GGUF exists for this checkpoint at the time of +this port, so it was self-converted directly from safetensors using this +repo's own `convert_hf_to_gguf.py --outtype q8_0`, no fixes needed — +converted cleanly on the first attempt. Produced a 4.55 GB file, +`sha256=6a352d7554a736c738cbef069aa1bb2fbf95c929567d48e9c4eb3b24f5942f16`. + +## Hosting + +This GGUF (4.55 GB) exceeds GitHub's 2 GB release-asset limit, so unlike +the earlier `jamba_tiny_dev` self-converted port (hosted as a GitHub +Release), this one is hosted on Hugging Face instead: +`darthceltic85/smallthinker-4ba0.6b-gguf`, file +`smallthinker-4ba0.6b-Q8_0.gguf`. + +## Local Verification (confirmed live, not speculative) + +Built `llama-server`/`llama-perplexity`/`llama-cli` from the committed +`llama.cpp-et` submodule (CPU backend — see the `jamba_tiny_dev` recipe +for the verification-tier note on why CPU, not full ET sysemu) and ran +real inference: + +- Model loads cleanly: `arch = smallthinker`, clean 1640-node compute + graph, 1 split, "fused Gated Delta Net" MoE routing paths enabled. +- Real perplexity run against WikiText-2 raw (4 chunks, ctx=128, + batch=128): **PPL = 14.2763 +/- 2.92502** — solidly within this + campaign's normal range, confirming genuine, coherent-quality output. + +## Instructions for Reproduction + +```bash +python3 -c "from huggingface_hub import snapshot_download; print(snapshot_download('PowerInfer/SmallThinker-4BA0.6B-Instruct'))" +# from the llama.cpp-et submodule root: +python3 convert_hf_to_gguf.py --outfile smallthinker-4b-a0.6b-Q8_0.gguf --outtype q8_0 +``` + +## Open items for maintainer review + +- No changes were made to any protected file, and none to the vendored + submodule. diff --git a/ported_models/submissions/model_ports/smallthinker_4ba0_6b.json b/ported_models/submissions/model_ports/smallthinker_4ba0_6b.json new file mode 100644 index 00000000..3a56e292 --- /dev/null +++ b/ported_models/submissions/model_ports/smallthinker_4ba0_6b.json @@ -0,0 +1,14 @@ +{ + "schema_version": 1, + "track": "most_models_ported", + "benchmark_model": "smallthinker_4ba0_6b", + "identity_id": "smallthinker", + "source": { + "repo": "PowerInfer/SmallThinker-4BA0.6B-Instruct", + "revision": "a24932bb880c85c04c794283202a9194b1b42709", + "license": "apache-2.0" + }, + "implementation_paths": ["ported_models/smallthinker_4ba0_6b"], + "benchmark_config": "ported_models/llama_cpp_et/benchmarks/smallthinker_4ba0_6b.json", + "recipe": "ported_models/smallthinker_4ba0_6b/docs/RECIPE.md" +} From ef8934a0153b9add20403b5a5c08b9acbb9c4f45 Mon Sep 17 00:00:00 2001 From: DarthCeltic <178664201+DarthCeltic@users.noreply.github.com> Date: Sun, 26 Jul 2026 07:26:07 -0400 Subject: [PATCH 2/2] Add committed deterministic PPL oracle per maintainer review --- .../smallthinker_4ba0_6b/docs/RECIPE.md | 8 ++++++ .../oracle/perplexity_oracle.json | 25 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 ported_models/smallthinker_4ba0_6b/oracle/perplexity_oracle.json diff --git a/ported_models/smallthinker_4ba0_6b/docs/RECIPE.md b/ported_models/smallthinker_4ba0_6b/docs/RECIPE.md index d42695a6..b4dc9541 100644 --- a/ported_models/smallthinker_4ba0_6b/docs/RECIPE.md +++ b/ported_models/smallthinker_4ba0_6b/docs/RECIPE.md @@ -46,6 +46,14 @@ real inference: batch=128): **PPL = 14.2763 +/- 2.92502** — solidly within this campaign's normal range, confirming genuine, coherent-quality output. +## Committed deterministic oracle (added per maintainer review) + +`ported_models/smallthinker_4ba0_6b/oracle/perplexity_oracle.json` +commits the exact reproduction command, pinned corpus/artifact hashes, +the final PPL from this session's CPU reference run, and an explicit +±20% comparison threshold for independently verifying a future +full-offload ET-SoC1 run against this reference. + ## Instructions for Reproduction ```bash diff --git a/ported_models/smallthinker_4ba0_6b/oracle/perplexity_oracle.json b/ported_models/smallthinker_4ba0_6b/oracle/perplexity_oracle.json new file mode 100644 index 00000000..d634fe05 --- /dev/null +++ b/ported_models/smallthinker_4ba0_6b/oracle/perplexity_oracle.json @@ -0,0 +1,25 @@ +{ + "oracle_type": "perplexity_threshold", + "model_artifact": { + "repo": "darthceltic85/smallthinker-4ba0.6b-gguf", + "revision": "54918cdfe842275cb6330c190be5922f252e4fa7", + "filename": "smallthinker-4ba0.6b-Q8_0.gguf", + "sha256": "6a352d7554a736c738cbef069aa1bb2fbf95c929567d48e9c4eb3b24f5942f16" + }, + "corpus": { + "artifact": "wikitext2_raw_test", + "sha256": "173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08" + }, + "command": "llama-perplexity --model smallthinker-4ba0.6b-Q8_0.gguf -f wiki.test.raw -c 128 -b 128 -ub 128 --chunks 4", + "reference_run": { + "final_ppl": 14.2763, + "final_ppl_stderr": 2.92502, + "measured_on": "CPU (ggml-cpu backend, GGML_ET=OFF build)", + "measured_date": "2026-07-25" + }, + "comparison_threshold": { + "metric": "final_ppl", + "max_relative_deviation": 0.20, + "note": "Matches this repo's own leaderboard-gate policy (PPL must stay within 20% of best-seen value). A full-offload ET-SoC1 re-run against this exact command/corpus/artifact should land at final_ppl within [11.42, 17.13] to be considered consistent with this reference run. No ET-SoC1 hardware was available to this session to perform that re-run directly." + } +}