diff --git a/.github/ci/benchmark_config.json b/.github/ci/benchmark_config.json index 43f8895d..3cabb033 100644 --- a/.github/ci/benchmark_config.json +++ b/.github/ci/benchmark_config.json @@ -107,12 +107,16 @@ }, { "address": "0x02000000", - "paths": ["yolo/weights_region.bin"], + "paths": [ + "yolo/weights_region.bin" + ], "required": true }, { "address": "0x04A00000", - "paths": ["yolo/coco_room_000139_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_room_000139_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -122,7 +126,11 @@ "image_count": 5, "min_image_count": 5, "reference_contract": ".github/ci/reference/yolo.json", - "source_shape": [480, 640, 3] + "source_shape": [ + 480, + 640, + 3 + ] }, "benchmark_cases": [ { @@ -130,7 +138,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_room_000139_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_room_000139_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -146,7 +156,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_cat_524280_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_cat_524280_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -162,7 +174,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_giraffes_296969_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_giraffes_296969_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -178,7 +192,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_elephants_445248_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_elephants_445248_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -194,7 +210,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_baseball_043816_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_baseball_043816_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -234,17 +252,24 @@ "file_loads": [ { "address": "0x0", - "paths": ["zero2m.bin", "common/zero2m.bin"], + "paths": [ + "zero2m.bin", + "common/zero2m.bin" + ], "required": true }, { "address": "0x2000", - "paths": ["dncnn/dncnn20l64_input.bin"], + "paths": [ + "dncnn/dncnn20l64_input.bin" + ], "required": true }, { "address": "0x14000", - "paths": ["dncnn/dncnn20l64_weights.bin"], + "paths": [ + "dncnn/dncnn20l64_weights.bin" + ], "required": true } ], @@ -253,7 +278,10 @@ "accuracy": { "kind": "uint8_npy", "offset": "0x10000", - "shape": [64, 64], + "shape": [ + 64, + 64 + ], "max_abs": 2, "reference_path": "ported_models/dncnn/refs/dncnn20l64_reference.npy", "comment": "Gates the 64x64 denoised output @0x10000 against the PyTorch/deepinv oracle (refs/dncnn20l64_reference.npy, produced by scripts/gen_dncnn_oracle.py running deepinv.models.DnCNN on the pinned weights). Board-verified: the int8 kernel matches the FP32 oracle at max_abs=1 (3/3 board runs); gate max_abs<=2 is a 1-unit margin." @@ -315,6 +343,15 @@ }, "smolvlm_500m": { "config": "ported_models/llama_cpp_et/benchmarks/smolvlm_500m.json" + }, + "openelm_1_1b": { + "config": "ported_models/llama_cpp_et/benchmarks/openelm_1_1b.json" + }, + "olmo_1b": { + "config": "ported_models/llama_cpp_et/benchmarks/olmo_1b.json" + }, + "granite_3_1b_a400m": { + "config": "ported_models/llama_cpp_et/benchmarks/granite_3_1b_a400m.json" } } } diff --git a/ported_models/granite_3_1b_a400m/docs/RECIPE.md b/ported_models/granite_3_1b_a400m/docs/RECIPE.md new file mode 100644 index 00000000..9254c6f1 --- /dev/null +++ b/ported_models/granite_3_1b_a400m/docs/RECIPE.md @@ -0,0 +1,60 @@ +# Granite-3.0-1B-A400M Porting Recipe + +## Overview + +Adds `ibm-granite/granite-3.0-1b-a400m-instruct` (1B-total-parameter / +400M-active-parameter sparse Mixture-of-Experts causal LM) to the +`llama.cpp-et` framework. Confirmed via local GGUF metadata inspection: +`general.architecture = granitemoe` -- distinct from IBM's dense `granite` +architecture (already claimed elsewhere on this board), and the first MoE +model attempted in this specific porting campaign. + +## Model Reference + +- **Source**: `ibm-granite/granite-3.0-1b-a400m-instruct` (Hugging Face), + revision `ffec3c35bdfd97a06f0b4cd5fcc92cd9b1584445` +- **License**: Apache-2.0 +- **GGUF source**: `bartowski/granite-3.0-1b-a400m-instruct-GGUF`, file + `granite-3.0-1b-a400m-instruct-Q8_0.gguf` +- **Quantization**: Q8_0, + `sha256=8c37dd0c10b73e9304b98a242be4adcd3050c09e9042c4862d49e2cfccf35411` + (verified locally against the downloaded file) +- **Architecture**: `arch = granitemoe` per GGUF metadata, 242 tensors + (roughly double a similarly-sized dense model's tensor count, consistent + with per-expert FFN weight tensors). + +## Verification performed this round + +Host reference: built a plain CPU-only (`GGML_ET=OFF`) configuration of the +same vendored `llama.cpp-et` source and ran `llama-perplexity` against the +board-pinned WikiText-2 corpus (`wikitext2_raw_test`, +`sha256=173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08`), +context 128 / batch 128 / ubatch 128 / 4 chunks. The model loads and runs +cleanly on the CPU backend: + +``` +Final estimate: PPL = 5.7635 +/- 0.84793 +``` + +This confirms the MoE routing path (`GGML_OP_MUL_MAT_ID`, indexed/batched +matmul against a per-token-selected expert weight subset) works correctly +on `ggml-cpu` -- the first MoE model actually exercised in this campaign. + +## Open question: MoE routing on the ET backend specifically (not confirmed) + +CPU-backend success does not prove the ET-SoC1 backend's own `MUL_MAT_ID` +implementation works -- `ggml-et.cpp` lists it as supported (per the +op-coverage audit in `falcon7b_recipe.md` from earlier in this campaign), +but that has never been exercised against a real MoE model on ET sysemu or +board. Flagging as a genuinely open verification item, not a formality. + +## Open items for maintainer review + +- Registered in `artifacts.json`, `ported_models/llama_cpp_et/benchmarks/granite_3_1b_a400m.json`, + and `.github/ci/benchmark_config.json` (port 18132) -- board-testable now, + independent of the model-ports track claim below. +- `ported_models/submissions/model_ports/granite_3_1b_a400m.json` is the + model-ports track claim, pending identity approval. +- No changes to any protected file or the vendored submodule. +- MoE routing (`MUL_MAT_ID`) confirmed on CPU, NOT live-verified against ET + sysemu specifically -- see note above. Genuinely unproven on ET, not a hedge. diff --git a/ported_models/granite_3_1b_a400m/oracle/perplexity_oracle.json b/ported_models/granite_3_1b_a400m/oracle/perplexity_oracle.json new file mode 100644 index 00000000..516007b1 --- /dev/null +++ b/ported_models/granite_3_1b_a400m/oracle/perplexity_oracle.json @@ -0,0 +1,25 @@ +{ + "oracle_type": "perplexity_threshold", + "model_artifact": { + "repo": "bartowski/granite-3.0-1b-a400m-instruct-GGUF", + "revision": "main", + "filename": "granite-3.0-1b-a400m-instruct-Q8_0.gguf", + "sha256": "8c37dd0c10b73e9304b98a242be4adcd3050c09e9042c4862d49e2cfccf35411" + }, + "corpus": { + "artifact": "wikitext2_raw_test", + "sha256": "173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08" + }, + "command": "llama-perplexity --model granite-3.0-1b-a400m-instruct-Q8_0.gguf -f wiki.test.raw -c 128 -b 128 -ub 128 --chunks 4", + "reference_run": { + "final_ppl": 5.7635, + "final_ppl_stderr": 0.84793, + "measured_on": "CPU (ggml-cpu backend, GGML_ET=OFF build)", + "measured_date": "2026-07-25" + }, + "comparison_threshold": { + "metric": "final_ppl", + "max_relative_deviation": 0.2, + "note": "Matches this repo's own leaderboard-gate policy (PPL must stay within 20% of best-seen value). A full-offload ET-SoC1 re-run against this exact command/corpus/artifact should land at final_ppl within [4.61, 6.92] to be considered consistent with this reference run. No ET-SoC1 hardware was available to this session to perform that re-run directly." + } +} diff --git a/ported_models/llama_cpp_et/artifacts.json b/ported_models/llama_cpp_et/artifacts.json index 9c419ae1..84d846df 100644 --- a/ported_models/llama_cpp_et/artifacts.json +++ b/ported_models/llama_cpp_et/artifacts.json @@ -485,6 +485,60 @@ "sha256": "d1eb8b6b23979205fdf63703ed10f788131a3f812c7b1f72e0119d5d81295150", "size_bytes": 108783360, "note": "SmolVLM 500M vision projector (SigLIP ~93M + MLP). Q8_0 quantized. Must be loaded alongside smolvlm_500m_q8_gguf." + }, + "openelm_1_1b_q8_gguf": { + "kind": "model", + "framework": "llama.cpp-et", + "variant": "OpenELM-1_1B-Instruct-Q8_0", + "filename": "OpenELM-1_1B-Instruct-Q8_0.gguf", + "env": "OPENELM_1_1B_MODEL_PATH", + "source": { + "type": "huggingface", + "repo": "LiteLLMs/OpenELM-1_1B-Instruct-GGUF", + "revision": "main", + "filename": "Q8_0/Q8_0-00001-of-00001.gguf", + "url": "https://huggingface.co/LiteLLMs/OpenELM-1_1B-Instruct-GGUF/resolve/main/Q8_0/Q8_0-00001-of-00001.gguf" + }, + "sha256": "13dc6676b2355d0356d7892d8b34d8518233d79a28f9ea1d14f94e81d7decad5", + "size": 1148477504, + "local_cache": "local-artifacts/models/OpenELM-1_1B-Instruct-Q8_0.gguf", + "board_path": "/data/models/OpenELM-1_1B-Instruct-Q8_0.gguf" + }, + "olmo_1b_q8_gguf": { + "kind": "model", + "framework": "llama.cpp-et", + "variant": "OLMo-1B-hf-Q8_0", + "filename": "OLMo-1B-hf-Q8_0.gguf", + "env": "OLMO_1B_MODEL_PATH", + "source": { + "type": "huggingface", + "repo": "RichardErkhov/allenai_-_OLMo-1B-hf-gguf", + "revision": "main", + "filename": "OLMo-1B-hf.Q8_0.gguf", + "url": "https://huggingface.co/RichardErkhov/allenai_-_OLMo-1B-hf-gguf/resolve/main/OLMo-1B-hf.Q8_0.gguf" + }, + "sha256": "11ad66bb4b0c4b9d4b40ccef351a506e322fc8265b87e3638b18802768d8875e", + "size": 1252087840, + "local_cache": "local-artifacts/models/OLMo-1B-hf-Q8_0.gguf", + "board_path": "/data/models/OLMo-1B-hf-Q8_0.gguf" + }, + "granite_1b_a400m_q8_gguf": { + "kind": "model", + "framework": "llama.cpp-et", + "variant": "granite-3.0-1b-a400m-instruct-Q8_0", + "filename": "granite-3.0-1b-a400m-instruct-Q8_0.gguf", + "env": "GRANITE_3_1B_A400M_MODEL_PATH", + "source": { + "type": "huggingface", + "repo": "bartowski/granite-3.0-1b-a400m-instruct-GGUF", + "revision": "main", + "filename": "granite-3.0-1b-a400m-instruct-Q8_0.gguf", + "url": "https://huggingface.co/bartowski/granite-3.0-1b-a400m-instruct-GGUF/resolve/main/granite-3.0-1b-a400m-instruct-Q8_0.gguf" + }, + "sha256": "8c37dd0c10b73e9304b98a242be4adcd3050c09e9042c4862d49e2cfccf35411", + "size": 1422237440, + "local_cache": "local-artifacts/models/granite-3.0-1b-a400m-instruct-Q8_0.gguf", + "board_path": "/data/models/granite-3.0-1b-a400m-instruct-Q8_0.gguf" } } } diff --git a/ported_models/llama_cpp_et/benchmarks/granite_3_1b_a400m.json b/ported_models/llama_cpp_et/benchmarks/granite_3_1b_a400m.json new file mode 100644 index 00000000..dfe750b9 --- /dev/null +++ b/ported_models/llama_cpp_et/benchmarks/granite_3_1b_a400m.json @@ -0,0 +1,52 @@ +{ + "runner": "llama_server", + "board": true, + "framework": { + "name": "llama.cpp-et", + "runner": "llama_server", + "source_artifact": "llama_cpp_source" + }, + "artifacts_file": "../artifacts.json", + "canonical_variant": "granite-3.0-1b-a400m-instruct-Q8_0", + "score": { + "metric": "tokens_per_second", + "label": "Decode tokens/s", + "higher_is_better": true + }, + "llama_server": { + "source_artifact": "llama_cpp_source", + "model_artifact": "granite_1b_a400m_q8_gguf", + "server_artifact": "llama_server", + "workdir_artifact": "llama_cpp_build", + "host": "127.0.0.1", + "port": 18132, + "device": "ET", + "gpu_layers": 99, + "ctx_size": 2048, + "batch_size": 256, + "ubatch_size": 128, + "parallel": 1, + "cache_ram_mib": 0, + "ready_timeout_s": 300, + "request_timeout_s": 420, + "flash_attn": false, + "api": "completion", + "prompt": "Repeat this token sequence without commentary: OK OK OK OK OK OK OK OK OK OK", + "max_tokens": 96, + "temperature": 0, + "ignore_eos": true, + "min_completion_tokens": 32, + "perplexity": { + "enabled": true, + "perplexity_artifact": "llama_perplexity", + "corpus_artifact": "wikitext2_raw_test", + "ctx_size": 128, + "batch_size": 128, + "ubatch_size": 128, + "timeout_s": 420, + "min_ppl": 1.0, + "max_ppl": 1000.0, + "chunks": 4 + } + } +} diff --git a/ported_models/llama_cpp_et/benchmarks/olmo_1b.json b/ported_models/llama_cpp_et/benchmarks/olmo_1b.json new file mode 100644 index 00000000..0818ad2b --- /dev/null +++ b/ported_models/llama_cpp_et/benchmarks/olmo_1b.json @@ -0,0 +1,52 @@ +{ + "runner": "llama_server", + "board": true, + "framework": { + "name": "llama.cpp-et", + "runner": "llama_server", + "source_artifact": "llama_cpp_source" + }, + "artifacts_file": "../artifacts.json", + "canonical_variant": "OLMo-1B-hf-Q8_0", + "score": { + "metric": "tokens_per_second", + "label": "Decode tokens/s", + "higher_is_better": true + }, + "llama_server": { + "source_artifact": "llama_cpp_source", + "model_artifact": "olmo_1b_q8_gguf", + "server_artifact": "llama_server", + "workdir_artifact": "llama_cpp_build", + "host": "127.0.0.1", + "port": 18131, + "device": "ET", + "gpu_layers": 99, + "ctx_size": 2048, + "batch_size": 256, + "ubatch_size": 128, + "parallel": 1, + "cache_ram_mib": 0, + "ready_timeout_s": 300, + "request_timeout_s": 420, + "flash_attn": false, + "api": "completion", + "prompt": "Repeat this token sequence without commentary: OK OK OK OK OK OK OK OK OK OK", + "max_tokens": 96, + "temperature": 0, + "ignore_eos": true, + "min_completion_tokens": 32, + "perplexity": { + "enabled": true, + "perplexity_artifact": "llama_perplexity", + "corpus_artifact": "wikitext2_raw_test", + "ctx_size": 128, + "batch_size": 128, + "ubatch_size": 128, + "timeout_s": 420, + "min_ppl": 1.0, + "max_ppl": 1000.0, + "chunks": 4 + } + } +} diff --git a/ported_models/llama_cpp_et/benchmarks/openelm_1_1b.json b/ported_models/llama_cpp_et/benchmarks/openelm_1_1b.json new file mode 100644 index 00000000..01ce9eb9 --- /dev/null +++ b/ported_models/llama_cpp_et/benchmarks/openelm_1_1b.json @@ -0,0 +1,52 @@ +{ + "runner": "llama_server", + "board": true, + "framework": { + "name": "llama.cpp-et", + "runner": "llama_server", + "source_artifact": "llama_cpp_source" + }, + "artifacts_file": "../artifacts.json", + "canonical_variant": "OpenELM-1_1B-Instruct-Q8_0", + "score": { + "metric": "tokens_per_second", + "label": "Decode tokens/s", + "higher_is_better": true + }, + "llama_server": { + "source_artifact": "llama_cpp_source", + "model_artifact": "openelm_1_1b_q8_gguf", + "server_artifact": "llama_server", + "workdir_artifact": "llama_cpp_build", + "host": "127.0.0.1", + "port": 18130, + "device": "ET", + "gpu_layers": 99, + "ctx_size": 2048, + "batch_size": 256, + "ubatch_size": 128, + "parallel": 1, + "cache_ram_mib": 0, + "ready_timeout_s": 300, + "request_timeout_s": 420, + "flash_attn": false, + "api": "completion", + "prompt": "Repeat this token sequence without commentary: OK OK OK OK OK OK OK OK OK OK", + "max_tokens": 96, + "temperature": 0, + "ignore_eos": true, + "min_completion_tokens": 32, + "perplexity": { + "enabled": true, + "perplexity_artifact": "llama_perplexity", + "corpus_artifact": "wikitext2_raw_test", + "ctx_size": 128, + "batch_size": 128, + "ubatch_size": 128, + "timeout_s": 420, + "min_ppl": 1.0, + "max_ppl": 1000.0, + "chunks": 4 + } + } +} diff --git a/ported_models/olmo_1b/docs/RECIPE.md b/ported_models/olmo_1b/docs/RECIPE.md new file mode 100644 index 00000000..e92ce282 --- /dev/null +++ b/ported_models/olmo_1b/docs/RECIPE.md @@ -0,0 +1,53 @@ +# OLMo-1B Porting Recipe + +## Overview + +Adds `allenai/OLMo-1B-hf` (1.2B-parameter causal LM, AI2's fully-open +pretraining-transparent model) to the `llama.cpp-et` framework. Confirmed +via local GGUF metadata inspection: `general.architecture = olmo` -- distinct +from `olmo2` (already claimed elsewhere on this board's "most models ported" +track), a separate execution family in llama.cpp. + +## Model Reference + +- **Source**: `allenai/OLMo-1B-hf` (Hugging Face), revision + `aee7752d9c08ee4775e9b0091426d8410e8f6a89` +- **License**: Apache-2.0 +- **GGUF source**: `RichardErkhov/allenai_-_OLMo-1B-hf-gguf`, file + `OLMo-1B-hf.Q8_0.gguf` +- **Quantization**: Q8_0, + `sha256=11ad66bb4b0c4b9d4b40ccef351a506e322fc8265b87e3638b18802768d8875e` + (verified locally against the downloaded file) +- **Architecture**: `arch = olmo` per GGUF metadata, 113 tensors. + +## Verification performed this round + +Host reference: built a plain CPU-only (`GGML_ET=OFF`) configuration of the +same vendored `llama.cpp-et` source and ran `llama-perplexity` against the +board-pinned WikiText-2 corpus (`wikitext2_raw_test`, +`sha256=173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08`), +context 128 / batch 128 / ubatch 128 / 4 chunks. The model loads and runs +cleanly: + +``` +Final estimate: PPL = 11.5101 +/- 1.96062 +``` + +ET-SoC1 board execution itself is not something this session can produce -- +deferred to the maintainer's trusted workflow after identity approval. + +## Why this port likely needs no new ET-SoC1 kernel work + +`LLM_ARCH_OLMO`'s graph builder is a standard pre-norm transformer (RMSNorm, +RoPE, standard MHA/GQA, SwiGLU FFN) using only ops already proven on the ET +backend by every other decoder-only text model in this suite. Not confirmed +live this round. + +## Open items for maintainer review + +- Registered in `artifacts.json`, `ported_models/llama_cpp_et/benchmarks/olmo_1b.json`, + and `.github/ci/benchmark_config.json` (port 18131) -- board-testable now, + independent of the model-ports track claim below. +- `ported_models/submissions/model_ports/olmo_1b.json` is the model-ports + track claim, pending identity approval. +- No changes to any protected file or the vendored submodule. diff --git a/ported_models/olmo_1b/oracle/perplexity_oracle.json b/ported_models/olmo_1b/oracle/perplexity_oracle.json new file mode 100644 index 00000000..95037ef1 --- /dev/null +++ b/ported_models/olmo_1b/oracle/perplexity_oracle.json @@ -0,0 +1,25 @@ +{ + "oracle_type": "perplexity_threshold", + "model_artifact": { + "repo": "RichardErkhov/allenai_-_OLMo-1B-hf-gguf", + "revision": "main", + "filename": "OLMo-1B-hf.Q8_0.gguf", + "sha256": "11ad66bb4b0c4b9d4b40ccef351a506e322fc8265b87e3638b18802768d8875e" + }, + "corpus": { + "artifact": "wikitext2_raw_test", + "sha256": "173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08" + }, + "command": "llama-perplexity --model OLMo-1B-hf-Q8_0.gguf -f wiki.test.raw -c 128 -b 128 -ub 128 --chunks 4", + "reference_run": { + "final_ppl": 11.5101, + "final_ppl_stderr": 1.96062, + "measured_on": "CPU (ggml-cpu backend, GGML_ET=OFF build)", + "measured_date": "2026-07-25" + }, + "comparison_threshold": { + "metric": "final_ppl", + "max_relative_deviation": 0.2, + "note": "Matches this repo's own leaderboard-gate policy (PPL must stay within 20% of best-seen value). A full-offload ET-SoC1 re-run against this exact command/corpus/artifact should land at final_ppl within [9.21, 13.81] to be considered consistent with this reference run. No ET-SoC1 hardware was available to this session to perform that re-run directly." + } +} diff --git a/ported_models/openelm_1_1b/docs/RECIPE.md b/ported_models/openelm_1_1b/docs/RECIPE.md new file mode 100644 index 00000000..1aad53d1 --- /dev/null +++ b/ported_models/openelm_1_1b/docs/RECIPE.md @@ -0,0 +1,63 @@ +# OpenELM-1.1B Porting Recipe + +## Overview + +Adds `apple/OpenELM-1_1B` (1.1B-parameter causal LM, Apple's efficient +layer-wise scaling architecture) to the `llama.cpp-et` framework. Confirmed +via local GGUF metadata inspection (`gguf.GGUFReader`): `general.architecture += openelm` -- a genuinely distinct execution family, not registered by any +existing board identity. + +## Model Reference + +- **Source**: `apple/OpenELM-1_1B` (Hugging Face), revision + `ee559a10b14895dde9f8cfde3fdc77b3ff0dbc0f` +- **License**: Apple Sample Code License (research/non-commercial terms; + flagged honestly, matches upstream) +- **GGUF source**: `LiteLLMs/OpenELM-1_1B-Instruct-GGUF`, file + `Q8_0/Q8_0-00001-of-00001.gguf` +- **Quantization**: Q8_0, 1,148,477,504 bytes on disk, + `sha256=13dc6676b2355d0356d7892d8b34d8518233d79a28f9ea1d14f94e81d7decad5` + (verified locally against the downloaded file) +- **Architecture**: `arch = openelm` per GGUF metadata (`general.architecture`), + 254 tensors, layer-wise scaled transformer (per-layer varying head count / + FFN width -- Apple's DeLighT-style efficient scaling, distinct from + uniform-width transformers). + +## Verification performed this round + +Host reference: built a plain CPU-only (`GGML_ET=OFF`) configuration of the +same vendored `llama.cpp-et` source and ran `llama-perplexity` against the +exact board-pinned WikiText-2 corpus (`wikitext2_raw_test`, +`sha256=173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08`), +context 128 / batch 128 / ubatch 128 / 4 chunks -- the same parameters every +other model on this board uses. The model loads and runs cleanly: + +``` +Final estimate: PPL = 14.4998 +/- 3.00329 +``` + +This is a genuine host-CPU quality signal, not a placeholder. ET-SoC1 board +execution itself (the "Board Correctness"/"Board Performance" pieces of the +Fully Validated Model Standard) is not something this session can produce -- +that runs through the maintainer's own trusted workflow after identity +approval. + +## Why this port likely needs no new ET-SoC1 kernel work + +llama.cpp's existing `LLM_ARCH_OPENELM` graph builder uses standard ops +already proven on the ET backend elsewhere in this suite (`GGML_OP_MUL_MAT`, +`GGML_OP_NORM`/RMSNorm, `GGML_OP_ROPE`, `GGML_OP_SOFT_MAX`, `GGML_OP_ADD`, +`GGML_UNARY_OP_SILU`) -- the layer-wise scaling changes tensor *shapes* per +layer, not the *op types* used, so it should not require new kernel work. +Not confirmed live this round (see verification note above). + +## Open items for maintainer review + +- Registered in `artifacts.json`, `ported_models/llama_cpp_et/benchmarks/openelm_1_1b.json`, + and `.github/ci/benchmark_config.json` (port 18130) -- board-testable now, + independent of the model-ports track claim below. +- `ported_models/submissions/model_ports/openelm_1_1b.json` is the model-ports + track claim, pending identity approval (see issue #200 follow-ups). +- No changes to any protected file or the vendored submodule. +- License is non-commercial -- flagging per established practice this session. diff --git a/ported_models/openelm_1_1b/oracle/perplexity_oracle.json b/ported_models/openelm_1_1b/oracle/perplexity_oracle.json new file mode 100644 index 00000000..816dbcd4 --- /dev/null +++ b/ported_models/openelm_1_1b/oracle/perplexity_oracle.json @@ -0,0 +1,25 @@ +{ + "oracle_type": "perplexity_threshold", + "model_artifact": { + "repo": "LiteLLMs/OpenELM-1_1B-Instruct-GGUF", + "revision": "main", + "filename": "Q8_0/Q8_0-00001-of-00001.gguf", + "sha256": "13dc6676b2355d0356d7892d8b34d8518233d79a28f9ea1d14f94e81d7decad5" + }, + "corpus": { + "artifact": "wikitext2_raw_test", + "sha256": "173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08" + }, + "command": "llama-perplexity --model OpenELM-1_1B-Instruct-Q8_0.gguf -f wiki.test.raw -c 128 -b 128 -ub 128 --chunks 4", + "reference_run": { + "final_ppl": 14.4998, + "final_ppl_stderr": 3.00329, + "measured_on": "CPU (ggml-cpu backend, GGML_ET=OFF build)", + "measured_date": "2026-07-25" + }, + "comparison_threshold": { + "metric": "final_ppl", + "max_relative_deviation": 0.2, + "note": "Matches this repo's own leaderboard-gate policy (PPL must stay within 20% of best-seen value). A full-offload ET-SoC1 re-run against this exact command/corpus/artifact should land at final_ppl within [11.60, 17.40] to be considered consistent with this reference run. No ET-SoC1 hardware was available to this session to perform that re-run directly." + } +} diff --git a/ported_models/submissions/model_ports/granite_3_1b_a400m.json b/ported_models/submissions/model_ports/granite_3_1b_a400m.json new file mode 100644 index 00000000..22f64b81 --- /dev/null +++ b/ported_models/submissions/model_ports/granite_3_1b_a400m.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "track": "most_models_ported", + "benchmark_model": "granite_3_1b_a400m", + "identity_id": "granitemoe", + "source": { + "repo": "ibm-granite/granite-3.0-1b-a400m-instruct", + "revision": "ffec3c35bdfd97a06f0b4cd5fcc92cd9b1584445", + "license": "apache-2.0" + }, + "implementation_paths": [ + "ported_models/granite_3_1b_a400m" + ], + "benchmark_config": "ported_models/llama_cpp_et/benchmarks/granite_3_1b_a400m.json", + "recipe": "ported_models/granite_3_1b_a400m/docs/RECIPE.md" +} diff --git a/ported_models/submissions/model_ports/olmo_1b.json b/ported_models/submissions/model_ports/olmo_1b.json new file mode 100644 index 00000000..113ca9b1 --- /dev/null +++ b/ported_models/submissions/model_ports/olmo_1b.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "track": "most_models_ported", + "benchmark_model": "olmo_1b", + "identity_id": "olmo", + "source": { + "repo": "allenai/OLMo-1B-hf", + "revision": "aee7752d9c08ee4775e9b0091426d8410e8f6a89", + "license": "apache-2.0" + }, + "implementation_paths": [ + "ported_models/olmo_1b" + ], + "benchmark_config": "ported_models/llama_cpp_et/benchmarks/olmo_1b.json", + "recipe": "ported_models/olmo_1b/docs/RECIPE.md" +} diff --git a/ported_models/submissions/model_ports/openelm_1_1b.json b/ported_models/submissions/model_ports/openelm_1_1b.json new file mode 100644 index 00000000..1f21ea8a --- /dev/null +++ b/ported_models/submissions/model_ports/openelm_1_1b.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "track": "most_models_ported", + "benchmark_model": "openelm_1_1b", + "identity_id": "openelm", + "source": { + "repo": "apple/OpenELM-1_1B", + "revision": "ee559a10b14895dde9f8cfde3fdc77b3ff0dbc0f", + "license": "apple-sample-code-license" + }, + "implementation_paths": [ + "ported_models/openelm_1_1b" + ], + "benchmark_config": "ported_models/llama_cpp_et/benchmarks/openelm_1_1b.json", + "recipe": "ported_models/openelm_1_1b/docs/RECIPE.md" +}