From 4c0a3999e408770e0355b842507cb28874d57e2f Mon Sep 17 00:00:00 2001
From: Crutcher Dunnavant <crutcher@gmail.com>
Date: Mon, 20 Apr 2026 20:06:47 -0700
Subject: [PATCH 1/7] Add HuggingFace pretrained vocab provider and refactor
 bench data iteration

- Add `huggingface` feature and `HFVocabProvider` for loading tokenizers from HuggingFace Hub
- Refactor bench data methods to use `.values()` instead of `.iter()` for cleaner iteration
- Fix clippy warning by allowing explicit counter loop in datagym vocab.

This is broken still:
```
crutcher@HeatLamp:~/git/wordchipper$ echo 'hello world' | cargo run -p wordchipper-cli -- cat --model 'hf:Xenova/gpt2' --encode
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.16s
     Running `target/debug/wordchipper-cli cat --model 'hf:Xenova/gpt2' --encode`
Error: External("No pre-tokenizer")
```
---
 crates/wordchipper/Cargo.toml                 |   9 ++
 .../src/pretrained/huggingface/hf_factory.rs  | 116 ++++++++++++++++++
 .../src/pretrained/huggingface/mod.rs         |   3 +
 crates/wordchipper/src/pretrained/mod.rs      |   3 +
 4 files changed, 131 insertions(+)
 create mode 100644 crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
 create mode 100644 crates/wordchipper/src/pretrained/huggingface/mod.rs

diff --git a/crates/wordchipper/Cargo.toml b/crates/wordchipper/Cargo.toml
index 855bdfc9..fbc33b63 100644
--- a/crates/wordchipper/Cargo.toml
+++ b/crates/wordchipper/Cargo.toml
@@ -34,6 +34,7 @@ client = [
     "download",
     "datagym",
     "default-tls",
+    "huggingface",
 ]
 
 ## The download feature enables downloading vocabularies from the internet.
@@ -102,6 +103,12 @@ tracing = [
 testing = []
 
 
+## Enable loading pretrained huggingface modules.
+huggingface = [
+    "std",
+    "dep:tokenizers",
+]
+
 
 [dependencies]
 # macro packages.
@@ -141,6 +148,8 @@ rayon = { workspace = true, optional = true }
 # "tracing" feature deps:
 tracing = { workspace = true, optional = true }
 
+tokenizers = { workspace = true, features = ["http"], optional = true }
+
 
 [dev-dependencies]
 tempdir = { workspace = true }
diff --git a/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs b/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
new file mode 100644
index 00000000..9ffc0a75
--- /dev/null
+++ b/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
@@ -0,0 +1,116 @@
+use tokenizers::{
+    ModelWrapper::BPE,
+    PreTokenizerWrapper::Split,
+    pre_tokenizers::split::SplitPattern,
+    tokenizer::Tokenizer,
+};
+
+use crate::{
+    LabeledVocab,
+    UnifiedTokenVocab,
+    VocabDescription,
+    VocabQuery,
+    WCError,
+    WCResult,
+    alloc::sync::Arc,
+    prelude::*,
+    pretrained::factory::{
+        VocabProvider,
+        VocabProviderInventoryHook,
+    },
+    spanners::TextSpanningConfig,
+    support::{
+        regex::RegexPattern,
+        resources::ResourceLoader,
+    },
+    vocab::{
+        ByteMapVocab,
+        SpanMapVocab,
+        SpanTokenMap,
+    },
+};
+
+pub struct HFVocabProvider {}
+
+inventory::submit! {
+    VocabProviderInventoryHook::new(|| Arc::new(HFVocabProvider{}))
+}
+
+impl VocabProvider for HFVocabProvider {
+    fn name(&self) -> String {
+        "hf".to_string()
+    }
+
+    fn description(&self) -> String {
+        "HuggingFace vocabularies".to_string()
+    }
+
+    fn list_vocabs(&self) -> Vec<VocabDescription> {
+        vec![]
+    }
+
+    fn load_vocab(
+        &self,
+        query: &VocabQuery,
+        _loader: &mut dyn ResourceLoader,
+    ) -> WCResult<LabeledVocab<u32>> {
+        type T = u32;
+
+        let key = format!("{}/{}", query.path().unwrap(), query.name());
+        match Tokenizer::from_pretrained(&key, None) {
+            Ok(tok) => {
+                let span_config = if let Some(Split(split)) = tok.get_pre_tokenizer() {
+                    let pattern = match &split.pattern {
+                        SplitPattern::Regex(str) => RegexPattern::Adaptive(str.to_string()),
+                        _ => return Err(WCError::External("No regex pattern".to_string())),
+                    };
+
+                    TextSpanningConfig::from_pattern(pattern)
+                } else {
+                    return Err(WCError::External("No pre-tokenizer".to_string()));
+                };
+
+                if let BPE(bpe) = tok.get_model() {
+                    let vocab = bpe.get_vocab();
+
+                    let span_map: SpanTokenMap<T> = vocab
+                        .iter()
+                        .map(|(s, t)| (s.as_bytes().to_vec(), *t))
+                        .collect();
+
+                    let byte_map: ByteMapVocab<T> = if let Some(byte_tokens) = (0..256)
+                        .map(|b| {
+                            let k = format!("<{b:#04X}>");
+                            vocab.get(&k).copied()
+                        })
+                        .collect::<Option<Vec<T>>>()
+                    {
+                        ByteMapVocab::<T>::from_byte_to_token(&byte_tokens)
+                    } else {
+                        return Err(WCError::External(
+                            "Unable to translate: no byte map".to_string(),
+                        ));
+                    };
+
+                    let span_vocab = SpanMapVocab::<T>::new(byte_map, span_map)?;
+
+                    let vocab: Arc<UnifiedTokenVocab<T>> =
+                        Arc::new(UnifiedTokenVocab::from_span_vocab(span_config, span_vocab)?);
+
+                    let id = VocabQuery::new(Some("hf"), query.path(), query.name());
+
+                    let descr: VocabDescription = VocabDescription::new(
+                        id,
+                        &["hf", query.path().unwrap(), query.name()],
+                        "Model loaded from hf",
+                    );
+
+                    Ok(LabeledVocab::new(descr, vocab))
+                } else {
+                    Err(WCError::ResourceNotFound(query.to_string()))
+                }
+            }
+            Err(_) => Err(WCError::ResourceNotFound(query.to_string())),
+        }
+    }
+}
diff --git a/crates/wordchipper/src/pretrained/huggingface/mod.rs b/crates/wordchipper/src/pretrained/huggingface/mod.rs
new file mode 100644
index 00000000..3f74b586
--- /dev/null
+++ b/crates/wordchipper/src/pretrained/huggingface/mod.rs
@@ -0,0 +1,3 @@
+//! # `HuggingFace` Pretrained Models
+
+mod hf_factory;
diff --git a/crates/wordchipper/src/pretrained/mod.rs b/crates/wordchipper/src/pretrained/mod.rs
index acc66fa4..4fc52574 100644
--- a/crates/wordchipper/src/pretrained/mod.rs
+++ b/crates/wordchipper/src/pretrained/mod.rs
@@ -36,6 +36,9 @@
 pub mod factory;
 pub mod openai;
 
+#[cfg(feature = "huggingface")]
+pub mod huggingface;
+
 #[doc(inline)]
 pub use factory::{
     LabeledVocab,

From 643dec6ef07ed8889cc95c72ef2db762976ecf3c Mon Sep 17 00:00:00 2001
From: Crutcher Dunnavant <crutcher@gmail.com>
Date: Tue, 21 Apr 2026 17:48:42 -0700
Subject: [PATCH 2/7] Extend HF vocab provider to support byte-level BPE
 tokenizers

- Add support for ByteLevel and Sequence pre-tokenizers in addition to Split
- Implement proper byte-to-unicode mapping for byte-level BPE vocabularies
- Filter out special tokens when building span maps
- Extract and validate split patterns from pre-tokenizer configurations
- Add `bytes_char()` function implementing GPT-2 style byte-to-unicode conversion
- Use GPT2_PATTERN as default for bare ByteLevel pre-tokenizers
- Improve error messages with detailed context about unsupported configurations

```terminaloutput
crutcher@HeatLamp:~/git/wordchipper$ echo 'hello world' | cargo run -p wordchipper-cli -- cat --model 'cl100k_base' --encode
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.19s
     Running `target/debug/wordchipper-cli cat --model cl100k_base --encode`
15339 1917 198
crutcher@HeatLamp:~/git/wordchipper$ echo 'hello world' | cargo run -p wordchipper-cli -- cat --model 'hf:Xenova/text-embedding-ada-002' --encode
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.16s
     Running `target/debug/wordchipper-cli cat --model 'hf:Xenova/text-embedding-ada-002' --encode`
15339 1917 198
crutcher@HeatLamp:~/git/wordchipper$ echo 'hello world' | cargo run -p wordchipper-cli -- cat --model 'openai:gpt2' --encode
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.16s
     Running `target/debug/wordchipper-cli cat --model 'openai:gpt2' --encode`
31373 995 198
crutcher@HeatLamp:~/git/wordchipper$ echo 'hello world' | cargo run -p wordchipper-cli -- cat --model 'hf:Xenova/gpt2' --encode
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.16s
     Running `target/debug/wordchipper-cli cat --model 'hf:Xenova/gpt2' --encode`
31373 995 198
```
---
 .../src/pretrained/huggingface/hf_factory.rs  | 194 +++++++++++++-----
 1 file changed, 146 insertions(+), 48 deletions(-)

diff --git a/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs b/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
index 9ffc0a75..9a567c66 100644
--- a/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
+++ b/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
@@ -1,6 +1,11 @@
 use tokenizers::{
     ModelWrapper::BPE,
-    PreTokenizerWrapper::Split,
+    PreTokenizerWrapper,
+    PreTokenizerWrapper::{
+        ByteLevel,
+        Sequence,
+        Split,
+    },
     pre_tokenizers::split::SplitPattern,
     tokenizer::Tokenizer,
 };
@@ -11,6 +16,8 @@ use crate::{
     VocabDescription,
     VocabQuery,
     WCError,
+    WCHashMap,
+    WCHashSet,
     WCResult,
     alloc::sync::Arc,
     prelude::*,
@@ -54,63 +61,154 @@ impl VocabProvider for HFVocabProvider {
         query: &VocabQuery,
         _loader: &mut dyn ResourceLoader,
     ) -> WCResult<LabeledVocab<u32>> {
+        match query.schema() {
+            None => return Err(WCError::ResourceNotFound(query.to_string())),
+            Some(schema) => {
+                if schema != "hf" {
+                    return Err(WCError::ResourceNotFound(query.to_string()));
+                }
+            }
+        }
+
         type T = u32;
 
         let key = format!("{}/{}", query.path().unwrap(), query.name());
+        let id = format!("{}/{}", query.path().unwrap(), query.name());
+
         match Tokenizer::from_pretrained(&key, None) {
             Ok(tok) => {
-                let span_config = if let Some(Split(split)) = tok.get_pre_tokenizer() {
-                    let pattern = match &split.pattern {
-                        SplitPattern::Regex(str) => RegexPattern::Adaptive(str.to_string()),
-                        _ => return Err(WCError::External("No regex pattern".to_string())),
-                    };
-
-                    TextSpanningConfig::from_pattern(pattern)
-                } else {
-                    return Err(WCError::External("No pre-tokenizer".to_string()));
+                let pattern = extract_pattern(tok.get_pre_tokenizer())?;
+                let span_config = TextSpanningConfig::from_pattern(RegexPattern::Adaptive(pattern));
+
+                let BPE(bpe) = tok.get_model() else {
+                    return Err(WCError::External(
+                        format!("{} is not BPE compatible", id).to_string(),
+                    ));
                 };
 
-                if let BPE(bpe) = tok.get_model() {
-                    let vocab = bpe.get_vocab();
-
-                    let span_map: SpanTokenMap<T> = vocab
-                        .iter()
-                        .map(|(s, t)| (s.as_bytes().to_vec(), *t))
-                        .collect();
-
-                    let byte_map: ByteMapVocab<T> = if let Some(byte_tokens) = (0..256)
-                        .map(|b| {
-                            let k = format!("<{b:#04X}>");
-                            vocab.get(&k).copied()
-                        })
-                        .collect::<Option<Vec<T>>>()
-                    {
-                        ByteMapVocab::<T>::from_byte_to_token(&byte_tokens)
-                    } else {
-                        return Err(WCError::External(
-                            "Unable to translate: no byte map".to_string(),
-                        ));
-                    };
-
-                    let span_vocab = SpanMapVocab::<T>::new(byte_map, span_map)?;
-
-                    let vocab: Arc<UnifiedTokenVocab<T>> =
-                        Arc::new(UnifiedTokenVocab::from_span_vocab(span_config, span_vocab)?);
-
-                    let id = VocabQuery::new(Some("hf"), query.path(), query.name());
-
-                    let descr: VocabDescription = VocabDescription::new(
-                        id,
-                        &["hf", query.path().unwrap(), query.name()],
-                        "Model loaded from hf",
-                    );
-
-                    Ok(LabeledVocab::new(descr, vocab))
-                } else {
-                    Err(WCError::ResourceNotFound(query.to_string()))
+                // TODO: Add support for unknown token.
+                if let Some(unk) = bpe.get_unk_token() {
+                    return Err(WCError::External(format!("BPE has unk_token {unk:?}")));
                 }
+
+                let vocab = bpe.get_vocab();
+
+                let specials: WCHashSet<u32> =
+                    tok.get_added_tokens_decoder().keys().copied().collect();
+
+                // Forward and inverse bytes_to_unicode maps.
+                let b2c = bytes_char();
+                let c2b: WCHashMap<char, u8> = b2c.iter().map(|(&b, &c)| (c, b)).collect();
+
+                // Span map: decode every non-special vocab string back to bytes.
+                let mut span_map: SpanTokenMap<T> = SpanTokenMap::default();
+                for (s, id) in &vocab {
+                    if specials.contains(id) {
+                        continue;
+                    }
+                    let mut bytes = Vec::with_capacity(s.len());
+                    for ch in s.chars() {
+                        match c2b.get(&ch) {
+                            Some(&b) => bytes.push(b),
+                            None => {
+                                return Err(WCError::External(format!(
+                                    "token {s:?} (id {id}) has non-byte-level codepoint {ch:?}"
+                                )));
+                            }
+                        }
+                    }
+                    span_map.insert(bytes, (*id));
+                }
+
+                // Byte map: the single-char string for each byte must resolve in the vocab.
+                let byte_tokens: Vec<T> = (0u8..=255)
+                    .map(|b| {
+                        let key: String = std::iter::once(b2c[&b]).collect();
+                        vocab.get(&key).copied().ok_or(b)
+                    })
+                    .collect::<Result<Vec<_>, _>>()
+                    .map_err(|b| WCError::External(format!("missing byte token for 0x{b:02x}")))?;
+
+                let byte_map = ByteMapVocab::<T>::from_byte_to_token(&byte_tokens);
+                let span_vocab = SpanMapVocab::<T>::new(byte_map, span_map)?;
+
+                let vocab: Arc<UnifiedTokenVocab<T>> =
+                    Arc::new(UnifiedTokenVocab::from_span_vocab(span_config, span_vocab)?);
+
+                let id = VocabQuery::new(Some("hf"), query.path(), query.name());
+
+                let descr: VocabDescription = VocabDescription::new(
+                    id,
+                    &["hf", query.path().unwrap(), query.name()],
+                    "Model loaded from hf",
+                );
+
+                Ok(LabeledVocab::new(descr, vocab))
             }
             Err(_) => Err(WCError::ResourceNotFound(query.to_string())),
         }
     }
 }
+// GPT-2 / r50k-style default, used when pretokenizer is a bare ByteLevel.
+const GPT2_PATTERN: &str =
+    r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+";
+
+fn extract_pattern(pt: Option<&PreTokenizerWrapper>) -> Result<String, WCError> {
+    fn split_regex(s: &tokenizers::pre_tokenizers::split::Split) -> Result<String, WCError> {
+        match &s.pattern {
+            SplitPattern::Regex(r) => Ok(r.clone()),
+            _ => Err(WCError::External("Split without Regex pattern".into())),
+        }
+    }
+    match pt {
+        Some(Split(s)) => split_regex(s),
+        Some(ByteLevel(_)) => Ok(GPT2_PATTERN.to_string()),
+        Some(Sequence(seq)) => {
+            let mut found = None;
+            for sub in seq.as_ref() {
+                match &sub {
+                    Split(s) => {
+                        if found.is_some() {
+                            return Err(WCError::External("Sequence has multiple Splits".into()));
+                        }
+                        found = Some(split_regex(s)?);
+                    }
+                    ByteLevel(_) => {} // sibling byte-encoder, fine
+                    _ => return Err(WCError::External("unsupported member in Sequence".into())),
+                }
+            }
+            found.ok_or_else(|| WCError::External("Sequence has no Split regex".into()))
+        }
+        Some(_) => Err(WCError::External("unsupported pre-tokenizer".into())),
+        None => Err(WCError::External("no pre-tokenizer".into())),
+    }
+}
+
+/// Converts bytes to unicode characters.
+/// See <https://github.com/openai/gpt-2/blob/master/src/encoder.py#L9>
+fn bytes_char() -> WCHashMap<u8, char> {
+    let mut bs: Vec<u8> = vec![];
+    bs.extend(b'!'..=b'~');
+    bs.extend(b'\xA1'..=b'\xAC');
+    bs.extend(b'\xAE'..=b'\xFF');
+
+    let mut cs: Vec<u32> = bs.iter().map(|i| *i as u32).collect();
+    let mut n = 0;
+
+    for b in 0..=255u8 {
+        if !bs.contains(&b) {
+            bs.push(b);
+            cs.push(u32::pow(2, 8) + n);
+            n += 1;
+        }
+    }
+
+    // Safety: cs contains all values from bs (between 0 and 255),
+    // and some values of value 2⁸ + n, where n is between 0 and 255. This is
+    // between 255 and 512. Both ranges are valid UTF-32 values (which is fully
+    // saturated until 0xD000)
+    bs.into_iter()
+        .zip(cs)
+        .map(|(f, t)| (f, unsafe { std::char::from_u32_unchecked(t) }))
+        .collect()
+}

From 366dd4e5d6a4f959c74476d69fc49719bbbc51d8 Mon Sep 17 00:00:00 2001
From: Crutcher Dunnavant <crutcher@gmail.com>
Date: Tue, 21 Apr 2026 18:02:27 -0700
Subject: [PATCH 3/7] Make hf factory a better pass-through factory.

Will now search hf for pretrained models if there is no schema.
---
 .../src/pretrained/huggingface/hf_factory.rs  | 168 +++++++++---------
 1 file changed, 86 insertions(+), 82 deletions(-)

diff --git a/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs b/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
index 9a567c66..03a8dd69 100644
--- a/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
+++ b/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
@@ -37,6 +37,76 @@ use crate::{
     },
 };
 
+// GPT-2 / r50k-style default, used when pretokenizer is a bare ByteLevel.
+const GPT2_PATTERN: &str =
+    r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+";
+
+fn extract_pattern(pt: Option<&PreTokenizerWrapper>) -> Result<String, WCError> {
+    fn split_regex(s: &tokenizers::pre_tokenizers::split::Split) -> Result<String, WCError> {
+        match &s.pattern {
+            SplitPattern::Regex(r) => Ok(r.clone()),
+            _ => Err(WCError::External("Split without Regex pattern".into())),
+        }
+    }
+    match pt {
+        Some(Split(s)) => split_regex(s),
+        Some(ByteLevel(_)) => Ok(GPT2_PATTERN.to_string()),
+        Some(Sequence(seq)) => {
+            let mut found = None;
+            for sub in seq.as_ref() {
+                match &sub {
+                    Split(s) => {
+                        if found.is_some() {
+                            return Err(WCError::External("Sequence has multiple Splits".into()));
+                        }
+                        found = Some(split_regex(s)?);
+                    }
+                    ByteLevel(_) => {} // sibling byte-encoder, fine
+                    _ => return Err(WCError::External("unsupported member in Sequence".into())),
+                }
+            }
+            found.ok_or_else(|| WCError::External("Sequence has no Split regex".into()))
+        }
+        Some(_) => Err(WCError::External("unsupported pre-tokenizer".into())),
+        None => Err(WCError::External("no pre-tokenizer".into())),
+    }
+}
+
+/// Converts bytes to Unicode characters.
+/// See <https://github.com/openai/gpt-2/blob/master/src/encoder.py#L9>
+///
+/// This is from tokenizers; but is private in that crate.
+///
+/// TODO: Workout what this is doing, relative to the bytemap.
+/// This seems to be some default map for gpt2; and might be shared
+/// with the `BytMap` code for loading datagym.
+fn bytes_char() -> WCHashMap<u8, char> {
+    let mut bs: Vec<u8> = vec![];
+    bs.extend(b'!'..=b'~');
+    bs.extend(b'\xA1'..=b'\xAC');
+    bs.extend(b'\xAE'..=b'\xFF');
+
+    let mut cs: Vec<u32> = bs.iter().map(|i| *i as u32).collect();
+    let mut n = 0;
+
+    for b in 0..=255u8 {
+        if !bs.contains(&b) {
+            bs.push(b);
+            cs.push(u32::pow(2, 8) + n);
+            n += 1;
+        }
+    }
+
+    // Safety: cs contains all values from bs (between 0 and 255),
+    // and some values of value 2⁸ + n, where n is between 0 and 255. This is
+    // between 255 and 512. Both ranges are valid UTF-32 values (which is fully
+    // saturated until 0xD000)
+    bs.into_iter()
+        .zip(cs)
+        .map(|(f, t)| (f, unsafe { std::char::from_u32_unchecked(t) }))
+        .collect()
+}
+
 pub struct HFVocabProvider {}
 
 inventory::submit! {
@@ -61,28 +131,22 @@ impl VocabProvider for HFVocabProvider {
         query: &VocabQuery,
         _loader: &mut dyn ResourceLoader,
     ) -> WCResult<LabeledVocab<u32>> {
-        match query.schema() {
-            None => return Err(WCError::ResourceNotFound(query.to_string())),
-            Some(schema) => {
-                if schema != "hf" {
-                    return Err(WCError::ResourceNotFound(query.to_string()));
-                }
-            }
+        if let Some(schema) = query.schema()
+            && schema != "hf"
+        {
+            return Err(WCError::ResourceNotFound(query.to_string()));
         }
 
         type T = u32;
 
-        let key = format!("{}/{}", query.path().unwrap(), query.name());
-        let id = format!("{}/{}", query.path().unwrap(), query.name());
-
-        match Tokenizer::from_pretrained(&key, None) {
+        match Tokenizer::from_pretrained(&query.clone().with_schema(None).to_string(), None) {
             Ok(tok) => {
                 let pattern = extract_pattern(tok.get_pre_tokenizer())?;
                 let span_config = TextSpanningConfig::from_pattern(RegexPattern::Adaptive(pattern));
 
                 let BPE(bpe) = tok.get_model() else {
                     return Err(WCError::External(
-                        format!("{} is not BPE compatible", id).to_string(),
+                        format!("{} is not BPE compatible", query).to_string(),
                     ));
                 };
 
@@ -117,7 +181,7 @@ impl VocabProvider for HFVocabProvider {
                             }
                         }
                     }
-                    span_map.insert(bytes, (*id));
+                    span_map.insert(bytes, *id);
                 }
 
                 // Byte map: the single-char string for each byte must resolve in the vocab.
@@ -135,13 +199,16 @@ impl VocabProvider for HFVocabProvider {
                 let vocab: Arc<UnifiedTokenVocab<T>> =
                     Arc::new(UnifiedTokenVocab::from_span_vocab(span_config, span_vocab)?);
 
-                let id = VocabQuery::new(Some("hf"), query.path(), query.name());
+                let id = query.clone().with_schema(Some("hf"));
+
+                let mut context = vec!["hf"];
+                if query.path().is_some() {
+                    context.push(query.path().unwrap());
+                }
+                context.push(query.name());
 
-                let descr: VocabDescription = VocabDescription::new(
-                    id,
-                    &["hf", query.path().unwrap(), query.name()],
-                    "Model loaded from hf",
-                );
+                let descr: VocabDescription =
+                    VocabDescription::new(id, &context, "Model loaded from hf");
 
                 Ok(LabeledVocab::new(descr, vocab))
             }
@@ -149,66 +216,3 @@ impl VocabProvider for HFVocabProvider {
         }
     }
 }
-// GPT-2 / r50k-style default, used when pretokenizer is a bare ByteLevel.
-const GPT2_PATTERN: &str =
-    r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+";
-
-fn extract_pattern(pt: Option<&PreTokenizerWrapper>) -> Result<String, WCError> {
-    fn split_regex(s: &tokenizers::pre_tokenizers::split::Split) -> Result<String, WCError> {
-        match &s.pattern {
-            SplitPattern::Regex(r) => Ok(r.clone()),
-            _ => Err(WCError::External("Split without Regex pattern".into())),
-        }
-    }
-    match pt {
-        Some(Split(s)) => split_regex(s),
-        Some(ByteLevel(_)) => Ok(GPT2_PATTERN.to_string()),
-        Some(Sequence(seq)) => {
-            let mut found = None;
-            for sub in seq.as_ref() {
-                match &sub {
-                    Split(s) => {
-                        if found.is_some() {
-                            return Err(WCError::External("Sequence has multiple Splits".into()));
-                        }
-                        found = Some(split_regex(s)?);
-                    }
-                    ByteLevel(_) => {} // sibling byte-encoder, fine
-                    _ => return Err(WCError::External("unsupported member in Sequence".into())),
-                }
-            }
-            found.ok_or_else(|| WCError::External("Sequence has no Split regex".into()))
-        }
-        Some(_) => Err(WCError::External("unsupported pre-tokenizer".into())),
-        None => Err(WCError::External("no pre-tokenizer".into())),
-    }
-}
-
-/// Converts bytes to unicode characters.
-/// See <https://github.com/openai/gpt-2/blob/master/src/encoder.py#L9>
-fn bytes_char() -> WCHashMap<u8, char> {
-    let mut bs: Vec<u8> = vec![];
-    bs.extend(b'!'..=b'~');
-    bs.extend(b'\xA1'..=b'\xAC');
-    bs.extend(b'\xAE'..=b'\xFF');
-
-    let mut cs: Vec<u32> = bs.iter().map(|i| *i as u32).collect();
-    let mut n = 0;
-
-    for b in 0..=255u8 {
-        if !bs.contains(&b) {
-            bs.push(b);
-            cs.push(u32::pow(2, 8) + n);
-            n += 1;
-        }
-    }
-
-    // Safety: cs contains all values from bs (between 0 and 255),
-    // and some values of value 2⁸ + n, where n is between 0 and 255. This is
-    // between 255 and 512. Both ranges are valid UTF-32 values (which is fully
-    // saturated until 0xD000)
-    bs.into_iter()
-        .zip(cs)
-        .map(|(f, t)| (f, unsafe { std::char::from_u32_unchecked(t) }))
-        .collect()
-}

From 7ed89a87ae590071edd76274d91047549e7b349a Mon Sep 17 00:00:00 2001
From: Crutcher Dunnavant <crutcher@gmail.com>
Date: Tue, 21 Apr 2026 18:55:50 -0700
Subject: [PATCH 4/7] Add `VocabQuery::to_context()` method and refactor HF
 vocab provider

- Add `VocabQuery::to_context()` to build cache context from schema, path, and name
- Make `VocabDescription::new()` generic over context and description string types
- Extract `vocab_from_hf_tokenizer()` as reusable conversion function
- Add special token handling to span_config instead of filtering them out
- Fix ByteLevel pre-tokenizer to require `use_regex=true` and use OA_GPT2_PATTERN
- Update HF vocab description context to use `to_context()` helper
- Add tests for `to_context()` with various query formats
---
 .../pretrained/factory/vocab_description.rs   |  14 +-
 .../src/pretrained/factory/vocab_query.rs     |  28 ++-
 .../src/pretrained/huggingface/hf_factory.rs  | 170 ++++++++++--------
 3 files changed, 133 insertions(+), 79 deletions(-)

diff --git a/crates/wordchipper/src/pretrained/factory/vocab_description.rs b/crates/wordchipper/src/pretrained/factory/vocab_description.rs
index 9e7fedbc..a8cf2782 100644
--- a/crates/wordchipper/src/pretrained/factory/vocab_description.rs
+++ b/crates/wordchipper/src/pretrained/factory/vocab_description.rs
@@ -21,20 +21,24 @@ pub struct VocabDescription {
 
 impl VocabDescription {
     /// Build a new vocabulary description.
-    pub fn new<Q>(
+    pub fn new<Q, C, D>(
         id: Q,
-        context: &[&str],
-        description: &str,
+        context: &[C],
+        description: D,
     ) -> Self
     where
         Q: Into<VocabQuery>,
+        C: AsRef<str>,
+        D: AsRef<str>,
     {
         let id = id.into();
+        let context = context.iter().map(|c| c.as_ref().to_string()).collect();
+        let description = description.as_ref().to_string();
 
         Self {
             id,
-            context: context.iter().map(|&s| s.to_string()).collect(),
-            description: description.to_string(),
+            context,
+            description,
         }
     }
 
diff --git a/crates/wordchipper/src/pretrained/factory/vocab_query.rs b/crates/wordchipper/src/pretrained/factory/vocab_query.rs
index c33af521..1efbdb8b 100644
--- a/crates/wordchipper/src/pretrained/factory/vocab_query.rs
+++ b/crates/wordchipper/src/pretrained/factory/vocab_query.rs
@@ -66,7 +66,7 @@ impl Display for VocabQuery {
 }
 
 impl VocabQuery {
-    /// Build a new query from structure.
+    /// Build a new query.
     pub fn new(
         schema: Option<&str>,
         path: Option<&str>,
@@ -174,6 +174,19 @@ impl VocabQuery {
         }
         query.name() == self.name()
     }
+
+    /// Build a cache context for this query.
+    pub fn to_context(&self) -> Vec<String> {
+        let mut context = Vec::new();
+        if let Some(schema) = self.schema() {
+            context.push(schema.to_string());
+        }
+        if let Some(path) = self.path() {
+            context.extend(path.split('/').map(|p| p.to_string()));
+        }
+        context.push(self.name().to_string());
+        context
+    }
 }
 
 #[cfg(test)]
@@ -202,6 +215,19 @@ mod tests {
             VocabQuery::new(Some("xyz"), Some("foo/bar"), "vocab_name")
         );
     }
+
+    #[test]
+    fn test_to_context() {
+        let q = VocabQuery::from_str("vocab_name").unwrap();
+        assert_eq!(q.to_context(), vec!["vocab_name"]);
+
+        let q = VocabQuery::from_str("foo/bar/vocab_name").unwrap();
+        assert_eq!(q.to_context(), vec!["foo", "bar", "vocab_name"]);
+
+        let q = VocabQuery::from_str("xyz:foo/bar/vocab_name").unwrap();
+        assert_eq!(q.to_context(), vec!["xyz", "foo", "bar", "vocab_name"]);
+    }
+
     #[test]
     fn test_vocab_query_with_schema() {
         let query = VocabQuery::new(None, None, "vocab_name").with_schema(Some("provider"));
diff --git a/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs b/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
index 03a8dd69..0b851ed1 100644
--- a/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
+++ b/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
@@ -14,6 +14,7 @@ use crate::{
     LabeledVocab,
     UnifiedTokenVocab,
     VocabDescription,
+    VocabIndex,
     VocabQuery,
     WCError,
     WCHashMap,
@@ -21,9 +22,12 @@ use crate::{
     WCResult,
     alloc::sync::Arc,
     prelude::*,
-    pretrained::factory::{
-        VocabProvider,
-        VocabProviderInventoryHook,
+    pretrained::{
+        factory::{
+            VocabProvider,
+            VocabProviderInventoryHook,
+        },
+        openai::OA_GPT2_PATTERN,
     },
     spanners::TextSpanningConfig,
     support::{
@@ -37,20 +41,19 @@ use crate::{
     },
 };
 
-// GPT-2 / r50k-style default, used when pretokenizer is a bare ByteLevel.
-const GPT2_PATTERN: &str =
-    r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+";
-
-fn extract_pattern(pt: Option<&PreTokenizerWrapper>) -> Result<String, WCError> {
-    fn split_regex(s: &tokenizers::pre_tokenizers::split::Split) -> Result<String, WCError> {
+fn extract_pattern(pt: Option<&PreTokenizerWrapper>) -> Result<RegexPattern, WCError> {
+    fn split_regex(s: &tokenizers::pre_tokenizers::split::Split) -> Result<RegexPattern, WCError> {
         match &s.pattern {
-            SplitPattern::Regex(r) => Ok(r.clone()),
+            SplitPattern::Regex(r) => Ok(r.clone().into()),
             _ => Err(WCError::External("Split without Regex pattern".into())),
         }
     }
     match pt {
         Some(Split(s)) => split_regex(s),
-        Some(ByteLevel(_)) => Ok(GPT2_PATTERN.to_string()),
+        Some(ByteLevel(bl)) if bl.use_regex => Ok(OA_GPT2_PATTERN.into()),
+        Some(ByteLevel(_)) => Err(WCError::External(
+            "ByteLevel with use_regex=false has no splitting regex".into(),
+        )),
         Some(Sequence(seq)) => {
             let mut found = None;
             for sub in seq.as_ref() {
@@ -107,6 +110,84 @@ fn bytes_char() -> WCHashMap<u8, char> {
         .collect()
 }
 
+/// Attempt to convert a `HuggingFace` tokenizer to a `WordChipper` vocabulary.
+pub fn vocab_from_hf_tokenizer(tok: &Tokenizer) -> WCResult<Arc<UnifiedTokenVocab<u32>>> {
+    type T = u32;
+
+    let pattern = extract_pattern(tok.get_pre_tokenizer())?;
+    let mut span_config: TextSpanningConfig<T> = TextSpanningConfig::from_pattern(pattern);
+
+    let BPE(bpe) = tok.get_model() else {
+        return Err(WCError::External(
+            "Tokenizer is not BPE compatible".to_string(),
+        ));
+    };
+
+    // TODO: Add support for unknown token.
+    if let Some(unk) = bpe.get_unk_token() {
+        return Err(WCError::External(format!("BPE has unk_token {unk:?}")));
+    }
+
+    let hf_vocab = bpe.get_vocab();
+
+    let special_tokens: WCHashSet<u32> = tok.get_added_tokens_decoder().keys().copied().collect();
+
+    // Forward and inverse bytes_to_unicode maps.
+    let b2c = bytes_char();
+    let c2b: WCHashMap<char, u8> = b2c.iter().map(|(&b, &c)| (c, b)).collect();
+
+    // Span map: decode every non-special vocab string back to bytes.
+    let mut span_map: SpanTokenMap<T> = SpanTokenMap::default();
+    for (s, id) in &hf_vocab {
+        if special_tokens.contains(id) {
+            span_config.specials_mut().add_str_word(s, *id);
+        } else {
+            let mut bytes = Vec::with_capacity(s.len());
+            for ch in s.chars() {
+                match c2b.get(&ch) {
+                    Some(&b) => bytes.push(b),
+                    None => {
+                        return Err(WCError::External(format!(
+                            "token {s:?} (id {id}) has non-byte-level codepoint {ch:?}"
+                        )));
+                    }
+                }
+            }
+            span_map.insert(bytes, *id);
+        }
+    }
+
+    assert_eq!(span_config.specials().len(), special_tokens.len());
+
+    // Byte map: the single-char string for each byte must resolve in the vocab.
+    let byte_tokens: Vec<T> = (0u8..=255)
+        .map(|b| {
+            let key: String = std::iter::once(b2c[&b]).collect();
+            hf_vocab.get(&key).copied().ok_or(b)
+        })
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|b| WCError::External(format!("missing byte token for 0x{b:02x}")))?;
+
+    let byte_map = ByteMapVocab::<T>::from_byte_to_token(&byte_tokens);
+    let span_vocab = SpanMapVocab::<T>::new(byte_map, span_map)?;
+
+    let expected_len = span_vocab.len() + span_config.specials().len();
+
+    let vocab: Arc<UnifiedTokenVocab<T>> =
+        Arc::new(UnifiedTokenVocab::from_span_vocab(span_config, span_vocab)?);
+
+    // TODO: should `vocab.len()` include the special len()?
+    if vocab.len() + vocab.special_vocab().len() != expected_len {
+        return Err(WCError::External(format!(
+            "Expected {} tokens, got {}",
+            expected_len,
+            vocab.len()
+        )));
+    }
+
+    Ok(vocab)
+}
+
 pub struct HFVocabProvider {}
 
 inventory::submit! {
@@ -137,69 +218,9 @@ impl VocabProvider for HFVocabProvider {
             return Err(WCError::ResourceNotFound(query.to_string()));
         }
 
-        type T = u32;
-
-        match Tokenizer::from_pretrained(&query.clone().with_schema(None).to_string(), None) {
+        match Tokenizer::from_pretrained(query.clone().with_schema(None).to_string(), None) {
             Ok(tok) => {
-                let pattern = extract_pattern(tok.get_pre_tokenizer())?;
-                let span_config = TextSpanningConfig::from_pattern(RegexPattern::Adaptive(pattern));
-
-                let BPE(bpe) = tok.get_model() else {
-                    return Err(WCError::External(
-                        format!("{} is not BPE compatible", query).to_string(),
-                    ));
-                };
-
-                // TODO: Add support for unknown token.
-                if let Some(unk) = bpe.get_unk_token() {
-                    return Err(WCError::External(format!("BPE has unk_token {unk:?}")));
-                }
-
-                let vocab = bpe.get_vocab();
-
-                let specials: WCHashSet<u32> =
-                    tok.get_added_tokens_decoder().keys().copied().collect();
-
-                // Forward and inverse bytes_to_unicode maps.
-                let b2c = bytes_char();
-                let c2b: WCHashMap<char, u8> = b2c.iter().map(|(&b, &c)| (c, b)).collect();
-
-                // Span map: decode every non-special vocab string back to bytes.
-                let mut span_map: SpanTokenMap<T> = SpanTokenMap::default();
-                for (s, id) in &vocab {
-                    if specials.contains(id) {
-                        continue;
-                    }
-                    let mut bytes = Vec::with_capacity(s.len());
-                    for ch in s.chars() {
-                        match c2b.get(&ch) {
-                            Some(&b) => bytes.push(b),
-                            None => {
-                                return Err(WCError::External(format!(
-                                    "token {s:?} (id {id}) has non-byte-level codepoint {ch:?}"
-                                )));
-                            }
-                        }
-                    }
-                    span_map.insert(bytes, *id);
-                }
-
-                // Byte map: the single-char string for each byte must resolve in the vocab.
-                let byte_tokens: Vec<T> = (0u8..=255)
-                    .map(|b| {
-                        let key: String = std::iter::once(b2c[&b]).collect();
-                        vocab.get(&key).copied().ok_or(b)
-                    })
-                    .collect::<Result<Vec<_>, _>>()
-                    .map_err(|b| WCError::External(format!("missing byte token for 0x{b:02x}")))?;
-
-                let byte_map = ByteMapVocab::<T>::from_byte_to_token(&byte_tokens);
-                let span_vocab = SpanMapVocab::<T>::new(byte_map, span_map)?;
-
-                let vocab: Arc<UnifiedTokenVocab<T>> =
-                    Arc::new(UnifiedTokenVocab::from_span_vocab(span_config, span_vocab)?);
-
-                let id = query.clone().with_schema(Some("hf"));
+                let vocab = vocab_from_hf_tokenizer(&tok)?;
 
                 let mut context = vec!["hf"];
                 if query.path().is_some() {
@@ -207,6 +228,9 @@ impl VocabProvider for HFVocabProvider {
                 }
                 context.push(query.name());
 
+                let id = query.clone().with_schema(Some("hf"));
+                let context = id.to_context();
+
                 let descr: VocabDescription =
                     VocabDescription::new(id, &context, "Model loaded from hf");
 

From b0e9475f3976d9a160cb989e89eae75bd8f5a1bc Mon Sep 17 00:00:00 2001
From: Crutcher Dunnavant <crutcher@gmail.com>
Date: Tue, 21 Apr 2026 20:51:12 -0700
Subject: [PATCH 5/7] Partial work on Qwen3.5.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is broken; the vocab seems to have aggregate tokens with no prior piecewise merges that are not special tokens. Which wordchipper enforces is never the case.

I need to investigate if this is true for this dataset; and explore what needs to change about wordchipper to handle this.

It may be that we need to support a situation where there multi-byte strings in the span map that are not in the byte map, but also *products* in the merge map (so they can only be produced by exact-match lookup, never by merges); but can still be members of merge pairs.

```terminaloutput
echo 'hello world' | RUST_BACKTRACE=1 cargo run -p wordchipper-cli -- cat --model Qwen/Qwen3.5-9B --encode
    Blocking waiting for file lock on artifact directory
   Compiling wordchipper v0.9.1 (/home/crutcher/git/wordchipper/crates/wordchipper)
   Compiling wordchipper-training v0.9.1 (/home/crutcher/git/wordchipper/crates/wordchipper-training)
   Compiling wordchipper-cli-util v0.9.1 (/home/crutcher/git/wordchipper/crates/wordchipper-cli-util)
   Compiling wordchipper-cli v0.9.1 (/home/crutcher/git/wordchipper/crates/wordchipper-cli)
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 15.19s
     Running `target/debug/wordchipper-cli cat --model Qwen/Qwen3.5-9B --encode`
Debug: Some(("ĠаÑįÑĢо", 157513))
Debug: {
    248067: AddedToken {
        content: "</tool_response>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248064: AddedToken {
        content: "<|repo_name|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248046: AddedToken {
        content: "<|im_end|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248059: AddedToken {
        content: "</tool_call>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248054: AddedToken {
        content: "<|vision_end|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248044: AddedToken {
        content: "<|endoftext|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248058: AddedToken {
        content: "<tool_call>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248049: AddedToken {
        content: "<|box_start|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248057: AddedToken {
        content: "<|video_pad|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248056: AddedToken {
        content: "<|image_pad|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248062: AddedToken {
        content: "<|fim_suffix|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248050: AddedToken {
        content: "<|box_end|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248055: AddedToken {
        content: "<|vision_pad|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248048: AddedToken {
        content: "<|object_ref_end|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248045: AddedToken {
        content: "<|im_start|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248066: AddedToken {
        content: "<tool_response>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248068: AddedToken {
        content: "<think>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248069: AddedToken {
        content: "</think>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248052: AddedToken {
        content: "<|quad_end|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248053: AddedToken {
        content: "<|vision_start|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248060: AddedToken {
        content: "<|fim_prefix|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248047: AddedToken {
        content: "<|object_ref_start|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248051: AddedToken {
        content: "<|quad_start|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248061: AddedToken {
        content: "<|fim_middle|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248063: AddedToken {
        content: "<|fim_pad|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248065: AddedToken {
        content: "<|file_sep|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
}

thread 'main' (852780) panicked at crates/wordchipper/src/vocab/span_vocab.rs:247:51:
called `Result::unwrap()` on an `Err` value: VocabConflict("Pair (157513, 157853) -> 229601 parent 157513 is not defined")
stack backtrace:
   0: __rustc::rust_begin_unwind
             at /rustc/59807616e1fa2540724bfbac14d7976d7e4a3860/library/std/src/panicking.rs:689:5
   1: core::panicking::panic_fmt
             at /rustc/59807616e1fa2540724bfbac14d7976d7e4a3860/library/core/src/panicking.rs:80:14
   2: core::result::unwrap_failed
             at /rustc/59807616e1fa2540724bfbac14d7976d7e4a3860/library/core/src/result.rs:1867:5
   3: core::result::Result<T,E>::unwrap
             at /home/crutcher/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/result.rs:1233:23
   4: wordchipper::vocab::span_vocab::SpanMapVocab<T>::to_pair_vocab
             at ./crates/wordchipper/src/vocab/span_vocab.rs:247:51
   5: wordchipper::vocab::unified_vocab::UnifiedTokenVocab<T>::from_span_vocab
             at ./crates/wordchipper/src/vocab/unified_vocab.rs:85:37
   6: wordchipper::pretrained::huggingface::hf_factory::vocab_from_hf_tokenizer
             at ./crates/wordchipper/src/pretrained/huggingface/hf_factory.rs:199:18
   7: <wordchipper::pretrained::huggingface::hf_factory::HFVocabProvider as wordchipper::pretrained::factory::vocab_provider::VocabProvider>::load_vocab
             at ./crates/wordchipper/src/pretrained/huggingface/hf_factory.rs:245:29
   8: wordchipper::pretrained::factory::vocab_factory::VocabFactory::load_vocab
             at ./crates/wordchipper/src/pretrained/factory/vocab_factory.rs:229:28
   9: wordchipper::pretrained::factory::vocab_factory::load_vocab::{{closure}}
             at ./crates/wordchipper/src/pretrained/factory/vocab_factory.rs:99:55
  10: wordchipper::pretrained::factory::vocab_factory::with_vocab_factory
             at ./crates/wordchipper/src/pretrained/factory/vocab_factory.rs:76:5
  11: wordchipper::pretrained::factory::vocab_factory::load_vocab
             at ./crates/wordchipper/src/pretrained/factory/vocab_factory.rs:99:5
  12: wordchipper_cli_util::model_selector::ModelSelectorArgs::load_vocab
             at ./crates/wordchipper-cli-util/src/model_selector.rs:29:21
  13: wordchipper_cli_util::model_selector::ModelSelectorArgs::load_tokenizer
             at ./crates/wordchipper-cli-util/src/model_selector.rs:41:26
  14: wordchipper_cli::commands::cat_cmd::CatArgs::run
             at ./crates/wordchipper-cli/src/commands/cat_cmd.rs:51:45
  15: wordchipper_cli::Commands::run
             at ./crates/wordchipper-cli/src/main.rs:44:39
  16: wordchipper_cli::main
             at ./crates/wordchipper-cli/src/main.rs:18:18
  17: core::ops::function::FnOnce::call_once
             at /home/crutcher/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.

```terminaloutput
$ echo 'hello world' | RUST_BACKTRACE=1 cargo run -p wordchipper-cli -- cat --model Qwen/Qwen3.5-9B --encode
    Blocking waiting for file lock on artifact directory
   Compiling wordchipper v0.9.1 (/home/crutcher/git/wordchipper/crates/wordchipper)
   Compiling wordchipper-training v0.9.1 (/home/crutcher/git/wordchipper/crates/wordchipper-training)
   Compiling wordchipper-cli-util v0.9.1 (/home/crutcher/git/wordchipper/crates/wordchipper-cli-util)
   Compiling wordchipper-cli v0.9.1 (/home/crutcher/git/wordchipper/crates/wordchipper-cli)
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 15.19s
     Running `target/debug/wordchipper-cli cat --model Qwen/Qwen3.5-9B --encode`
Debug: Some(("ĠаÑįÑĢо", 157513))
Debug: {
    248067: AddedToken {
        content: "</tool_response>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248064: AddedToken {
        content: "<|repo_name|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248046: AddedToken {
        content: "<|im_end|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248059: AddedToken {
        content: "</tool_call>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248054: AddedToken {
        content: "<|vision_end|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248044: AddedToken {
        content: "<|endoftext|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248058: AddedToken {
        content: "<tool_call>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248049: AddedToken {
        content: "<|box_start|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248057: AddedToken {
        content: "<|video_pad|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248056: AddedToken {
        content: "<|image_pad|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248062: AddedToken {
        content: "<|fim_suffix|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248050: AddedToken {
        content: "<|box_end|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248055: AddedToken {
        content: "<|vision_pad|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248048: AddedToken {
        content: "<|object_ref_end|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248045: AddedToken {
        content: "<|im_start|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248066: AddedToken {
        content: "<tool_response>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248068: AddedToken {
        content: "<think>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248069: AddedToken {
        content: "</think>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248052: AddedToken {
        content: "<|quad_end|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248053: AddedToken {
        content: "<|vision_start|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248060: AddedToken {
        content: "<|fim_prefix|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248047: AddedToken {
        content: "<|object_ref_start|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248051: AddedToken {
        content: "<|quad_start|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: true,
    },
    248061: AddedToken {
        content: "<|fim_middle|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248063: AddedToken {
        content: "<|fim_pad|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
    248065: AddedToken {
        content: "<|file_sep|>",
        single_word: false,
        lstrip: false,
        rstrip: false,
        normalized: false,
        special: false,
    },
}

thread 'main' (852780) panicked at crates/wordchipper/src/vocab/span_vocab.rs:247:51:
called `Result::unwrap()` on an `Err` value: VocabConflict("Pair (157513, 157853) -> 229601 parent 157513 is not defined")
stack backtrace:
   0: __rustc::rust_begin_unwind
             at /rustc/59807616e1fa2540724bfbac14d7976d7e4a3860/library/std/src/panicking.rs:689:5
   1: core::panicking::panic_fmt
             at /rustc/59807616e1fa2540724bfbac14d7976d7e4a3860/library/core/src/panicking.rs:80:14
   2: core::result::unwrap_failed
             at /rustc/59807616e1fa2540724bfbac14d7976d7e4a3860/library/core/src/result.rs:1867:5
   3: core::result::Result<T,E>::unwrap
             at /home/crutcher/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/result.rs:1233:23
   4: wordchipper::vocab::span_vocab::SpanMapVocab<T>::to_pair_vocab
             at ./crates/wordchipper/src/vocab/span_vocab.rs:247:51
   5: wordchipper::vocab::unified_vocab::UnifiedTokenVocab<T>::from_span_vocab
             at ./crates/wordchipper/src/vocab/unified_vocab.rs:85:37
   6: wordchipper::pretrained::huggingface::hf_factory::vocab_from_hf_tokenizer
             at ./crates/wordchipper/src/pretrained/huggingface/hf_factory.rs:199:18
   7: <wordchipper::pretrained::huggingface::hf_factory::HFVocabProvider as wordchipper::pretrained::factory::vocab_provider::VocabProvider>::load_vocab
             at ./crates/wordchipper/src/pretrained/huggingface/hf_factory.rs:245:29
   8: wordchipper::pretrained::factory::vocab_factory::VocabFactory::load_vocab
             at ./crates/wordchipper/src/pretrained/factory/vocab_factory.rs:229:28
   9: wordchipper::pretrained::factory::vocab_factory::load_vocab::{{closure}}
             at ./crates/wordchipper/src/pretrained/factory/vocab_factory.rs:99:55
  10: wordchipper::pretrained::factory::vocab_factory::with_vocab_factory
             at ./crates/wordchipper/src/pretrained/factory/vocab_factory.rs:76:5
  11: wordchipper::pretrained::factory::vocab_factory::load_vocab
             at ./crates/wordchipper/src/pretrained/factory/vocab_factory.rs:99:5
  12: wordchipper_cli_util::model_selector::ModelSelectorArgs::load_vocab
             at ./crates/wordchipper-cli-util/src/model_selector.rs:29:21
  13: wordchipper_cli_util::model_selector::ModelSelectorArgs::load_tokenizer
             at ./crates/wordchipper-cli-util/src/model_selector.rs:41:26
  14: wordchipper_cli::commands::cat_cmd::CatArgs::run
             at ./crates/wordchipper-cli/src/commands/cat_cmd.rs:51:45
  15: wordchipper_cli::Commands::run
             at ./crates/wordchipper-cli/src/main.rs:44:39
  16: wordchipper_cli::main
             at ./crates/wordchipper-cli/src/main.rs:18:18
  17: core::ops::function::FnOnce::call_once
             at /home/crutcher/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
```
---
 .../src/pretrained/huggingface/hf_factory.rs  | 28 +++++++++++++++++--
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs b/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
index 0b851ed1..5c6406f4 100644
--- a/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
+++ b/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
@@ -1,3 +1,5 @@
+use std::println;
+
 use tokenizers::{
     ModelWrapper::BPE,
     PreTokenizerWrapper,
@@ -130,7 +132,21 @@ pub fn vocab_from_hf_tokenizer(tok: &Tokenizer) -> WCResult<Arc<UnifiedTokenVoca
 
     let hf_vocab = bpe.get_vocab();
 
-    let special_tokens: WCHashSet<u32> = tok.get_added_tokens_decoder().keys().copied().collect();
+    println!(
+        "Debug: {:?}",
+        hf_vocab.iter().find(|(_, id)| **id == 157513)
+    );
+
+    // TODO: This is broken for Qwen/Qwen3.5-9B for some reason.
+    let mut special_tokens: WCHashSet<T> = Default::default();
+
+    let decoder = tok.get_added_tokens_decoder();
+    println!("Debug: {:#?}", decoder);
+
+    for (t, at) in decoder.iter() {
+        span_config.specials_mut().add_str_word(&at.content, *t);
+        special_tokens.insert(*t);
+    }
 
     // Forward and inverse bytes_to_unicode maps.
     let b2c = bytes_char();
@@ -140,7 +156,7 @@ pub fn vocab_from_hf_tokenizer(tok: &Tokenizer) -> WCResult<Arc<UnifiedTokenVoca
     let mut span_map: SpanTokenMap<T> = SpanTokenMap::default();
     for (s, id) in &hf_vocab {
         if special_tokens.contains(id) {
-            span_config.specials_mut().add_str_word(s, *id);
+            continue;
         } else {
             let mut bytes = Vec::with_capacity(s.len());
             for ch in s.chars() {
@@ -157,7 +173,13 @@ pub fn vocab_from_hf_tokenizer(tok: &Tokenizer) -> WCResult<Arc<UnifiedTokenVoca
         }
     }
 
-    assert_eq!(span_config.specials().len(), special_tokens.len());
+    if span_config.specials().len() != special_tokens.len() {
+        return Err(WCError::External(format!(
+            "hf vocab identifies {} special tokens, but only {} special tokens found in span_config",
+            special_tokens.len(),
+            span_config.specials().len()
+        )));
+    }
 
     // Byte map: the single-char string for each byte must resolve in the vocab.
     let byte_tokens: Vec<T> = (0u8..=255)

From 460dacfa427dfe44410c940000aaa66d33a04551 Mon Sep 17 00:00:00 2001
From: Crutcher Dunnavant <crutcher@gmail.com>
Date: Sun, 10 May 2026 23:10:21 -0700
Subject: [PATCH 6/7] Update partial support for HF with basic errors.

---
 Cargo.lock                                    | 12 +++-
 Cargo.toml                                    |  1 +
 crates/wordchipper/Cargo.toml                 |  1 +
 crates/wordchipper/README.md                  |  5 ++
 crates/wordchipper/src/errors.rs              | 10 ++-
 .../src/pretrained/huggingface/hf_factory.rs  |  6 +-
 crates/wordchipper/src/support/mod.rs         |  2 +
 .../src/support/with_ok_or_panic.rs           | 61 +++++++++++++++++++
 crates/wordchipper/src/vocab/pair_vocab.rs    | 15 +++--
 crates/wordchipper/src/vocab/span_vocab.rs    |  7 ++-
 10 files changed, 107 insertions(+), 13 deletions(-)
 create mode 100644 crates/wordchipper/src/support/with_ok_or_panic.rs

diff --git a/Cargo.lock b/Cargo.lock
index 7a8b0c04..74ed0b65 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1631,6 +1631,15 @@ dependencies = [
  "web-time",
 ]
 
+[[package]]
+name = "indoc"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "integer-encoding"
 version = "3.0.4"
@@ -2471,7 +2480,7 @@ dependencies = [
  "once_cell",
  "socket2",
  "tracing",
- "windows-sys 0.52.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -4317,6 +4326,7 @@ dependencies = [
  "fancy-regex",
  "foldhash 0.2.0",
  "hashbrown 0.16.1",
+ "indoc",
  "inventory",
  "log",
  "logos",
diff --git a/Cargo.toml b/Cargo.toml
index 71bb8825..832fa430 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -82,6 +82,7 @@ divan = { package = "codspeed-divan-compat", version = "4.3.0" }
 document-features = "0.2.12"
 humansize = "2.1.3"
 indicatif = "0.18.4"
+indoc = "2.0.7"
 js-sys = "0.3"
 parquet = "58.0.0"
 proptest = "1.10.0"
diff --git a/crates/wordchipper/Cargo.toml b/crates/wordchipper/Cargo.toml
index fbc33b63..5e9c8ba1 100644
--- a/crates/wordchipper/Cargo.toml
+++ b/crates/wordchipper/Cargo.toml
@@ -125,6 +125,7 @@ ringbuffer = { workspace = true }
 regex = { workspace = true, features = ["unicode"] }
 regex-automata = { workspace = true, features = ["alloc", "meta", "nfa-thompson", "hybrid", "unicode"] }
 strum = { workspace = true }
+indoc = { workspace = true }
 
 # Provides HashMap/HashSet in no_std mode (non-optional so `default-features = false` just works).
 hashbrown = { workspace = true, features = ["alloc"] }
diff --git a/crates/wordchipper/README.md b/crates/wordchipper/README.md
index 32d33601..6f9c07a6 100644
--- a/crates/wordchipper/README.md
+++ b/crates/wordchipper/README.md
@@ -76,6 +76,11 @@ configuration.
 For a number of pretrained models, simplified constructors are
 available to download, cache, and load the vocabulary.
 
+At this time, we have support for the following loaders:
+
+* `openai:[{PATH}/]{NAME}` - Lod pre-trained OpenAI models.
+* `hf:[{PATH}/]{NAME}` - Load pre-trained HuggingFace models.
+
 See: [wordchipper::get_model](
 https://docs.rs/wordchipper/latest/wordchipper/fn.get_model.html)
 
diff --git a/crates/wordchipper/src/errors.rs b/crates/wordchipper/src/errors.rs
index 7396ea4c..c7ae4141 100644
--- a/crates/wordchipper/src/errors.rs
+++ b/crates/wordchipper/src/errors.rs
@@ -5,12 +5,16 @@ use crate::alloc::string::String;
 /// Errors from wordchipper operations.
 #[derive(Debug, thiserror::Error)]
 pub enum WCError {
+    /// Not Implemented Error.
+    #[error("Not Implemented: {0}")]
+    NotImplemented(String),
+
     /// Resource not found.
-    #[error("{0}")]
+    #[error("Resource Not Found: {0}")]
     ResourceNotFound(String),
 
     /// The resource is a duplicate.
-    #[error("{0}")]
+    #[error("Duplicate: {0}")]
     DuplicatedResource(String),
 
     /// Vocab size exceeds the capacity of the target token type.
@@ -28,7 +32,7 @@ pub enum WCError {
     },
 
     /// Vocabulary data is inconsistent.
-    #[error("{0}")]
+    #[error("Vocab Conflict: {0}")]
     VocabConflict(String),
 
     /// Token value out of range for the target type.
diff --git a/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs b/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
index 5c6406f4..ea9e9f84 100644
--- a/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
+++ b/crates/wordchipper/src/pretrained/huggingface/hf_factory.rs
@@ -1,5 +1,3 @@
-use std::println;
-
 use tokenizers::{
     ModelWrapper::BPE,
     PreTokenizerWrapper,
@@ -132,16 +130,20 @@ pub fn vocab_from_hf_tokenizer(tok: &Tokenizer) -> WCResult<Arc<UnifiedTokenVoca
 
     let hf_vocab = bpe.get_vocab();
 
+    /*
     println!(
         "Debug: {:?}",
         hf_vocab.iter().find(|(_, id)| **id == 157513)
     );
+     */
 
     // TODO: This is broken for Qwen/Qwen3.5-9B for some reason.
     let mut special_tokens: WCHashSet<T> = Default::default();
 
     let decoder = tok.get_added_tokens_decoder();
+    /*
     println!("Debug: {:#?}", decoder);
+     */
 
     for (t, at) in decoder.iter() {
         span_config.specials_mut().add_str_word(&at.content, *t);
diff --git a/crates/wordchipper/src/support/mod.rs b/crates/wordchipper/src/support/mod.rs
index 6e643bfc..478f9c56 100644
--- a/crates/wordchipper/src/support/mod.rs
+++ b/crates/wordchipper/src/support/mod.rs
@@ -2,6 +2,7 @@
 
 #[cfg(feature = "concurrent")]
 pub mod concurrency;
+
 pub mod ranges;
 pub mod regex;
 pub mod resources;
@@ -9,3 +10,4 @@ pub mod slices;
 pub mod strings;
 pub mod timers;
 pub mod traits;
+pub mod with_ok_or_panic;
diff --git a/crates/wordchipper/src/support/with_ok_or_panic.rs b/crates/wordchipper/src/support/with_ok_or_panic.rs
new file mode 100644
index 00000000..82be4482
--- /dev/null
+++ b/crates/wordchipper/src/support/with_ok_or_panic.rs
@@ -0,0 +1,61 @@
+//! # Result Utilities
+//!
+//! Methods for [`std::result::Result`] manipulation.
+
+use core::fmt::Display;
+
+/// Extension trait for `Result<T, E>` to add `ok_or_panic` method.
+pub trait WithOkOrPanic<T> {
+    /// Unwraps the `Result`, or panics with the error message.
+    ///
+    /// This differs from the behavior of [`Result::unwrap`]
+    /// in that the [`Debug`] format of the wrapped error is used
+    /// directly as the panic message; and not escaped.
+    fn ok_or_panic(self) -> T;
+}
+
+impl<T, E> WithOkOrPanic<T> for Result<T, E>
+where
+    E: Display,
+{
+    fn ok_or_panic(self) -> T {
+        match self {
+            Ok(t) => t,
+            Err(e) => panic!("{e}"),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        WCError,
+        WCResult,
+        prelude::*,
+    };
+
+    fn try_example(
+        value: i32,
+        throw: bool,
+    ) -> WCResult<i32> {
+        if throw {
+            Err(WCError::External("throwing".to_string()))
+        } else {
+            Ok(value)
+        }
+    }
+
+    #[test]
+    fn test_expect_unwrap() {
+        let result = try_example(42, false);
+        assert_eq!(result.ok_or_panic(), 42);
+    }
+
+    #[should_panic(expected = "throwing")]
+    #[test]
+    fn test_expect_unwrap_panic() {
+        let result = try_example(42, true);
+        result.ok_or_panic();
+    }
+}
diff --git a/crates/wordchipper/src/vocab/pair_vocab.rs b/crates/wordchipper/src/vocab/pair_vocab.rs
index ac0ff320..b8d39b4d 100644
--- a/crates/wordchipper/src/vocab/pair_vocab.rs
+++ b/crates/wordchipper/src/vocab/pair_vocab.rs
@@ -47,19 +47,26 @@ pub fn try_validate_pair_map<T: TokenType>(
         }
     }
 
+    const ORPHAN_TOKENS_ERROR: &str = indoc::indoc! {r#"
+        This vocab has orphan tokens, which wordchipper does not yet support.
+        See: https://github.com/zspacelabs/wordchipper/issues/386
+    "#};
+
     for (&pair, &t) in pairs.iter() {
         for pt in [pair.0, pair.1] {
             let is_pair_target = pair_targets.contains(&pt);
             let byte_target = byte_vocab.get_byte(pt);
 
             if is_pair_target && let Some(b) = byte_target {
-                return Err(crate::WCError::VocabConflict(crate::alloc::format!(
-                    "Pair {pair:?} -> {t:?} parent {pt:?} is a pair target and byte target: {b:0x?}"
+                return Err(crate::WCError::NotImplemented(crate::alloc::format!(
+                    "{PRE}Pair {pair:?} -> {t:?} parent {pt:?} is a pair target and byte target: {b:0x?}",
+                    PRE = ORPHAN_TOKENS_ERROR,
                 )));
             }
             if !is_pair_target && byte_target.is_none() {
-                return Err(crate::WCError::VocabConflict(crate::alloc::format!(
-                    "Pair {pair:?} -> {t:?} parent {pt:?} is not defined"
+                return Err(crate::WCError::NotImplemented(crate::alloc::format!(
+                    "{PRE}Pair {pair:?} -> {t:?} parent {pt:?} is not defined",
+                    PRE = ORPHAN_TOKENS_ERROR,
                 )));
             }
         }
diff --git a/crates/wordchipper/src/vocab/span_vocab.rs b/crates/wordchipper/src/vocab/span_vocab.rs
index da96c756..3b3234ff 100644
--- a/crates/wordchipper/src/vocab/span_vocab.rs
+++ b/crates/wordchipper/src/vocab/span_vocab.rs
@@ -3,6 +3,7 @@
 use crate::{
     WCResult,
     alloc::vec::Vec,
+    support::with_ok_or_panic::WithOkOrPanic,
     types::{
         TokenType,
         WCHashMap,
@@ -96,7 +97,7 @@ impl<T: TokenType> SpanMapVocab<T> {
     pub fn from_byte_vocab(byte_vocab: ByteMapVocab<T>) -> Self {
         let span_map: SpanTokenMap<T> = byte_vocab.span_pairs().collect();
 
-        Self::new(byte_vocab, span_map).unwrap()
+        Self::new(byte_vocab, span_map).ok_or_panic()
     }
 
     /// Build a [`Self`] from a [`SpanTokenMap`].
@@ -126,7 +127,7 @@ impl<T: TokenType> SpanMapVocab<T> {
 
         let byte_vocab: ByteMapVocab<T> = ByteMapVocab::from_byte_to_token(&byte_to_token);
 
-        Self::new(byte_vocab, span_map).unwrap()
+        Self::new(byte_vocab, span_map).ok_or_panic()
     }
 
     /// Initialize a [`SpanMapVocab`].
@@ -244,7 +245,7 @@ impl<T: TokenType> SpanMapVocab<T> {
             }
         }
 
-        PairMapVocab::<T>::new(byte_vocab, pairs).unwrap()
+        PairMapVocab::<T>::new(byte_vocab, pairs).ok_or_panic()
     }
 }
 

From acf7aef284f0224341b65cc8aad126eea5535ca2 Mon Sep 17 00:00:00 2001
From: Crutcher Dunnavant <crutcher@gmail.com>
Date: Mon, 11 May 2026 01:03:27 -0700
Subject: [PATCH 7/7] Fix no-std build.

---
 crates/wordchipper/src/pretrained/factory/vocab_query.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/crates/wordchipper/src/pretrained/factory/vocab_query.rs b/crates/wordchipper/src/pretrained/factory/vocab_query.rs
index 1efbdb8b..4f48e93b 100644
--- a/crates/wordchipper/src/pretrained/factory/vocab_query.rs
+++ b/crates/wordchipper/src/pretrained/factory/vocab_query.rs
@@ -194,6 +194,7 @@ mod tests {
     use core::str::FromStr;
 
     use crate::{
+        alloc::vec,
         prelude::*,
         pretrained::factory::vocab_query::VocabQuery,
     };