Provide the V1 tokenizer inventory activation bridge

crates/mesh-native-serving-plugin-api/README.md

@@ -14,6 +14,10 @@ layout of the prefix understood by an older plugin. Removing or reordering
 fields, changing a field's type, or changing a callback signature requires a
 new versioned ABI table and continued support for the previous table.
 
+The V1 activation contract lends the complete native tokenizer inventory through
+`ActivationContext`. A plugin copies or transforms the inventory before
+returning from `activate`.
+
 Each proposal query carries the capacity Skippy can verify at that exact decode
 position. The native host adapter also applies a bounded implementation cap so
 a request cannot force an unbounded host allocation.

crates/mesh-native-serving-plugin-api/src/lib.rs

@@ -17,6 +17,29 @@ use std::ffi::{c_char, c_void};
 pub const NATIVE_SERVING_PLUGIN_ABI_V1: u32 = 1;
 pub const NATIVE_SERVING_PLUGIN_ENTRY_V1: &[u8] = b"mesh_native_serving_plugin_v1\0";
 pub const MAX_DECISION_ID_BYTES: usize = 64;
+pub const TOKENIZER_INVENTORY_SCHEMA: u32 = 1;
+
+/// Host-owned typed inventory. This Rust value never crosses the ABI directly.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct TokenizerInventory {
+    pub schema_version: u32,
+    pub model_id: String,
+    pub source_model_sha256: String,
+    pub tokenizer_id: String,
+    pub tokens: Vec<TokenizerInventoryToken>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct TokenizerInventoryToken {
+    pub id: u32,
+    pub piece: TokenizerInventoryPiece,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum TokenizerInventoryPiece {
+    Bytes { bytes: Vec<u8> },
+    Control { identity: String },
+}
 
 pub type PluginInstance = *mut c_void;
 pub type ProposalOperation = u64;
@@ -38,6 +61,37 @@ impl ByteSlice {
     }
 }
 
+#[repr(transparent)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct TokenizerPieceKind(pub u32);
+
+impl TokenizerPieceKind {
+    pub const BYTES: Self = Self(0);
+    pub const CONTROL: Self = Self(1);
+}
+
+/// Borrowed ABI view of one immutable native token. The referenced bytes are
+/// valid only for the duration of `activate`; a plugin must copy or transform
+/// them before it returns.
+#[repr(C)]
+#[derive(Clone, Copy, Debug)]
+pub struct TokenizerInventoryEntry {
+    pub id: u32,
+    pub piece_kind: TokenizerPieceKind,
+    pub bytes: ByteSlice,
+}
+
+/// Borrowed ABI view of the complete vocabulary. The host owns the entries and
+/// their bytes and passes them only while activating the plugin.
+#[repr(C)]
+#[derive(Clone, Copy, Debug)]
+pub struct TokenizerInventoryView {
+    pub struct_size: usize,
+    pub schema_version: u32,
+    pub entries: *const TokenizerInventoryEntry,
+    pub entry_count: usize,
+}
+
 #[repr(C)]
 #[derive(Clone, Copy, Debug, Default)]
 pub struct TokenSlice {
@@ -136,6 +190,7 @@ pub struct ActivationContext {
     pub model_id: ByteSlice,
     pub source_model_sha256: ByteSlice,
     pub tokenizer_id: ByteSlice,
+    pub tokenizer_inventory: *const TokenizerInventoryView,
     pub config_path: ByteSlice,
     pub state_directory: ByteSlice,
     pub proposal_deadline_ns: u64,
@@ -324,7 +379,7 @@ mod tests {
     }
 
     #[test]
-    fn stable_contract_keeps_only_the_correlation_bound() {
+    fn initial_contract_is_v1() {
         assert_eq!(MAX_DECISION_ID_BYTES, 64);
         assert_eq!(NATIVE_SERVING_PLUGIN_ABI_V1, 1);
     }

crates/mesh-native-serving-plugin-host/README.md

@@ -5,3 +5,7 @@ local-model OpenAI surface. Plugin proposal calls run on an isolated worker;
 Skippy's decode thread waits only until its own absolute deadline and never
 joins or drains plugin work. Lifecycle and authoritative verification outcomes
 remain ordered through the same bounded queue.
+
+At activation, the host passes a borrowed view of the model's already-bound
+native tokenizer inventory over the plugin ABI. The view and its byte slices
+are valid only for that call; plugins must not retain them.

crates/mesh-native-serving-plugin-host/src/lib.rs

@@ -86,13 +86,20 @@ impl NativeServingPluginFactory {
 impl ModelServingHooksFactory for NativeServingPluginFactory {
     fn create(&self, tokenizer: TokenizerCapability) -> Result<ModelServingHooks> {
         let identity = tokenizer.identity();
+        let inventory = ActivationInventory::from_inventory(
+            tokenizer
+                .inventory()
+                .context("bound model does not expose a tokenizer inventory")?,
+        )?;
+        let inventory_view = inventory.view();
         let context = abi::ActivationContext {
             struct_size: size_of::<abi::ActivationContext>(),
             model_id: abi::ByteSlice::from_bytes(identity.model_id.as_bytes()),
             source_model_sha256: abi::ByteSlice::from_bytes(
                 identity.source_model_sha256.as_bytes(),
             ),
             tokenizer_id: abi::ByteSlice::from_bytes(identity.tokenizer_id.as_bytes()),
+            tokenizer_inventory: &raw const inventory_view,
             config_path: path_slice(&self.config_path),
             state_directory: path_slice(&self.state_directory),
             proposal_deadline_ns: u64::try_from(self.proposal_deadline.as_nanos())
@@ -135,6 +142,49 @@ impl ModelServingHooksFactory for NativeServingPluginFactory {
     }
 }
 
+struct ActivationInventory {
+    entries: Vec<abi::TokenizerInventoryEntry>,
+}
+
+impl ActivationInventory {
+    fn from_inventory(inventory: &abi::TokenizerInventory) -> Result<Self> {
+        if inventory.schema_version != abi::TOKENIZER_INVENTORY_SCHEMA
+            || inventory.tokens.is_empty()
+        {
+            bail!("bound model exposes an unsupported or empty tokenizer inventory");
+        }
+        let entries = inventory
+            .tokens
+            .iter()
+            .map(|entry| {
+                let (piece_kind, bytes) = match &entry.piece {
+                    abi::TokenizerInventoryPiece::Bytes { bytes } => {
+                        (abi::TokenizerPieceKind::BYTES, bytes.as_slice())
+                    }
+                    abi::TokenizerInventoryPiece::Control { identity } => {
+                        (abi::TokenizerPieceKind::CONTROL, identity.as_bytes())
+                    }
+                };
+                abi::TokenizerInventoryEntry {
+                    id: entry.id,
+                    piece_kind,
+                    bytes: abi::ByteSlice::from_bytes(bytes),
+                }
+            })
+            .collect();
+        Ok(Self { entries })
+    }
+
+    fn view(&self) -> abi::TokenizerInventoryView {
+        abi::TokenizerInventoryView {
+            struct_size: size_of::<abi::TokenizerInventoryView>(),
+            schema_version: abi::TOKENIZER_INVENTORY_SCHEMA,
+            entries: self.entries.as_ptr(),
+            entry_count: self.entries.len(),
+        }
+    }
+}
+
 struct LoadedDefinition {
     _library: Option<Library>,
     api: NonNull<abi::NativeServingPluginV1>,
@@ -773,6 +823,32 @@ mod tests {
     static FAKE_NAME: &[u8] = b"test-serving-plugin";
     static CANCEL_COUNT: AtomicUsize = AtomicUsize::new(0);
 
+    #[test]
+    fn tokenizer_inventory_view_borrows_host_owned_bytes_for_activation() {
+        let inventory = abi::TokenizerInventory {
+            schema_version: abi::TOKENIZER_INVENTORY_SCHEMA,
+            model_id: "glm".to_string(),
+            source_model_sha256: "a".repeat(64),
+            tokenizer_id: "gguf-source-sha256:test".to_string(),
+            tokens: vec![abi::TokenizerInventoryToken {
+                id: 0,
+                piece: abi::TokenizerInventoryPiece::Bytes {
+                    bytes: b"hello".to_vec(),
+                },
+            }],
+        };
+        let activation = ActivationInventory::from_inventory(&inventory).unwrap();
+        let view = activation.view();
+        assert_eq!(view.schema_version, abi::TOKENIZER_INVENTORY_SCHEMA);
+        assert_eq!(view.entry_count, 1);
+        let entry = unsafe { &*view.entries };
+        assert_eq!(entry.piece_kind, abi::TokenizerPieceKind::BYTES);
+        assert_eq!(
+            unsafe { std::slice::from_raw_parts(entry.bytes.pointer, entry.bytes.length) },
+            b"hello"
+        );
+    }
+
     struct FakeState {
         start_delay: Duration,
         begin_fails: bool,