fix(build): forward tokenjuice-treesitter + gate feature-forwarding drift in CI (#4918, #4919)

.github/workflows/ci-lite.yml

@@ -665,6 +665,30 @@ jobs:
       - name: Run orchestration IP gate
         run: bash scripts/ci/orch-ip-gate.sh
 
+  feature-forwarding-gate:
+    name: Feature Forwarding Gate (shell forwards core defaults)
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+
+      # The shell sets `default-features = false` on `openhuman_core`, so it does
+      # NOT inherit the core's default gates — each must be forwarded by hand.
+      # When that list drifts, the domain is compiled out of the shipped app with
+      # no build error and no failing test: that is how #4901 (voice — 56 users,
+      # ~93k Sentry events) and #4918 (tokenjuice-treesitter — silent) shipped.
+      #
+      # Deliberately NOT filtered on `changes`: drift can be introduced by
+      # editing either manifest, so a path filter watching one would miss it, and
+      # a skipped job counts as a pass in the gate below. The check is a few
+      # seconds of pure Node with no dependencies.
+      - name: Verify the desktop shell forwards every default-ON core gate
+        run: node scripts/ci/check-feature-forwarding.mjs
+
   pester-install:
     name: PowerShell Install Test (Pester)
     needs: [changes]
@@ -731,6 +755,7 @@ jobs:
       - pester-install
       - tinycortex-tests
       - orch-ip-gate
+      - feature-forwarding-gate
     if: always()
     runs-on: ubuntu-latest
     timeout-minutes: 15
@@ -749,6 +774,7 @@ jobs:
             ["PowerShell Install Test"]="${{ needs['pester-install'].result }}"
             ["TinyCortex Memory Tests"]="${{ needs['tinycortex-tests'].result }}"
             ["Orchestration IP Gate"]="${{ needs['orch-ip-gate'].result }}"
+            ["Feature Forwarding Gate"]="${{ needs['feature-forwarding-gate'].result }}"
           )
 
           failed=0

AGENTS.md

@@ -222,6 +222,10 @@ Two independent runtime axes on `CoreBuilder` (`src/core/runtime/builder.rs`):
 
 Per-domain Cargo features drop whole domains **at compile time** (smaller binary, fewer deps), composing with the runtime `DomainSet` axis above. Each gate is **default-ON**, so the desktop build is byte-identical; slim builds opt out explicitly.
 
+> **Adding a default-ON gate? You must forward it to the desktop shell.**
+> `app/src-tauri/Cargo.toml` declares `openhuman_core` with `default-features = false` (set in #1061, before gates existed), so the shipped app does **not** inherit the core's `default` list. A gate you add to `default` but not to the shell's `features` list is **compiled out of the shipped desktop app** — with no build error and no failing test. This is not hypothetical: `voice` shipped missing from v0.58.19 to v0.61.x (56 users, ~93k Sentry events, #4901), and `tokenjuice-treesitter` was never forwarded once since #4123 and failed *soft*, silently degrading AST compression (#4918).
+> `scripts/ci/check-feature-forwarding.mjs` (the **Feature Forwarding Gate** lane) now fails CI on drift and covers new gates automatically. If a gate genuinely must not ship, add it to `INTENTIONALLY_NOT_FORWARDED` in that script **with a reason** — an explicit exclusion is the only way "deliberate" stays distinguishable from "forgotten".
+
 **Slim-profile convention** (no `full` meta-feature): build slim variants with `cargo build --no-default-features --features "<explicit list of gates you want>"`. This mirrors the existing standalone-feature style (`sandbox-landlock`, `browser-native`, …). Example — everything except voice:
 
 ```bash

app/src-tauri/Cargo.toml

@@ -152,11 +152,17 @@ cef = { version = "=146.4.1", default-features = false }
 #   the pre-gate desktop tool surface.
 # - `web3` — keeps the wallet/web3/x402 domains and their agent tools in the
 #   desktop build while allowing slim builds to omit the crypto-only deps.
+# - `tokenjuice-treesitter` — forwards `tinyjuice/tinyjuice-treesitter`, which
+#   pulls the tree-sitter Rust/TS/Python grammars. Without it TokenJuice falls
+#   back to the brace-depth heuristic, so the desktop app compresses code worse
+#   than intended. This failed *soft* (no error, no Sentry signal) and had never
+#   once been forwarded since the gate was added in #4123 (#4918).
 #
-# `tokenjuice-treesitter` is the remaining un-forwarded default — tracked in
-# #4918, deliberately not bundled here because dropping it may be intentional.
+# `scripts/ci/check-feature-forwarding.mjs` fails CI when this list drifts from
+# the core's `[features] default` (#4919) — do not hand-maintain it from memory.
 openhuman_core = { path = "../..", package = "openhuman", default-features = false, features = [
     "media",
+    "tokenjuice-treesitter",
     "voice",
     "web3",
 ] }

scripts/__tests__/feature-forwarding.test.mjs

@@ -0,0 +1,187 @@
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { test } from 'node:test';
+import { fileURLToPath } from 'node:url';
+
+import {
+  diffForwarding,
+  parseCoreDefaultFeatures,
+  parseShellForwardedFeatures,
+  stripComments,
+} from '../lib/feature-forwarding.mjs';
+
+const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
+const CHECKER = resolve(REPO_ROOT, 'scripts/ci/check-feature-forwarding.mjs');
+
+// ── parsing ────────────────────────────────────────────────────────────────
+
+test('parses the core default gate list', () => {
+  const toml = `
+[features]
+default = ["tokenjuice-treesitter", "voice", "media"]
+voice = ["dep:hound"]
+`;
+  assert.deepEqual(parseCoreDefaultFeatures(toml), ['tokenjuice-treesitter', 'voice', 'media']);
+});
+
+test('parses a multi-line default gate list', () => {
+  const toml = `
+[features]
+default = [
+    "voice",
+    "media",
+]
+`;
+  assert.deepEqual(parseCoreDefaultFeatures(toml), ['voice', 'media']);
+});
+
+test('ignores a default key belonging to another table', () => {
+  const toml = `
+[some-other-table]
+default = ["not-a-gate"]
+
+[features]
+default = ["voice"]
+`;
+  assert.deepEqual(parseCoreDefaultFeatures(toml), ['voice']);
+});
+
+test('parses the shell forwarded list across multiple lines', () => {
+  const toml = `
+openhuman_core = { path = "../..", package = "openhuman", default-features = false, features = [
+    "media",
+    "voice",
+] }
+`;
+  assert.deepEqual(parseShellForwardedFeatures(toml), {
+    defaultFeatures: false,
+    features: ['media', 'voice'],
+  });
+});
+
+test('detects when the shell inherits defaults instead of forwarding', () => {
+  const toml = 'openhuman_core = { path = "../..", package = "openhuman" }\n';
+  assert.deepEqual(parseShellForwardedFeatures(toml), { defaultFeatures: true, features: [] });
+});
+
+test('comment stripping does not truncate on a # inside a quoted value', () => {
+  const stripped = stripComments('a = "issue #4901"  # trailing comment\n');
+  assert.match(stripped, /issue #4901/);
+  assert.doesNotMatch(stripped, /trailing comment/);
+});
+
+test('a commented-out gate does not count as forwarded', () => {
+  const toml = `
+openhuman_core = { path = "../..", package = "openhuman", default-features = false, features = [
+    # "voice",
+    "media",
+] }
+`;
+  assert.deepEqual(parseShellForwardedFeatures(toml).features, ['media']);
+});
+
+// ── drift detection ────────────────────────────────────────────────────────
+
+test('passes when every default gate is forwarded', () => {
+  const result = diffForwarding({
+    coreDefaults: ['voice', 'media'],
+    shell: { defaultFeatures: false, features: ['media', 'voice'] },
+  });
+  assert.equal(result.ok, true);
+  assert.deepEqual(result.missing, []);
+});
+
+test('reproduces #4901: a dropped voice gate is reported missing', () => {
+  const result = diffForwarding({
+    coreDefaults: ['tokenjuice-treesitter', 'voice', 'media'],
+    shell: { defaultFeatures: false, features: ['media', 'tokenjuice-treesitter'] },
+  });
+  assert.equal(result.ok, false);
+  assert.deepEqual(result.missing, ['voice']);
+});
+
+test('reproduces #4918: a dropped tokenjuice-treesitter gate is reported missing', () => {
+  const result = diffForwarding({
+    coreDefaults: ['tokenjuice-treesitter', 'voice', 'media'],
+    shell: { defaultFeatures: false, features: ['media', 'voice'] },
+  });
+  assert.equal(result.ok, false);
+  assert.deepEqual(result.missing, ['tokenjuice-treesitter']);
+});
+
+test('a brand new default gate is covered automatically, with no per-gate wiring', () => {
+  const result = diffForwarding({
+    coreDefaults: ['voice', 'media', 'some-future-gate'],
+    shell: { defaultFeatures: false, features: ['voice', 'media'] },
+  });
+  assert.equal(result.ok, false);
+  assert.deepEqual(result.missing, ['some-future-gate']);
+});
+
+test('an allow-listed gate passes and is reported as intentional', () => {
+  const result = diffForwarding({
+    coreDefaults: ['voice', 'heavy-gate'],
+    shell: { defaultFeatures: false, features: ['voice'] },
+    allowlist: { 'heavy-gate': 'Adds 400MB of models to the bundle.' },
+  });
+  assert.equal(result.ok, true);
+  assert.deepEqual(result.allowed, ['heavy-gate']);
+  assert.deepEqual(result.missing, []);
+});
+
+test('an allow-list entry for a gate that IS forwarded is flagged as stale', () => {
+  const result = diffForwarding({
+    coreDefaults: ['voice'],
+    shell: { defaultFeatures: false, features: ['voice'] },
+    allowlist: { voice: 'stale entry' },
+  });
+  assert.equal(result.ok, false);
+  assert.deepEqual(result.stale, ['voice']);
+});
+
+test('inheriting defaults needs no forwarding', () => {
+  const result = diffForwarding({
+    coreDefaults: ['voice'],
+    shell: { defaultFeatures: true, features: [] },
+  });
+  assert.equal(result.ok, true);
+});
+
+test('a missing dependency fails rather than passing vacuously', () => {
+  const result = diffForwarding({ coreDefaults: ['voice'], shell: null });
+  assert.equal(result.ok, false);
+  assert.equal(result.reason, 'dependency-not-found');
+});
+
+// ── the real manifests + CLI ───────────────────────────────────────────────
+
+test('the checked-in manifests pass the guard', () => {
+  const out = execFileSync('node', [CHECKER], { encoding: 'utf8' });
+  assert.match(out, /every default-ON core gate is forwarded/);
+});
+
+test('--help exits 0', () => {
+  const out = execFileSync('node', [CHECKER, '--help'], { encoding: 'utf8' });
+  assert.match(out, /Usage:/);
+});
+
+test('the real shell manifest forwards every real core default', () => {
+  const coreDefaults = parseCoreDefaultFeatures(
+    readFileSync(resolve(REPO_ROOT, 'Cargo.toml'), 'utf8')
+  );
+  const shell = parseShellForwardedFeatures(
+    readFileSync(resolve(REPO_ROOT, 'app/src-tauri/Cargo.toml'), 'utf8')
+  );
+  // Guards the guard: if the parser silently returned nothing, the assertions
+  // below would pass against empty input and prove nothing.
+  assert.ok(coreDefaults.length > 0, 'expected to parse at least one core default gate');
+  assert.equal(shell.defaultFeatures, false, 'shell is expected to set default-features = false');
+  for (const gate of coreDefaults) {
+    assert.ok(
+      shell.features.includes(gate),
+      `core default gate not forwarded to the shell: ${gate}`
+    );
+  }
+});