From 3900d44c75872c051dd6c6b944b1751057bec0ea Mon Sep 17 00:00:00 2001 From: Antonio Antenore Date: Sun, 19 Jul 2026 03:51:43 +0200 Subject: [PATCH] feat: add verified WebLLM browser runtime --- CHANGELOG.md | 14 + README.md | 61 +++- demo/webllm.html | 263 +++++++++++++++ demo/webllm.ts | 271 +++++++++++++++ docs/adr/0002-optional-webllm-adapter.md | 29 ++ docs/compatibility.md | 28 +- docs/delivery-contract.md | 29 +- docs/integrations.md | 24 +- docs/market-scan.md | 22 +- docs/runbook.md | 6 +- docs/threat-model.md | 2 + docs/visual-qa.md | 11 +- package.json | 18 +- playwright.webllm.config.ts | 35 ++ pnpm-lock.yaml | 16 + scripts/package-smoke.mjs | 22 ++ src/adapters/webllm.ts | 409 +++++++++++++++++++++++ tests/live/webllm.spec.ts | 165 +++++++++ tests/unit/webllm-adapter.test.ts | 355 ++++++++++++++++++++ tsconfig.json | 2 + tsup.config.ts | 1 + 21 files changed, 1734 insertions(+), 49 deletions(-) create mode 100644 demo/webllm.html create mode 100644 demo/webllm.ts create mode 100644 docs/adr/0002-optional-webllm-adapter.md create mode 100644 playwright.webllm.config.ts create mode 100644 src/adapters/webllm.ts create mode 100644 tests/live/webllm.spec.ts create mode 100644 tests/unit/webllm-adapter.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index f819193..6ba755b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ All notable changes are documented here. +## [0.2.0-alpha.1] - 2026-07-19 + +### Added + +- Optional WebLLM 0.2.84 adapter with lazy provider loading, host-selected model configuration, OpenAI-shaped streaming, usage aggregation, and provider-runtime evidence. +- Abort-safe owner initialization, engine-wide cancellation with stream drain, ordered disposal, and single-generation enforcement. +- Opt-in two-tab Chrome/WebGPU live lab and Playwright gate using a real SmolLM2 model. +- Dedicated `@aantenore/tabloom/adapters/webllm` package export and bundle-isolation smoke assertion. + +### Changed + +- Runtime integration guidance now distinguishes TabLoom's fenced elected-page topology from WebLLM's worker topologies. +- Delivery, compatibility, operations, threat-model, and visual evidence now cover the real provider seam without widening browser or model claims. + ## [0.1.0-alpha.1] - 2026-07-17 ### Added diff --git a/README.md b/README.md index 1c273c9..61eb9d4 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,9 @@ TabLoom is a same-origin browser inference broker. It coordinates sibling pages It supplies the coordination layer, not a model runtime: exclusive ownership, monotonic fencing epochs, protocol validation, bounded admission, streaming sessions, cancellation, timeouts, takeover, and privacy-safe telemetry. -> **Alpha:** the release-gated adapter is deterministic simulation. WebLLM and Transformers.js are documented integration seams, not verified GPU or model evidence. +The core stays provider-neutral. A deterministic adapter exercises lifecycle behavior in every CI job, while a dedicated optional adapter composes [WebLLM 0.2.84](https://github.com/mlc-ai/web-llm) without bundling it into the core package. + +> **Alpha:** the WebLLM path is verified separately on Chrome/WebGPU with an actual model. It is not a compatibility claim for every browser, GPU, model, or future WebLLM version. Transformers.js remains an unverified integration seam. ## Why @@ -34,11 +36,11 @@ flowchart LR ## Install the prerelease archive -The first alpha is distributed as a GitHub release archive rather than an npm registry publication. +The alpha is distributed as a GitHub release archive rather than an npm registry publication. ```bash -curl -LO https://github.com/aantenore/tabloom/releases/download/v0.1.0-alpha.1/tabloom-0.1.0-alpha.1.tgz -pnpm add ./tabloom-0.1.0-alpha.1.tgz +curl -LO https://github.com/aantenore/tabloom/releases/download/v0.2.0-alpha.1/tabloom-0.2.0-alpha.1.tgz +pnpm add ./tabloom-0.2.0-alpha.1.tgz ``` Verify the adjacent `.sha256` asset before installing in a controlled delivery pipeline. @@ -83,6 +85,47 @@ try { The deterministic adapter makes lifecycle behavior reproducible. Replace it with an application adapter for a real runtime; see [adapter integrations](docs/integrations.md). +## Optional WebLLM adapter + +Install the tested peer explicitly, then import only the dedicated subpath: + +```bash +pnpm add @mlc-ai/web-llm@0.2.84 +``` + +```ts +import { createBrowserBroker } from '@aantenore/tabloom'; +import { WebLlmInferenceAdapter } from '@aantenore/tabloom/adapters/webllm'; + +const broker = createBrowserBroker({ + adapter: new WebLlmInferenceAdapter({ + modelId: 'SmolLM2-360M-Instruct-q4f16_1-MLC', + onProgress: ({ progress, text }) => { + console.log(Math.round(progress * 100), text); + }, + }), + config: { + maxConcurrent: 1, + namespace: 'my-app-webllm', + queueCapacity: 4, + requestTimeoutMs: 180_000, + }, +}); + +await broker.start(); +const session = broker.request({ + messages: [{ role: 'user', content: 'Explain fenced ownership.' }], + stream_options: { include_usage: true }, +}); + +for await (const chunk of session) { + console.log(chunk.choices[0]?.delta.content ?? ''); +} +console.log(await session.result); +``` + +The host chooses the model, model source, runtime config, cache policy, and prompt history. The request cannot switch the configured model. Keep `maxConcurrent: 1`: WebLLM interruption is engine-wide and the adapter rejects a competing generation. + ## Session semantics | Concern | Alpha contract | @@ -100,7 +143,7 @@ The deterministic adapter makes lifecycle behavior reproducible. Replace it with Serve from HTTPS, or loopback for development. The alpha requires Web Locks, BroadcastChannel, local storage, and cryptographic UUID support in the same storage partition. -The multi-page suite is locally verified with Playwright 1.61.1 against Chromium 149.0.7827.55, Firefox 151.0, and WebKit 26.5. Each engine exercises one-owner convergence, peer streaming, cancellation, backpressure, and owner takeover. +The deterministic multi-page suite is locally verified with Playwright 1.61.1 against Chromium 149.0.7827.55, Firefox 151.0, and WebKit 26.5. Each engine exercises one-owner convergence, peer streaming, cancellation, backpressure, and owner takeover. The separate live lab targets installed Chrome with WebGPU; see the [compatibility matrix](docs/compatibility.md). ## Development @@ -122,6 +165,14 @@ corepack pnpm test:browser corepack pnpm run audit ``` +Run the opt-in real-model gate only when downloading the configured model is acceptable: + +```bash +TABLOOM_WEBLLM_LIVE=1 \ +TABLOOM_WEBLLM_MODEL=SmolLM2-360M-Instruct-q4f16_1-MLC \ +corepack pnpm test:live:webllm +``` + ## Evidence and boundaries - [Delivery contract](docs/delivery-contract.md) diff --git a/demo/webllm.html b/demo/webllm.html new file mode 100644 index 0000000..953fb00 --- /dev/null +++ b/demo/webllm.html @@ -0,0 +1,263 @@ + + + + + + + + + TabLoom WebLLM live lab + + + +
+
+

WebLLM live lab

+

+ Open one peer tab, then submit from that peer. TabLoom keeps a single + fenced runtime owner while the request and stream cross the + same-origin broker. +

+ +
+ +
+
+
Role
+
stopped
+
+
+
Readiness
+
idle
+
+
+
Runtime evidence
+
provider-runtime
+
+
+
WebGPU
+
checking
+
+
+
Model
+
+
+
+
Peers seen
+
0
+
+
+ +
+

Owner initialization

+

Waiting to start.

+

+ Model initialization runs only in the elected owner tab. A peer + observes broker readiness without loading its own runtime. +

+
+ +
+

Peer request

+ + +
+ +
+

+ Status: + idle +

+ +
+ + +
+ + + diff --git a/demo/webllm.ts b/demo/webllm.ts new file mode 100644 index 0000000..53ff7c2 --- /dev/null +++ b/demo/webllm.ts @@ -0,0 +1,271 @@ +import { WebLlmInferenceAdapter } from '../src/adapters/webllm.js'; +import { createBrowserBroker } from '../src/browser.js'; + +const DEFAULT_MODEL_ID = 'SmolLM2-360M-Instruct-q4f16_1-MLC'; + +const elements = { + epoch: requiredElement('epoch'), + evidence: requiredElement('evidence'), + progressEventCount: requiredElement('progress-event-count'), + modelId: requiredElement('model-id'), + openPeer: requiredButton('open-peer'), + output: requiredOutput('output'), + ownerId: requiredElement('owner-id'), + peerCount: requiredElement('peer-count'), + progress: requiredElement('progress'), + prompt: requiredTextArea('prompt'), + readiness: requiredElement('readiness'), + requestStatus: requiredElement('request-status'), + resultText: requiredElement('result-text'), + role: requiredElement('role'), + send: requiredButton('send'), + tabId: requiredElement('tab-id'), + terminalCount: requiredElement('terminal-count'), + usageTokens: requiredElement('usage-tokens'), + webgpu: requiredElement('webgpu'), +}; + +void runLiveLab(); + +async function runLiveLab(): Promise { + const search = new URLSearchParams(window.location.search); + const namespace = sanitizeNamespace( + search.get('namespace') ?? `tabloom-webllm-${crypto.randomUUID()}`, + ); + const modelId = sanitizeModelId(search.get('model') ?? DEFAULT_MODEL_ID); + const gpuAvailable = Reflect.has(navigator, 'gpu'); + + elements.modelId.textContent = modelId; + elements.webgpu.textContent = gpuAvailable ? 'available' : 'unavailable'; + elements.openPeer.addEventListener('click', () => { + const peerUrl = new URL(window.location.href); + peerUrl.searchParams.set('namespace', namespace); + peerUrl.searchParams.set('model', modelId); + window.open(peerUrl, '_blank', 'noopener'); + }); + + if (!gpuAvailable) { + setStatus('WebGPU is unavailable in this browser.', true); + elements.progress.textContent = 'The provider runtime was not started.'; + return; + } + + try { + let progressEventCount = 0; + const adapter = new WebLlmInferenceAdapter({ + modelId, + onProgress: (progress) => { + progressEventCount += 1; + elements.progressEventCount.textContent = String(progressEventCount); + elements.progress.textContent = progressMessage(progress); + }, + }); + const broker = createBrowserBroker({ + adapter, + config: { + heartbeatIntervalMs: 500, + leaderTimeoutMs: 3_000, + maxConcurrent: 1, + namespace, + queueCapacity: 2, + requestTimeoutMs: 180_000, + }, + }); + let active = false; + let role = broker.snapshot.role; + let readiness = broker.snapshot.readiness; + + const refreshSend = () => { + elements.send.disabled = + active || role !== 'peer' || readiness !== 'ready'; + }; + + const unsubscribe = broker.subscribe((snapshot) => { + role = snapshot.role; + readiness = snapshot.readiness; + elements.role.textContent = snapshot.role; + elements.role.className = `role-${snapshot.role}`; + elements.readiness.textContent = snapshot.readiness; + elements.readiness.className = + snapshot.readiness === 'ready' ? 'status-ready' : ''; + elements.evidence.textContent = snapshot.adapter.evidence; + elements.peerCount.textContent = String(snapshot.knownPeers.length); + elements.ownerId.textContent = + snapshot.role === 'leader' ? snapshot.tabId : (snapshot.leaderId ?? ''); + elements.tabId.textContent = snapshot.tabId; + elements.epoch.textContent = String(snapshot.epoch); + elements.terminalCount.textContent = String(snapshot.terminalCount); + if ( + snapshot.readiness === 'ready' && + elements.progress.textContent === 'Waiting to start.' + ) { + elements.progress.textContent = 'Runtime ready on the elected owner.'; + } + refreshSend(); + }); + + elements.send.addEventListener('click', () => { + if (active || role !== 'peer' || readiness !== 'ready') { + return; + } + const prompt = elements.prompt.value.trim(); + if (prompt.length === 0) { + setStatus('Enter a prompt before submitting.', true); + return; + } + + active = true; + elements.output.textContent = ''; + setStatus('waiting for owner'); + refreshSend(); + + const session = broker.request({ + max_tokens: 32, + messages: [{ content: prompt, role: 'user' }], + stream_options: { include_usage: true }, + temperature: 0, + }); + const resultPromise = session.result; + void resultPromise.catch(() => undefined); + void (async () => { + let streamedText = ''; + try { + setStatus('streaming'); + for await (const chunk of session) { + streamedText += deltaText(chunk); + elements.output.textContent = streamedText; + } + const result = await resultPromise; + elements.resultText.textContent = result.text; + elements.usageTokens.textContent = String( + result.usage?.total_tokens ?? 0, + ); + elements.output.textContent = streamedText || resultText(result); + setStatus('completed'); + } catch (cause) { + setStatus(safeMessage(cause), true); + } finally { + active = false; + refreshSend(); + } + })(); + }); + + window.addEventListener( + 'pagehide', + () => { + unsubscribe(); + void broker.stop(); + }, + { once: true }, + ); + + setStatus('starting broker'); + await broker.start(); + } catch (cause) { + setStatus(safeMessage(cause), true); + elements.progress.textContent = 'The provider runtime could not start.'; + } +} + +function deltaText(chunk: unknown): string { + if (!isRecord(chunk)) { + return ''; + } + const choices = chunk['choices']; + if (!Array.isArray(choices)) { + return ''; + } + const firstChoice: unknown = choices[0]; + if (!isRecord(firstChoice)) { + return ''; + } + const delta = firstChoice['delta']; + if (!isRecord(delta)) { + return ''; + } + const content = delta['content']; + return typeof content === 'string' ? content : ''; +} + +function resultText(result: unknown): string { + if (!isRecord(result)) { + return ''; + } + const text = result['text']; + return typeof text === 'string' ? text : ''; +} + +function progressMessage(progress: unknown): string { + if (typeof progress === 'string') { + return progress; + } + if (!isRecord(progress)) { + return 'Initializing provider runtime.'; + } + + const label = [progress['text'], progress['message']].find( + (value): value is string => typeof value === 'string' && value.length > 0, + ); + const ratio = progress['progress']; + if (typeof ratio === 'number' && Number.isFinite(ratio)) { + const percent = Math.round(Math.max(0, Math.min(1, ratio)) * 100); + return label === undefined ? `${percent}%` : `${label} (${percent}%)`; + } + return label ?? 'Initializing provider runtime.'; +} + +function setStatus(message: string, error = false): void { + elements.requestStatus.textContent = message; + elements.requestStatus.className = error ? 'status-error' : ''; +} + +function safeMessage(cause: unknown): string { + return cause instanceof Error ? cause.message : 'The live request failed.'; +} + +function sanitizeNamespace(value: string): string { + const safe = value.replace(/[^a-zA-Z0-9._-]/gu, '').slice(0, 80); + return safe.length > 0 ? safe : 'tabloom-webllm'; +} + +function sanitizeModelId(value: string): string { + const safe = value.replace(/[^a-zA-Z0-9._/-]/gu, '').slice(0, 160); + return safe.length > 0 ? safe : DEFAULT_MODEL_ID; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function requiredElement(id: string): HTMLElement { + const element = document.getElementById(id); + if (element === null) { + throw new Error(`Missing live lab element: ${id}`); + } + return element; +} + +function requiredButton(id: string): HTMLButtonElement { + const element = requiredElement(id); + if (!(element instanceof HTMLButtonElement)) { + throw new Error(`Live lab element is not a button: ${id}`); + } + return element; +} + +function requiredOutput(id: string): HTMLOutputElement { + const element = requiredElement(id); + if (!(element instanceof HTMLOutputElement)) { + throw new Error(`Live lab element is not an output: ${id}`); + } + return element; +} + +function requiredTextArea(id: string): HTMLTextAreaElement { + const element = requiredElement(id); + if (!(element instanceof HTMLTextAreaElement)) { + throw new Error(`Live lab element is not a textarea: ${id}`); + } + return element; +} diff --git a/docs/adr/0002-optional-webllm-adapter.md b/docs/adr/0002-optional-webllm-adapter.md new file mode 100644 index 0000000..83d3c3a --- /dev/null +++ b/docs/adr/0002-optional-webllm-adapter.md @@ -0,0 +1,29 @@ +# ADR 0002: keep WebLLM behind an optional adapter + +- Status: accepted +- Date: 2026-07-19 + +## Context + +The first TabLoom alpha proved fenced multi-page coordination with a deterministic adapter. Leaving every real runtime as application glue made the central composition claim harder to verify. Bundling a browser model runtime into the core would instead couple protocol evolution to a large, fast-moving provider and impose WebGPU code on consumers that do not use it. + +WebLLM already owns model loading, artifact caching, WebGPU execution, OpenAI-shaped streaming, and worker topologies. Its Service Worker option can preserve a runtime across page visits, but it has a different ownership model from TabLoom's elected live page. + +## Decision + +Ship `WebLlmInferenceAdapter` only through `@aantenore/tabloom/adapters/webllm`. + +- Pin the optional peer and development contract to WebLLM `0.2.84`. +- Import WebLLM lazily during elected-owner initialization. +- Let the host configure the model, engine, cache, chat options, and progress observer. +- Override payload-level model selection, choice count, and streaming mode. +- Allow one active generation per owner; reject a competing run. +- On cancellation, interrupt and drain the provider stream before releasing its lock. +- On disposal, wait for active cleanup and unload the engine before ownership ends. +- Keep deterministic adapters as the repeatable CI conformance authority. + +## Consequences + +Core users do not download or execute WebLLM. Provider users get a maintained composition point and an opt-in real-model test, but compatibility is deliberately narrow: exact provider version, selected model, installed Chrome, WebGPU-capable device, and same-origin secure context. + +WebLLM's Service Worker engine is not nested inside TabLoom. Applications choose either that worker lifecycle or TabLoom's provider-neutral page election according to their operational needs. diff --git a/docs/compatibility.md b/docs/compatibility.md index b96d8cb..6bdecce 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -13,14 +13,28 @@ MDN classifies both [Web Locks](https://developer.mozilla.org/en-US/docs/Web/API ## Release evidence -Playwright 1.61.1 executed the same four multi-page scenarios in every engine on 2026-07-17. +Playwright 1.61.1 executed the same four deterministic multi-page scenarios in every bundled engine on 2026-07-17. The separate provider smoke was executed on 2026-07-19. -| Browser engine | Version | One owner + stream | Cancel + drain | Backpressure | Takeover + single terminal | Real model adapter | -| -------------- | ------------- | ------------------ | -------------- | ------------ | -------------------------- | ------------------ | -| Chromium | 149.0.7827.55 | Pass | Pass | Pass | Pass | Not yet verified | -| Firefox | 151.0 | Pass | Pass | Pass | Pass | Not yet verified | -| WebKit | 26.5 | Pass | Pass | Pass | Pass | Not yet verified | +| Browser engine | Version | One owner + stream | Cancel + drain | Backpressure | Takeover + single terminal | Real model adapter | +| -------------- | ------------- | ------------------ | -------------- | ------------ | -------------------------- | ------------------------- | +| Chromium | 149.0.7827.55 | Pass | Pass | Pass | Pass | See Chrome evidence below | +| Firefox | 151.0 | Pass | Pass | Pass | Pass | Not yet verified | +| WebKit | 26.5 | Pass | Pass | Pass | Pass | Not yet verified | -The deterministic adapter is the only release-gated adapter in the first alpha. WebLLM and Transformers.js remain documented integration seams until dedicated runtime tests are added. +The deterministic adapter remains the repeatable cross-browser release gate. The WebLLM adapter has unit, package, and type gates in the normal suite plus a separate opt-in real-model test. Transformers.js remains a documented, unverified seam. + +## WebLLM provider evidence + +| Field | Verified value | +| -------- | ------------------------------------------------------------------------------------------------------------------ | +| Date | 2026-07-19 | +| Browser | Installed Google Chrome 150.0.0.0, headless Playwright channel | +| WebGPU | Available; adapter acquired; WebLLM reported Apple GPU | +| Provider | `@mlc-ai/web-llm` 0.2.84 | +| Model | `SmolLM2-360M-Instruct-q4f16_1-MLC` | +| Topology | Two same-origin pages, one owner and one peer | +| Result | Provider initialized on the owner; peer received a non-empty stream/result, positive token usage, and one terminal | + +The initial exploratory run completed the provider request but surfaced console noise from an already-running Vite server reoptimizing the newly installed dependency. The final acceptance run started a clean server and passed the complete gate in 46.5 seconds with zero console or page errors. That timing describes this one environment and is not a throughput benchmark. This matrix reports bundled test engines, not every vendor browser build or mobile lifecycle policy. CI reruns the scenarios on every pull request and `main` update. diff --git a/docs/delivery-contract.md b/docs/delivery-contract.md index 875a642..f70a80c 100644 --- a/docs/delivery-contract.md +++ b/docs/delivery-contract.md @@ -26,6 +26,7 @@ Should: - Keep election, transport, clock, identity, telemetry, and inference runtime replaceable. - Supply a visual demo that makes ownership and request lineage understandable. - Document optional WebLLM and Transformers.js adapter seams without bundling either runtime. +- Supply a lazy, optional WebLLM adapter whose model and runtime policy remain host configuration. Out of scope: @@ -37,22 +38,24 @@ Out of scope: ## Requirements and verification -| ID | Requirement | Priority | Acceptance criterion | Verification | -| --- | ------------ | -------- | ----------------------------------------------------------------- | -------------------------- | -| R1 | Single owner | Must | Three live pages expose one owner and two peers | Playwright multi-page test | -| R2 | Fencing | Must | A lower epoch cannot mutate a newer session | Unit and takeover tests | -| R3 | Streaming | Must | A peer receives ordered chunks and one terminal result | Unit and browser tests | -| R4 | Admission | Must | Work beyond configured capacity fails with a typed error | Unit and browser tests | -| R5 | Cancellation | Must | Queued or active work can be cancelled | Unit and browser tests | -| R6 | Timeout | Must | Expired work reaches one terminal timeout state | Unit test | -| R7 | Takeover | Must | Owner closure elects a successor and pending work can complete | Browser test | -| R8 | Versioning | Must | Unsupported protocol traffic is rejected before adapter execution | Unit test | -| R9 | Privacy | Must | Telemetry schemas cannot carry inference content | Type review and unit test | -| R10 | Packaging | Must | ESM package installs and imports in a clean consumer | Package smoke test | +| ID | Requirement | Priority | Acceptance criterion | Verification | +| --- | ------------------ | -------- | --------------------------------------------------------------------------------------------------------- | --------------------------------------- | +| R1 | Single owner | Must | Three live pages expose one owner and two peers | Playwright multi-page test | +| R2 | Fencing | Must | A lower epoch cannot mutate a newer session | Unit and takeover tests | +| R3 | Streaming | Must | A peer receives ordered chunks and one terminal result | Unit and browser tests | +| R4 | Admission | Must | Work beyond configured capacity fails with a typed error | Unit and browser tests | +| R5 | Cancellation | Must | Queued or active work can be cancelled | Unit and browser tests | +| R6 | Timeout | Must | Expired work reaches one terminal timeout state | Unit test | +| R7 | Takeover | Must | Owner closure elects a successor and pending work can complete | Browser test | +| R8 | Versioning | Must | Unsupported protocol traffic is rejected before adapter execution | Unit test | +| R9 | Privacy | Must | Telemetry schemas cannot carry inference content | Type review and unit test | +| R10 | Packaging | Must | ESM package installs and imports in a clean consumer | Package smoke test | +| R11 | Provider isolation | Must | Core and WebLLM subpath import without bundling or eagerly loading the provider | Package smoke and bundle-size assertion | +| R12 | Real runtime seam | Should | One Chrome/WebGPU owner serves a peer request through WebLLM with matching stream/result and one terminal | Opt-in live Playwright test | ## Acceptance threshold -All must-level requirements pass; unit coverage meets configured thresholds; lint, types, builds, package checks, dependency audit, and the supported browser matrix pass; no known critical or high-severity defect remains. Real-runtime compatibility stays documented as unverified until exercised separately. +All must-level requirements pass; unit coverage meets configured thresholds; lint, types, builds, package checks, dependency audit, and the supported deterministic browser matrix pass; no known critical or high-severity defect remains. Real-runtime evidence records the exact browser, provider, model, GPU report, and date instead of expanding into a generic compatibility claim. ## Delivery mode diff --git a/docs/integrations.md b/docs/integrations.md index c233b8b..a6798d0 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -62,18 +62,22 @@ The adapter should: - Avoid logging inputs or chunks through the telemetry port. - Be safe for at-least-once execution after takeover, or enforce idempotency at its own side-effect boundary. -## WebLLM seam +## Shipped WebLLM adapter -[WebLLM](https://github.com/mlc-ai/web-llm) already supplies model loading, caching, WebGPU execution, and OpenAI-shaped chat streaming. A TabLoom adapter can map the contract as follows: +[WebLLM](https://github.com/mlc-ai/web-llm) already supplies model loading, caching, WebGPU execution, and OpenAI-shaped chat streaming. TabLoom ships an optional adapter at `@aantenore/tabloom/adapters/webllm`, pinned to the verified `0.2.84` contract. -| TabLoom hook | WebLLM responsibility | -| -------------------- | ------------------------------------------------------------------------- | -| `initialize(signal)` | Create one engine and select a model in the elected page | -| `run(request, ctx)` | Start a streaming chat completion; emit each delta through `ctx.emit` | -| `ctx.signal` | Stop consuming the stream and call the runtime interruption API if needed | -| `dispose()` | Unload model resources before releasing ownership | +| TabLoom hook | WebLLM responsibility | +| -------------------- | --------------------------------------------------------------------------------- | +| `initialize(signal)` | Create one engine and load the configured model only in the elected page | +| `run(request, ctx)` | Stream OpenAI-shaped chunks, aggregate choice zero, and retain final usage | +| `ctx.signal` | Interrupt generation, drain the provider iterator, then report typed cancellation | +| `dispose()` | Wait for active generation cleanup, unload resources, then release ownership | -This repository does not install WebLLM, choose a model, or claim runtime compatibility in the alpha. Pin a tested WebLLM version in the consuming application and add model-specific browser tests before changing the evidence label. +The package declares WebLLM as an optional exact peer. It uses a lazy import during owner initialization, so importing TabLoom or the adapter subpath does not load or bundle the provider. The host must install the peer, choose the model and cache policy, and provide the complete conversation history on every request. + +The adapter deliberately fixes the configured model, `n = 1`, and streaming mode after merging the request. Keep the broker at `maxConcurrent: 1`, because WebLLM serializes work per model and its interruption API applies to the engine rather than one arbitrary request. + +Do not place `ServiceWorkerMLCEngine` behind this adapter. WebLLM's service-worker topology and TabLoom's elected-page ownership are alternative runtime ownership strategies. TabLoom is useful when the application needs provider-neutral fencing, bounded admission, observable page ownership, and takeover semantics. ## Transformers.js seam @@ -87,3 +91,5 @@ Pipeline input/output shapes differ substantially by task. Keep that schema in t - The core package remains small and does not force WebGPU on consumers. - Provider upgrades do not change the coordination protocol. - The deterministic adapter keeps failure and takeover tests reproducible on CI runners without GPU claims. + +See [ADR 0002](adr/0002-optional-webllm-adapter.md) for the dependency and lifecycle decision. diff --git a/docs/market-scan.md b/docs/market-scan.md index f688b80..831a4be 100644 --- a/docs/market-scan.md +++ b/docs/market-scan.md @@ -1,22 +1,22 @@ # Market and build-vs-buy review -Reviewed 2026-07-17. +Reviewed 2026-07-19. ## Existing building blocks -| Project or API | What it covers | License / status | Gap relative to TabLoom | -| ----------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | ----------------------------------------------- | ---------------------------------------------------------------------------------------- | -| [Web Locks API](https://www.w3.org/TR/web-locks/) | Exclusive resource ownership inside a storage bucket | Web standard; secure context | No inference protocol, epoch store, queue, stream, or client terminal semantics | -| [Broadcast Channel API](https://developer.mozilla.org/en-US/docs/Web/API/Broadcast_Channel_API) | Same-partition message bus | Web standard | No message contract or negotiation | -| [tab-election](https://github.com/dabblewriter/tab-election) | Browser leader election and generic calls | MIT package, v4.6.1 reviewed | Generic ownership; no inference fencing, streaming lifecycle, or privacy contract | -| [broadcast-channel](https://github.com/pubkey/broadcast-channel) | Cross-runtime channel plus leader election | MIT, actively maintained | Broader compatibility than this alpha needs; inference semantics remain application work | -| [WebLLM](https://github.com/mlc-ai/web-llm) | In-browser LLM runtime with WebGPU and worker support | Apache-2.0 | Runtime rather than multi-page ownership protocol | -| [Transformers.js](https://huggingface.co/docs/transformers.js/main/en/index) | Browser inference across many model tasks | Apache-2.0 | Runtime rather than multi-page ownership protocol | -| react-brai | Closed React hook advertised for one WebGPU owner across tabs | Hosted demo; source not published when reviewed | Closest product thesis, but framework-specific and not independently auditable | +| Project or API | What it covers | License / status | Gap relative to TabLoom | +| ----------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | ----------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Web Locks API](https://www.w3.org/TR/web-locks/) | Exclusive resource ownership inside a storage bucket | Web standard; secure context | No inference protocol, epoch store, queue, stream, or client terminal semantics | +| [Broadcast Channel API](https://developer.mozilla.org/en-US/docs/Web/API/Broadcast_Channel_API) | Same-partition message bus | Web standard | No message contract or negotiation | +| [tab-election](https://github.com/dabblewriter/tab-election) | Browser leader election and generic calls | MIT package, v4.6.1 reviewed | Generic ownership; no inference fencing, streaming lifecycle, or privacy contract | +| [broadcast-channel](https://github.com/pubkey/broadcast-channel) | Cross-runtime channel plus leader election | MIT, actively maintained | Broader compatibility than this alpha needs; inference semantics remain application work | +| [WebLLM](https://github.com/mlc-ai/web-llm) | In-browser LLM runtime with WebGPU, Web Worker, and Service Worker support | Apache-2.0; adapter pinned to 0.2.84 | Its Service Worker can retain one runtime across visits, but it does not provide TabLoom's provider-neutral fenced page election, bounded broker admission, or client takeover contract | +| [Transformers.js](https://huggingface.co/docs/transformers.js/main/en/index) | Browser inference across many model tasks | Apache-2.0 | Runtime rather than multi-page ownership protocol | +| react-brai | Closed React hook advertised for one WebGPU owner across tabs | Hosted demo; source not published when reviewed | Closest product thesis, but framework-specific and not independently auditable | ## Decision -Build a narrow protocol layer and reuse browser primitives. Do not build a model runtime and do not bundle a provider. Keep adapters optional so WebLLM, Transformers.js, or a future runtime can be composed at the edge. +Build a narrow protocol layer and reuse browser primitives. Do not build a model runtime and do not bundle a provider. Keep adapters optional so WebLLM, Transformers.js, or a future runtime can be composed at the edge. Ship a thin WebLLM adapter because it proves the seam without absorbing model selection, caching, or provider lifecycle into the broker core. The global `tabloom` package name is occupied by an unrelated React Native table package. This project uses the available `@aantenore/tabloom` scope. The GitHub repository name is available under the `aantenore` account, although unrelated repositories use the same word elsewhere. diff --git a/docs/runbook.md b/docs/runbook.md index 89271f4..d203cbc 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -7,6 +7,8 @@ 3. Verify Web Locks, BroadcastChannel, local storage, and crypto UUID support. 4. Start the broker and wait for an owner announcement before admitting peer work. +For the WebLLM adapter, install exactly `@mlc-ai/web-llm@0.2.84`, keep `maxConcurrent: 1`, and select a model that fits the target device. The first elected owner may need to download model artifacts; peers remain unready until that initialization completes. + ## Monitoring Track safe event counts by type, queue depth, request duration, owner transitions, stale-envelope rejection, and protocol rejection. Do not attach prompts, generated text, token content, or full request objects. @@ -18,10 +20,12 @@ Track safe event counts by type, queue depth, request duration, owner transition - Admission rejection: reduce request rate or increase the validated capacity within the device budget. - Repeated takeover: inspect page suspension, crashes, adapter initialization, and deployment version skew. - Timeout: cancel provider work and surface a recoverable client state. +- WebLLM load failure: inspect WebGPU availability, model compatibility, artifact access, and device limits; do not silently widen execution to a remote provider. +- WebLLM cancellation stall: retain the owner until the interrupted iterator drains and unload completes. ## Rollback -Disable TabLoom at the application composition boundary and instantiate the existing per-page runtime. The broker does not own durable application data in this alpha. +Disable TabLoom at the application composition boundary and instantiate the existing per-page runtime. The broker does not own durable application data in this alpha. Stop the broker before replacing the adapter so the elected owner can interrupt, drain, and unload WebLLM. ## Support evidence diff --git a/docs/threat-model.md b/docs/threat-model.md index 4dd53de..1fcb0ae 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -25,6 +25,7 @@ | Sensitive content reaches diagnostics | Telemetry type excludes payload fields; demo stores counts and lengths only | Application adapters can violate their own logging policy | | Identifier collision | Cryptographic browser IDs by default | Custom ID providers must preserve uniqueness | | Cross-site leakage | Browser origin and storage-partition isolation | Compromised same-origin script remains in scope | +| Provider runtime survives owner loss | Lease loss aborts work; WebLLM adapter interrupts, drains, then unloads | Browser or GPU-driver failure can delay resource reclamation | ## Security non-claims @@ -36,3 +37,4 @@ TabLoom is not a sandbox, authorization service, or isolation boundary for mutua - Older epochs cannot complete a current session. - Capacity, cancellation, timeout, and takeover have regression tests. - Dependency and package audits run in CI. +- Package smoke verifies the optional WebLLM provider is not bundled into the adapter entry. diff --git a/docs/visual-qa.md b/docs/visual-qa.md index f3c5812..5bf008a 100644 --- a/docs/visual-qa.md +++ b/docs/visual-qa.md @@ -1,6 +1,6 @@ # Visual QA ledger -Reviewed 2026-07-17. +Reviewed 2026-07-19. ## Method @@ -31,3 +31,12 @@ Reviewed 2026-07-17. ## Result Pass. The implementation preserves the concept's visual system and primary information architecture while making copy, interactions, accessibility labels, and responsive behavior executable. + +## WebLLM live lab + +The opt-in provider page was inspected at a 1280 × 720 Playwright Chrome viewport with two pages sharing one namespace. + +- Owner view exposed role, readiness, provider evidence, WebGPU availability, selected model, peer count, and provider progress without showing prompt content in diagnostics. +- Peer view kept generation controls disabled until broker readiness, then displayed the exact streamed result and completed terminal state. +- The compact dark surface remained readable with the long model identifier and the generated response wrapped without horizontal overflow. +- No synthetic throughput or memory figure was added; token usage and lifecycle assertions remain hidden test evidence rather than decorative UI metrics. diff --git a/package.json b/package.json index 3228a69..86cab83 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@aantenore/tabloom", - "version": "0.1.0-alpha.1", + "version": "0.2.0-alpha.1", "description": "A fenced, same-origin broker for sharing browser inference across tabs.", "type": "module", "license": "Apache-2.0", @@ -43,6 +43,10 @@ "types": "./dist/browser.d.ts", "import": "./dist/browser.js" }, + "./adapters/webllm": { + "types": "./dist/webllm.d.ts", + "import": "./dist/webllm.js" + }, "./adapters": { "types": "./dist/adapters.d.ts", "import": "./dist/adapters.js" @@ -65,15 +69,25 @@ "test:all": "pnpm test:unit && pnpm test:browser", "test:browser": "playwright test", "test:browser:chromium": "playwright test --project=chromium", + "test:live:webllm": "playwright test --config=playwright.webllm.config.ts", "test:unit": "vitest run --coverage", "typecheck": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.demo.json --noEmit" }, "dependencies": { "zod": "4.4.3" }, + "peerDependencies": { + "@mlc-ai/web-llm": "0.2.84" + }, + "peerDependenciesMeta": { + "@mlc-ai/web-llm": { + "optional": true + } + }, "devDependencies": { - "@eslint/js": "10.0.1", "@arethetypeswrong/cli": "0.18.5", + "@eslint/js": "10.0.1", + "@mlc-ai/web-llm": "0.2.84", "@playwright/test": "1.61.1", "@types/node": "26.1.1", "@types/react": "19.2.17", diff --git a/playwright.webllm.config.ts b/playwright.webllm.config.ts new file mode 100644 index 0000000..f305f1c --- /dev/null +++ b/playwright.webllm.config.ts @@ -0,0 +1,35 @@ +import { defineConfig, devices } from '@playwright/test'; + +export default defineConfig({ + expect: { timeout: 20_000 }, + fullyParallel: false, + outputDir: 'test-results/webllm', + reporter: process.env['CI'] ? [['github'], ['line']] : 'line', + retries: 0, + testDir: './tests/live', + timeout: 900_000, + use: { + baseURL: 'http://127.0.0.1:4173', + screenshot: 'only-on-failure', + trace: 'retain-on-failure', + }, + webServer: { + command: 'pnpm dev', + reuseExistingServer: !process.env['CI'], + timeout: 30_000, + url: 'http://127.0.0.1:4173', + }, + workers: 1, + projects: [ + { + name: 'chrome-webgpu', + use: { + ...devices['Desktop Chrome'], + channel: 'chrome', + launchOptions: { + args: ['--enable-unsafe-webgpu'], + }, + }, + }, + ], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 46b9c4d..ba94417 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,6 +18,9 @@ importers: '@eslint/js': specifier: 10.0.1 version: 10.0.1(eslint@10.0.1) + '@mlc-ai/web-llm': + specifier: 0.2.84 + version: 0.2.84 '@playwright/test': specifier: 1.61.1 version: 1.61.1 @@ -408,6 +411,9 @@ packages: '@loaderkit/resolve@1.0.6': resolution: {integrity: sha512-G8FdIoF5CypfwmD9rl8BXod5HDn8JqB0CCNBXDTaRZ+yRYhARrrSToX1zg1zy9jX3zLqigsELwhT4gNtkdQAUg==} + '@mlc-ai/web-llm@0.2.84': + resolution: {integrity: sha512-hrOWzK4/nGNmgoRKT8pgVmZZ2oEPpbblIWQOwpqNyvK2dysHw3KVB1gNJOuRcQfKOPhucEhX1NJzXzgMDnwSCQ==} + '@napi-rs/wasm-runtime@1.1.6': resolution: {integrity: sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==} peerDependencies: @@ -1291,6 +1297,10 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} + loglevel@1.9.2: + resolution: {integrity: sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==} + engines: {node: '>= 0.6.0'} + lru-cache@11.5.2: resolution: {integrity: sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==} engines: {node: 20 || >=22} @@ -2089,6 +2099,10 @@ snapshots: dependencies: '@braidai/lang': 1.1.2 + '@mlc-ai/web-llm@0.2.84': + dependencies: + loglevel: 1.9.2 + '@napi-rs/wasm-runtime@1.1.6(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)': dependencies: '@emnapi/core': 1.11.1 @@ -2848,6 +2862,8 @@ snapshots: dependencies: p-locate: 5.0.0 + loglevel@1.9.2: {} + lru-cache@11.5.2: {} lru-cache@5.1.1: diff --git a/scripts/package-smoke.mjs b/scripts/package-smoke.mjs index e625f80..8b14adb 100644 --- a/scripts/package-smoke.mjs +++ b/scripts/package-smoke.mjs @@ -3,8 +3,10 @@ import { execFileSync } from 'node:child_process'; import { mkdirSync, mkdtempSync, + readFileSync, readdirSync, rmSync, + statSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -59,18 +61,38 @@ try { } from '@aantenore/tabloom'; import { BrowserBroadcastTransport } from '@aantenore/tabloom/browser'; import { DeterministicInferenceAdapter as AdapterSubpath } from '@aantenore/tabloom/adapters'; + import { WebLlmInferenceAdapter } from '@aantenore/tabloom/adapters/webllm'; assert.equal(typeof TabLoomBroker, 'function'); assert.equal(typeof createBrowserBroker, 'function'); assert.equal(typeof BrowserBroadcastTransport, 'function'); assert.equal(AdapterSubpath, DeterministicInferenceAdapter); assert.equal(new DeterministicInferenceAdapter().descriptor.evidence, 'deterministic-simulation'); + assert.equal(new WebLlmInferenceAdapter({ modelId: 'consumer-model' }).descriptor.evidence, 'provider-runtime'); `, ); execFileSync(process.execPath, ['smoke.mjs'], { cwd: consumerRoot, stdio: 'pipe', }); + const webLlmEntry = join( + consumerRoot, + 'node_modules', + '@aantenore', + 'tabloom', + 'dist', + 'webllm.js', + ); + assert.ok( + statSync(webLlmEntry).size < 64_000, + 'The optional WebLLM runtime was bundled into the adapter entry.', + ); + assert.ok( + /import\(["']@mlc-ai\/web-llm["']\)/u.test( + readFileSync(webLlmEntry, 'utf8'), + ), + 'The optional provider import must remain lazy.', + ); console.log(`Package smoke passed: ${archive}`); } finally { rmSync(temporaryRoot, { force: true, recursive: true }); diff --git a/src/adapters/webllm.ts b/src/adapters/webllm.ts new file mode 100644 index 0000000..b0afc0f --- /dev/null +++ b/src/adapters/webllm.ts @@ -0,0 +1,409 @@ +import type { + ChatCompletionChunk, + ChatCompletionFinishReason, + ChatCompletionRequestBase, + ChatCompletionRequestStreaming, + ChatOptions, + CompletionUsage, + InitProgressReport, + MLCEngineConfig, +} from '@mlc-ai/web-llm'; +import { TabLoomError } from '../core/errors.js'; +import type { InferenceAdapter, InferenceContext } from '../core/types.js'; + +const WEBLLM_VERSION = '0.2.84'; + +export type WebLlmRequest = Readonly< + Omit +>; +export type WebLlmChunk = ChatCompletionChunk; + +export interface WebLlmResult { + readonly chunkCount: number; + readonly finishReason: ChatCompletionFinishReason | null; + readonly text: string; + readonly usage?: CompletionUsage; +} + +export interface WebLlmEngine { + readonly chat: { + readonly completions: { + create( + request: ChatCompletionRequestStreaming, + ): Promise>; + }; + }; + interruptGenerate(): Promise | void; + unload(): Promise; +} + +export type WebLlmEngineFactory = ( + modelId: string, + engineConfig?: MLCEngineConfig, + chatOptions?: ChatOptions, + signal?: AbortSignal, +) => Promise; + +export interface WebLlmAdapterConfig { + readonly chatOptions?: ChatOptions; + readonly engineConfig?: MLCEngineConfig; + readonly engineFactory?: WebLlmEngineFactory; + readonly modelId: string; + readonly onProgress?: (report: InitProgressReport) => void; +} + +type AdapterState = 'idle' | 'initializing' | 'ready' | 'disposing'; + +export class WebLlmInferenceAdapter implements InferenceAdapter< + WebLlmRequest, + WebLlmChunk, + WebLlmResult +> { + readonly descriptor = { + evidence: 'provider-runtime', + id: 'webllm', + name: 'WebLLM browser runtime', + version: WEBLLM_VERSION, + } as const; + + #activeRun = false; + #activeRunDone: Promise | undefined; + #chatOptions: ChatOptions | undefined; + #disposeTask: Promise | undefined; + #engine: WebLlmEngine | undefined; + #engineConfig: MLCEngineConfig | undefined; + #engineFactory: WebLlmEngineFactory; + #initializationController: AbortController | undefined; + #initializationTask: Promise | undefined; + #lifecycle = 0; + #modelId: string; + #onProgress: ((report: InitProgressReport) => void) | undefined; + #state: AdapterState = 'idle'; + + constructor(config: WebLlmAdapterConfig) { + if (config.modelId.trim().length === 0) { + throw new TabLoomError( + 'INVALID_CONFIG', + 'A non-empty WebLLM model ID is required.', + ); + } + this.#chatOptions = config.chatOptions; + this.#engineConfig = config.engineConfig; + this.#engineFactory = config.engineFactory ?? createWebLlmEngine; + this.#modelId = config.modelId; + this.#onProgress = config.onProgress; + } + + async initialize(signal: AbortSignal): Promise { + if (signal.aborted) { + throw cancelled('WebLLM initialization was cancelled.'); + } + if (this.#state === 'ready') { + return; + } + if (this.#state !== 'idle') { + throw new TabLoomError( + 'ADAPTER_FAILED', + 'WebLLM initialization is already in progress.', + { state: this.#state }, + ); + } + + this.#state = 'initializing'; + const lifecycle = ++this.#lifecycle; + const controller = new AbortController(); + this.#initializationController = controller; + const onAbort = () => controller.abort(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + if (isAborted(signal)) { + onAbort(); + } + const engineConfig = this.#createEngineConfig(lifecycle, signal); + const initializationTask = this.#initializeEngine( + lifecycle, + controller.signal, + engineConfig, + ); + this.#initializationTask = initializationTask; + try { + await initializationTask; + } finally { + signal.removeEventListener('abort', onAbort); + if (this.#initializationTask === initializationTask) { + this.#initializationController = undefined; + this.#initializationTask = undefined; + } + if (this.#hasState('initializing') && lifecycle === this.#lifecycle) { + this.#state = 'idle'; + } + } + } + + async run( + request: WebLlmRequest, + context: InferenceContext, + ): Promise { + const engine = this.#engine; + if (this.#state !== 'ready' || engine === undefined) { + throw new TabLoomError( + 'CAPABILITY_UNAVAILABLE', + 'WebLLM is not initialized on the current owner.', + ); + } + if (this.#activeRun) { + throw new TabLoomError( + 'ADAPTER_FAILED', + 'WebLLM only accepts one active generation per owner.', + { reason: 'concurrent-run' }, + ); + } + if (context.signal.aborted) { + throw cancelled('WebLLM generation was cancelled.'); + } + + this.#activeRun = true; + let completeRun: () => void = () => undefined; + this.#activeRunDone = new Promise((resolve) => { + completeRun = resolve; + }); + const onAbort = () => interruptQuietly(engine); + context.signal.addEventListener('abort', onAbort, { once: true }); + + try { + const stream = await engine.chat.completions.create({ + ...request, + model: this.#modelId, + n: 1, + stream: true, + }); + let chunkCount = 0; + let finishReason: ChatCompletionFinishReason | null = null; + let text = ''; + let usage: CompletionUsage | undefined; + + for await (const chunk of stream) { + if (chunk.usage !== undefined) { + usage = chunk.usage; + } + if (isAborted(context.signal) || !this.#hasState('ready')) { + continue; + } + const choice = chunk.choices.find((candidate) => candidate.index === 0); + if (choice === undefined) { + continue; + } + context.emit(chunk); + chunkCount += 1; + text += choice.delta.content ?? ''; + if (choice.finish_reason != null) { + finishReason = choice.finish_reason; + } + } + + if (isAborted(context.signal) || !this.#hasState('ready')) { + throw cancelled('WebLLM generation was cancelled.'); + } + return usage === undefined + ? { chunkCount, finishReason, text } + : { chunkCount, finishReason, text, usage }; + } catch (error) { + if (isAborted(context.signal) || !this.#hasState('ready')) { + throw cancelled('WebLLM generation was cancelled.'); + } + throw adapterFailure('WebLLM generation failed.', error); + } finally { + context.signal.removeEventListener('abort', onAbort); + this.#activeRun = false; + this.#activeRunDone = undefined; + completeRun(); + } + } + + dispose(): Promise { + if (this.#disposeTask !== undefined) { + return this.#disposeTask; + } + const disposeTask = this.#dispose(); + this.#disposeTask = disposeTask; + const clear = () => { + if (this.#disposeTask === disposeTask) { + this.#disposeTask = undefined; + } + }; + void disposeTask.then(clear, clear); + return disposeTask; + } + + async #dispose(): Promise { + ++this.#lifecycle; + this.#state = 'disposing'; + this.#initializationController?.abort(); + const initializationTask = this.#initializationTask; + const engine = this.#engine; + this.#engine = undefined; + const activeRunDone = this.#activeRunDone; + if (engine !== undefined && this.#activeRun) { + interruptQuietly(engine); + } + try { + if (activeRunDone !== undefined) { + await activeRunDone; + } + if (initializationTask !== undefined) { + try { + await initializationTask; + } catch { + // The initialization caller owns its typed failure after cleanup settles. + } + } + await engine?.unload(); + } catch (error) { + throw adapterFailure('WebLLM disposal failed.', error); + } finally { + this.#state = 'idle'; + } + } + + async #initializeEngine( + lifecycle: number, + signal: AbortSignal, + engineConfig: MLCEngineConfig | undefined, + ): Promise { + let engine: WebLlmEngine; + try { + engine = await this.#engineFactory( + this.#modelId, + engineConfig, + this.#chatOptions, + signal, + ); + } catch (error) { + if (isAborted(signal) || lifecycle !== this.#lifecycle) { + throw cancelled('WebLLM initialization was cancelled.'); + } + throw adapterFailure('WebLLM initialization failed.', error); + } + + if ( + isAborted(signal) || + lifecycle !== this.#lifecycle || + !this.#hasState('initializing') + ) { + await unloadQuietly(engine); + throw cancelled('WebLLM initialization was cancelled.'); + } + this.#engine = engine; + this.#state = 'ready'; + } + + #createEngineConfig( + lifecycle: number, + signal: AbortSignal, + ): MLCEngineConfig | undefined { + const onProgress = + this.#onProgress ?? this.#engineConfig?.initProgressCallback; + if (onProgress === undefined) { + return this.#engineConfig; + } + return { + ...this.#engineConfig, + initProgressCallback: (report) => { + if (!signal.aborted && lifecycle === this.#lifecycle) { + onProgress(report); + } + }, + }; + } + + #hasState(state: AdapterState): boolean { + return this.#state === state; + } +} + +async function createWebLlmEngine( + modelId: string, + engineConfig?: MLCEngineConfig, + chatOptions?: ChatOptions, + signal?: AbortSignal, +): Promise { + const { MLCEngine } = await import('@mlc-ai/web-llm'); + const engine = new MLCEngine(engineConfig); + if (isAborted(signal)) { + await unloadQuietly(engine); + throw cancelled('WebLLM initialization was cancelled.'); + } + + let resolveAbort: () => void = () => undefined; + const aborted = new Promise((resolve) => { + resolveAbort = resolve; + }); + let interruptingUnload: Promise | undefined; + const onAbort = () => { + interruptingUnload ??= unloadQuietly(engine); + resolveAbort(); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + const reloadTask = engine.reload(modelId, chatOptions); + const reloadOutcome = reloadTask.then( + () => ({ kind: 'reloaded' }) as const, + (error: unknown) => ({ error, kind: 'failed' }) as const, + ); + try { + const outcome = await Promise.race([ + reloadOutcome, + aborted.then(() => ({ kind: 'aborted' }) as const), + ]); + if (outcome.kind === 'aborted' || isAborted(signal)) { + await interruptingUnload; + await reloadOutcome; + await unloadQuietly(engine); + throw cancelled('WebLLM initialization was cancelled.'); + } + if (outcome.kind === 'failed') { + await unloadQuietly(engine); + throw outcome.error; + } + return engine; + } finally { + signal?.removeEventListener('abort', onAbort); + } +} + +function isAborted(signal: AbortSignal | undefined): boolean { + return signal?.aborted === true; +} + +function adapterFailure(message: string, error: unknown): TabLoomError { + if (error instanceof TabLoomError) { + return error; + } + return new TabLoomError( + 'ADAPTER_FAILED', + message, + {}, + error instanceof Error ? { cause: error } : undefined, + ); +} + +function cancelled(message: string): TabLoomError { + return new TabLoomError('CANCELLED', message); +} + +function interruptQuietly(engine: WebLlmEngine): void { + try { + const interruption = engine.interruptGenerate(); + if (interruption !== undefined) { + void interruption.catch(() => undefined); + } + } catch { + // Cancellation remains authoritative even when the provider cannot interrupt. + } +} + +async function unloadQuietly(engine: WebLlmEngine): Promise { + try { + await engine.unload(); + } catch { + // A superseded initialization must never restore or retain its engine. + } +} diff --git a/tests/live/webllm.spec.ts b/tests/live/webllm.spec.ts new file mode 100644 index 0000000..78502e8 --- /dev/null +++ b/tests/live/webllm.spec.ts @@ -0,0 +1,165 @@ +import { expect, test, type BrowserContext, type Page } from '@playwright/test'; + +const DEFAULT_MODEL_ID = 'SmolLM2-360M-Instruct-q4f16_1-MLC'; +const LIVE_ENABLED = process.env['TABLOOM_WEBLLM_LIVE'] === '1'; +const MODEL_ID = + process.env['TABLOOM_WEBLLM_MODEL']?.trim() || DEFAULT_MODEL_ID; + +interface ObservedPage { + readonly errors: string[]; + readonly page: Page; +} + +test.describe('TabLoom WebLLM live evidence', () => { + test.skip( + !LIVE_ENABLED, + 'Set TABLOOM_WEBLLM_LIVE=1 to download and run the live model.', + ); + + test('serves a peer request from one provider runtime owner', async ({ + context, + }, testInfo) => { + const startedAt = Date.now(); + const cluster = await openCluster( + context, + namespace(testInfo.title), + MODEL_ID, + ); + + await expect + .poll(() => roles(cluster), { timeout: 600_000 }) + .toEqual(['leader', 'peer']); + await Promise.all( + cluster.map(async ({ page }) => { + await expect(page.getByTestId('webgpu')).toHaveText('available'); + await expect(page.getByTestId('readiness')).toHaveText('ready', { + timeout: 600_000, + }); + await expect(page.getByTestId('evidence')).toHaveText( + 'provider-runtime', + ); + }), + ); + + const ownerIds = await Promise.all( + cluster.map(async ({ page }) => + page.getByTestId('owner-id').textContent(), + ), + ); + expect(ownerIds[0]).not.toBe(''); + expect(new Set(ownerIds).size).toBe(1); + const progressEventCounts = await Promise.all( + cluster.map(async ({ page }) => numberText(page, 'progress-event-count')), + ); + expect(progressEventCounts.filter((count) => count > 0)).toHaveLength(1); + + const peer = await pageWithRole(cluster, 'peer'); + await peer.page + .getByTestId('prompt') + .fill('Respond with one short word confirming readiness.'); + await peer.page.getByTestId('send').click(); + await expect(peer.page.getByTestId('request-status')).toHaveText( + 'completed', + { timeout: 180_000 }, + ); + await expect + .poll( + async () => + (await peer.page.getByTestId('output').textContent())?.trim() + .length ?? 0, + { timeout: 180_000 }, + ) + .toBeGreaterThan(0); + await expect(peer.page.getByTestId('terminal-count')).toHaveText('1'); + await expect + .poll(() => numberText(peer.page, 'usage-tokens')) + .toBeGreaterThan(0); + expect(await peer.page.getByTestId('output').textContent()).toBe( + await peer.page.getByTestId('result-text').textContent(), + ); + + expect(cluster.flatMap((item) => item.errors)).toEqual([]); + const owner = await pageWithRole(cluster, 'leader'); + await testInfo.attach('runtime-evidence.json', { + body: Buffer.from( + JSON.stringify( + { + browserVersion: context.browser()?.version() ?? 'unknown', + durationMs: Date.now() - startedAt, + modelId: MODEL_ID, + ownerProgress: await owner.page + .getByTestId('progress') + .textContent(), + progressEventCounts, + providerVersion: '0.2.84', + terminalCount: await numberText(peer.page, 'terminal-count'), + usageTokens: await numberText(peer.page, 'usage-tokens'), + webgpu: await owner.page.getByTestId('webgpu').textContent(), + }, + null, + 2, + ), + ), + contentType: 'application/json', + }); + }); +}); + +async function openCluster( + context: BrowserContext, + brokerNamespace: string, + modelId: string, +): Promise { + const cluster: ObservedPage[] = []; + for (let index = 0; index < 2; index += 1) { + const page = await context.newPage(); + const errors: string[] = []; + page.on('console', (message) => { + if (message.type() === 'error') { + errors.push(message.text()); + } + }); + page.on('pageerror', (error) => errors.push(error.message)); + const search = new URLSearchParams({ + model: modelId, + namespace: brokerNamespace, + }); + await page.goto(`/webllm.html?${search.toString()}`); + await expect(page).toHaveTitle('TabLoom WebLLM live lab'); + cluster.push({ errors, page }); + } + return cluster; +} + +async function numberText(page: Page, testId: string): Promise { + return Number(await page.getByTestId(testId).textContent()); +} + +async function roles(cluster: readonly ObservedPage[]): Promise { + return ( + await Promise.all( + cluster.map(async ({ page }) => page.getByTestId('role').textContent()), + ) + ) + .map((role) => role ?? '') + .sort(); +} + +async function pageWithRole( + cluster: readonly ObservedPage[], + role: 'leader' | 'peer', +): Promise { + for (const item of cluster) { + if ((await item.page.getByTestId('role').textContent()) === role) { + return item; + } + } + throw new Error(`No ${role} page was found.`); +} + +function namespace(title: string): string { + return `live-${title + .toLowerCase() + .replace(/[^a-z0-9]+/gu, '-') + .slice(0, 32)}-${Date.now()}`; +} diff --git a/tests/unit/webllm-adapter.test.ts b/tests/unit/webllm-adapter.test.ts new file mode 100644 index 0000000..7152256 --- /dev/null +++ b/tests/unit/webllm-adapter.test.ts @@ -0,0 +1,355 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + WebLlmInferenceAdapter, + type WebLlmChunk, + type WebLlmEngine, + type WebLlmEngineFactory, +} from '../../src/adapters/webllm.js'; + +describe('WebLLM adapter', () => { + it('initializes the configured model and aggregates typed stream chunks', async () => { + const usage = { + completion_tokens: 2, + extra: { + decode_tokens_per_s: 10, + e2e_latency_s: 0.2, + prefill_tokens_per_s: 20, + time_per_output_token_s: 0.1, + time_to_first_token_s: 0.1, + }, + prompt_tokens: 1, + total_tokens: 3, + }; + const chunks = [ + makeChunk('Hel'), + makeChunk('lo', 'stop'), + makeUsageChunk(usage), + ]; + const fake = createFakeEngine(async function* () { + await Promise.resolve(); + yield* chunks; + }); + const onProgress = vi.fn(); + const factory = vi.fn((_modelId, engineConfig) => { + engineConfig?.initProgressCallback?.({ + progress: 0.5, + text: 'loading', + timeElapsed: 1, + }); + return Promise.resolve(fake.engine); + }); + const adapter = new WebLlmInferenceAdapter({ + chatOptions: { context_window_size: 512 }, + engineConfig: {}, + engineFactory: factory, + modelId: 'model-a', + onProgress, + }); + + await adapter.initialize(new AbortController().signal); + await adapter.initialize(new AbortController().signal); + const emitted: WebLlmChunk[] = []; + const result = await adapter.run( + { messages: [{ content: 'hello', role: 'user' }] }, + context((chunk) => emitted.push(chunk)), + ); + + expect(adapter.descriptor).toMatchObject({ + evidence: 'provider-runtime', + version: '0.2.84', + }); + expect(factory).toHaveBeenCalledOnce(); + expect(factory.mock.calls[0]?.[0]).toBe('model-a'); + expect(factory.mock.calls[0]?.[2]).toEqual({ context_window_size: 512 }); + expect(onProgress).toHaveBeenCalledOnce(); + expect(fake.requests[0]).toMatchObject({ + model: 'model-a', + n: 1, + stream: true, + }); + expect(emitted).toEqual(chunks.slice(0, 2)); + expect(result).toEqual({ + chunkCount: 2, + finishReason: 'stop', + text: 'Hello', + usage, + }); + + await adapter.dispose(); + expect(fake.unload).toHaveBeenCalledOnce(); + }); + + it('requires a model and owner initialization before generation', async () => { + expect(() => new WebLlmInferenceAdapter({ modelId: ' ' })).toThrowError( + expect.objectContaining({ code: 'INVALID_CONFIG' }), + ); + + const adapter = new WebLlmInferenceAdapter({ + engineFactory: () => Promise.resolve(createFakeEngine().engine), + modelId: 'model-a', + }); + await expect( + adapter.run({ messages: [] }, context()), + ).rejects.toMatchObject({ code: 'CAPABILITY_UNAVAILABLE' }); + }); + + it('keeps disposal pending until a cancelled late engine is unloaded', async () => { + const pending = deferred(); + const unloadStarted = deferred(); + const releaseUnload = deferred(); + const fake = createFakeEngine(); + fake.unload.mockImplementation(() => { + unloadStarted.resolve(); + return releaseUnload.promise; + }); + const adapter = new WebLlmInferenceAdapter({ + engineFactory: () => pending.promise, + modelId: 'model-a', + }); + const controller = new AbortController(); + const first = adapter.initialize(controller.signal); + + await expect( + adapter.initialize(new AbortController().signal), + ).rejects.toMatchObject({ code: 'ADAPTER_FAILED' }); + controller.abort(); + const initialization = expect(first).rejects.toMatchObject({ + code: 'CANCELLED', + }); + let disposed = false; + const disposal = adapter.dispose().then(() => { + disposed = true; + }); + await Promise.resolve(); + expect(disposed).toBe(false); + + pending.resolve(fake.engine); + await unloadStarted.promise; + expect(disposed).toBe(false); + releaseUnload.resolve(); + await initialization; + await disposal; + expect(disposed).toBe(true); + expect(fake.unload).toHaveBeenCalledOnce(); + }); + + it('rejects a second generation while one is active', async () => { + const release = deferred(); + const started = deferred(); + const fake = createFakeEngine(async function* () { + yield makeChunk('one'); + await release.promise; + yield makeChunk(' two', 'stop'); + }); + const adapter = await initializedAdapter(fake.engine); + const first = adapter.run( + { messages: [{ content: 'go', role: 'user' }] }, + context(() => started.resolve()), + ); + await started.promise; + + await expect( + adapter.run({ messages: [] }, context()), + ).rejects.toMatchObject({ + code: 'ADAPTER_FAILED', + details: { reason: 'concurrent-run' }, + }); + release.resolve(); + await expect(first).resolves.toMatchObject({ text: 'one two' }); + }); + + it('interrupts WebLLM when the owner cancels generation', async () => { + const release = deferred(); + const started = deferred(); + const fake = createFakeEngine(async function* () { + yield makeChunk('partial'); + await release.promise; + }); + fake.interrupt.mockImplementation(() => release.resolve()); + const adapter = await initializedAdapter(fake.engine); + const controller = new AbortController(); + const run = adapter.run( + { messages: [{ content: 'go', role: 'user' }] }, + context(() => started.resolve(), controller.signal), + ); + await started.promise; + controller.abort(); + + await expect(run).rejects.toMatchObject({ code: 'CANCELLED' }); + expect(fake.interrupt).toHaveBeenCalledOnce(); + }); + + it('drains an active generation before disposal unloads the engine', async () => { + const release = deferred(); + const started = deferred(); + const fake = createFakeEngine(async function* () { + yield makeChunk('partial'); + await release.promise; + yield makeChunk('discarded', 'abort'); + }); + fake.interrupt.mockImplementation(() => release.resolve()); + const adapter = await initializedAdapter(fake.engine); + const run = adapter.run( + { messages: [{ content: 'go', role: 'user' }] }, + context(() => started.resolve()), + ); + await started.promise; + + await adapter.dispose(); + await expect(run).rejects.toMatchObject({ code: 'CANCELLED' }); + expect(fake.interrupt).toHaveBeenCalledOnce(); + expect(fake.unload).toHaveBeenCalledOnce(); + }); + + it('shares one disposal task across concurrent callers', async () => { + const releaseUnload = deferred(); + const fake = createFakeEngine(); + fake.unload.mockImplementation(() => releaseUnload.promise); + const adapter = await initializedAdapter(fake.engine); + + const first = adapter.dispose(); + const second = adapter.dispose(); + expect(second).toBe(first); + await Promise.resolve(); + expect(fake.unload).toHaveBeenCalledOnce(); + releaseUnload.resolve(); + await Promise.all([first, second]); + }); + + it('wraps provider failures and reports unload failures', async () => { + const generationFailure = createFakeEngine(() => failingStream()); + const adapter = await initializedAdapter(generationFailure.engine); + await expect( + adapter.run({ messages: [] }, context()), + ).rejects.toMatchObject({ code: 'ADAPTER_FAILED' }); + + generationFailure.unload.mockRejectedValueOnce( + new Error('provider unload failed'), + ); + await expect(adapter.dispose()).rejects.toMatchObject({ + code: 'ADAPTER_FAILED', + }); + }); + + it('fails immediately for an already aborted initialization or run', async () => { + const controller = new AbortController(); + controller.abort(); + const adapter = new WebLlmInferenceAdapter({ + engineFactory: () => Promise.resolve(createFakeEngine().engine), + modelId: 'model-a', + }); + await expect(adapter.initialize(controller.signal)).rejects.toMatchObject({ + code: 'CANCELLED', + }); + + const ready = await initializedAdapter(createFakeEngine().engine); + await expect( + ready.run({ messages: [] }, context(undefined, controller.signal)), + ).rejects.toMatchObject({ code: 'CANCELLED' }); + }); +}); + +function context( + emit: ((chunk: WebLlmChunk) => void) | undefined = undefined, + signal = new AbortController().signal, +) { + return { + attempt: 1, + emit: emit ?? (() => undefined), + epoch: 1, + requestId: 'request-1', + signal, + }; +} + +function createFakeEngine( + stream: () => AsyncIterable = emptyStream, +) { + const requests: Array< + Parameters[0] + > = []; + const interrupt = vi.fn<() => void>(); + const unload = vi.fn<() => Promise>(() => Promise.resolve()); + const engine: WebLlmEngine = { + chat: { + completions: { + create: (request) => { + requests.push(request); + return Promise.resolve(stream()); + }, + }, + }, + interruptGenerate: interrupt, + unload, + }; + return { engine, interrupt, requests, unload }; +} + +async function initializedAdapter( + engine: WebLlmEngine, +): Promise { + const adapter = new WebLlmInferenceAdapter({ + engineFactory: () => Promise.resolve(engine), + modelId: 'model-a', + }); + await adapter.initialize(new AbortController().signal); + return adapter; +} + +function emptyStream(): AsyncIterable { + return { + [Symbol.asyncIterator]: () => ({ + next: () => Promise.resolve({ done: true, value: undefined }), + }), + }; +} + +function failingStream(): AsyncIterable { + return { + [Symbol.asyncIterator]: () => ({ + next: () => Promise.reject(new Error('provider generation failed')), + }), + }; +} + +function makeChunk( + content: string, + finishReason: 'abort' | 'length' | 'stop' | 'tool_calls' | null = null, + usage?: WebLlmChunk['usage'], +): WebLlmChunk { + const chunk: WebLlmChunk = { + choices: [ + { + delta: { content }, + finish_reason: finishReason, + index: 0, + }, + ], + created: 1, + id: 'chunk-1', + model: 'model-a', + object: 'chat.completion.chunk', + }; + return usage === undefined ? chunk : { ...chunk, usage }; +} + +function makeUsageChunk(usage: NonNullable): WebLlmChunk { + return { + choices: [], + created: 1, + id: 'chunk-1', + model: 'model-a', + object: 'chat.completion.chunk', + usage, + }; +} + +function deferred() { + let resolve: (value: T) => void = () => undefined; + let reject: (error: unknown) => void = () => undefined; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, reject, resolve }; +} diff --git a/tsconfig.json b/tsconfig.json index 1730714..751a5b6 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -30,8 +30,10 @@ "demo", "scripts", "tests/e2e", + "tests/live", "tests/unit", "playwright.config.ts", + "playwright.webllm.config.ts", "vite.config.ts", "vitest.config.ts", "tsup.config.ts" diff --git a/tsup.config.ts b/tsup.config.ts index 274bd80..53452c5 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -7,6 +7,7 @@ export default defineConfig({ adapters: 'src/adapters.ts', browser: 'src/browser.ts', index: 'src/index.ts', + webllm: 'src/adapters/webllm.ts', }, format: ['esm'], minify: false,