Add gearhash, xetchunk and blake3 WASM packages (#1500)

This PR is an experiment to see what a pure-wasm
[gearhash](https://github.com/srijs/rust-gearhash) looks like, using
[https://www.assemblyscript.org](AssemblyScript), a language purpose
made to generate WASM.

Generating WASM from rust uses a lot of glue, the glue for assembly
script seems a lot thinner:

```js
async function instantiate(module, imports = {}) {
  const { exports } = await WebAssembly.instantiate(module, imports);
  return exports;
}
export const {
  memory,
  add,
} = await (async url => instantiate(
  await (async () => {
    const isNodeOrBun = typeof process != "undefined" && process.versions != null && (process.versions.node != null || process.versions.bun != null);
    if (isNodeOrBun) { return globalThis.WebAssembly.compile(await (await import("node:fs/promises")).readFile(url)); }
    else { return await globalThis.WebAssembly.compileStreaming(globalThis.fetch(url)); }
  })(), {
  }
))(new URL("debug.wasm", import.meta.url));
```

It will also be interesting to compare wasm size between pure wasm &
from-rust wasm, as well as provide `gearhash` to the wider JS/NPM
ecosystem.

`gearhash` is a core component of the xet protocol chunking & hashing
and a cool piece of tech!

cc @Wauplin @hanouticelina @assafvayner @rajatarya for viz

Note: also added `xetChunk` and `blake3` packages

Author: coyotte508

.github/workflows/blake3-wasm-publish.yml

@@ -0,0 +1,64 @@
+name: Blake3 WASM - Version and Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      newversion:
+        type: choice
+        description: "Semantic Version Bump Type"
+        default: patch
+        options:
+          - patch
+          - minor
+          - major
+
+concurrency:
+  group: "push-to-main"
+
+defaults:
+  run:
+    working-directory: packages/blake3-wasm
+
+jobs:
+  version_and_release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # Needed to push the tag and the commit on the main branch, otherwise we get:
+          # > Run git push --follow-tags
+          # remote: error: GH006: Protected branch update failed for refs/heads/main.
+          # remote: error: Changes must be made through a pull request. Required status check "lint" is expected.
+          token: ${{ secrets.BOT_ACCESS_TOKEN }}
+      - run: npm install -g corepack@latest && corepack enable
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          cache: "pnpm"
+          cache-dependency-path: |
+            packages/blake3-wasm/pnpm-lock.yaml
+          # setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token.  REQUIRED
+          registry-url: "https://registry.npmjs.org"
+      - run: pnpm install
+      - run: git config --global user.name machineuser
+      - run: git config --global user.email [email protected]
+      - run: |
+          PACKAGE_VERSION=$(node -p "require('./package.json').version")
+          BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")
+          # Update package.json with the new version
+          node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
+          git commit . -m "🔖 @huggingface/blake3-wasm $BUMPED_VERSION"
+          git tag "blake3-wasm-v$BUMPED_VERSION"
+      - run: pnpm publish --no-git-checks .
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+      - run: (git pull --rebase && git push --follow-tags) || (git pull --rebase && git push --follow-tags)
+      # hack - reuse actions/setup-node@v3 just to set a new registry
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          registry-url: "https://npm.pkg.github.com"
+      # Disable for now, until github supports PATs for writing github packages (https://github.com/github/roadmap/issues/558)
+      # - run: pnpm publish --no-git-checks .
+      #   env:
+      #     NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/gearhash-wasm-publish.yml

@@ -0,0 +1,64 @@
+name: Gearhash WASM - Version and Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      newversion:
+        type: choice
+        description: "Semantic Version Bump Type"
+        default: patch
+        options:
+          - patch
+          - minor
+          - major
+
+concurrency:
+  group: "push-to-main"
+
+defaults:
+  run:
+    working-directory: packages/gearhash-wasm
+
+jobs:
+  version_and_release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # Needed to push the tag and the commit on the main branch, otherwise we get:
+          # > Run git push --follow-tags
+          # remote: error: GH006: Protected branch update failed for refs/heads/main.
+          # remote: error: Changes must be made through a pull request. Required status check "lint" is expected.
+          token: ${{ secrets.BOT_ACCESS_TOKEN }}
+      - run: npm install -g corepack@latest && corepack enable
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          cache: "pnpm"
+          cache-dependency-path: |
+            packages/gearhash-wasm/pnpm-lock.yaml
+          # setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token.  REQUIRED
+          registry-url: "https://registry.npmjs.org"
+      - run: pnpm install
+      - run: git config --global user.name machineuser
+      - run: git config --global user.email [email protected]
+      - run: |
+          PACKAGE_VERSION=$(node -p "require('./package.json').version")
+          BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")
+          # Update package.json with the new version
+          node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
+          git commit . -m "🔖 @huggingface/gearhash-wasm $BUMPED_VERSION"
+          git tag "gearhash-wasm-v$BUMPED_VERSION"
+      - run: pnpm publish --no-git-checks .
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+      - run: (git pull --rebase && git push --follow-tags) || (git pull --rebase && git push --follow-tags)
+      # hack - reuse actions/setup-node@v3 just to set a new registry
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          registry-url: "https://npm.pkg.github.com"
+      # Disable for now, until github supports PATs for writing github packages (https://github.com/github/roadmap/issues/558)
+      # - run: pnpm publish --no-git-checks .
+      #   env:
+      #     NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/splitmix64-wasm-publish.yml

@@ -0,0 +1,64 @@
+name: Splitmix64 WASM - Version and Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      newversion:
+        type: choice
+        description: "Semantic Version Bump Type"
+        default: patch
+        options:
+          - patch
+          - minor
+          - major
+
+concurrency:
+  group: "push-to-main"
+
+defaults:
+  run:
+    working-directory: packages/splitmix64-wasm
+
+jobs:
+  version_and_release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # Needed to push the tag and the commit on the main branch, otherwise we get:
+          # > Run git push --follow-tags
+          # remote: error: GH006: Protected branch update failed for refs/heads/main.
+          # remote: error: Changes must be made through a pull request. Required status check "lint" is expected.
+          token: ${{ secrets.BOT_ACCESS_TOKEN }}
+      - run: npm install -g corepack@latest && corepack enable
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          cache: "pnpm"
+          cache-dependency-path: |
+            packages/splitmix64-wasm/pnpm-lock.yaml
+          # setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token.  REQUIRED
+          registry-url: "https://registry.npmjs.org"
+      - run: pnpm install
+      - run: git config --global user.name machineuser
+      - run: git config --global user.email [email protected]
+      - run: |
+          PACKAGE_VERSION=$(node -p "require('./package.json').version")
+          BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")
+          # Update package.json with the new version
+          node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
+          git commit . -m "🔖 @huggingface/splitmix64-wasm $BUMPED_VERSION"
+          git tag "splitmix64-wasm-v$BUMPED_VERSION"
+      - run: pnpm publish --no-git-checks .
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+      - run: (git pull --rebase && git push --follow-tags) || (git pull --rebase && git push --follow-tags)
+      # hack - reuse actions/setup-node@v3 just to set a new registry
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          registry-url: "https://npm.pkg.github.com"
+      # Disable for now, until github supports PATs for writing github packages (https://github.com/github/roadmap/issues/558)
+      # - run: pnpm publish --no-git-checks .
+      #   env:
+      #     NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/xetchunk-wasm-publish.yml

@@ -0,0 +1,64 @@
+name: Xetchunk WASM - Version and Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      newversion:
+        type: choice
+        description: "Semantic Version Bump Type"
+        default: patch
+        options:
+          - patch
+          - minor
+          - major
+
+concurrency:
+  group: "push-to-main"
+
+defaults:
+  run:
+    working-directory: packages/xetchunk-wasm
+
+jobs:
+  version_and_release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # Needed to push the tag and the commit on the main branch, otherwise we get:
+          # > Run git push --follow-tags
+          # remote: error: GH006: Protected branch update failed for refs/heads/main.
+          # remote: error: Changes must be made through a pull request. Required status check "lint" is expected.
+          token: ${{ secrets.BOT_ACCESS_TOKEN }}
+      - run: npm install -g corepack@latest && corepack enable
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          cache: "pnpm"
+          cache-dependency-path: |
+            packages/xetchunk-wasm/pnpm-lock.yaml
+          # setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token.  REQUIRED
+          registry-url: "https://registry.npmjs.org"
+      - run: pnpm install
+      - run: git config --global user.name machineuser
+      - run: git config --global user.email [email protected]
+      - run: |
+          PACKAGE_VERSION=$(node -p "require('./package.json').version")
+          BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")
+          # Update package.json with the new version
+          node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
+          git commit . -m "🔖 @huggingface/xetchunk-wasm $BUMPED_VERSION"
+          git tag "xetchunk-wasm-v$BUMPED_VERSION"
+      - run: pnpm publish --no-git-checks .
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+      - run: (git pull --rebase && git push --follow-tags) || (git pull --rebase && git push --follow-tags)
+      # hack - reuse actions/setup-node@v3 just to set a new registry
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          registry-url: "https://npm.pkg.github.com"
+      # Disable for now, until github supports PATs for writing github packages (https://github.com/github/roadmap/issues/558)
+      # - run: pnpm publish --no-git-checks .
+      #   env:
+      #     NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

packages/blake3-wasm/BENCHMARK.md

@@ -0,0 +1,107 @@
+# BLAKE3 Performance Benchmark
+
+This benchmark measures the throughput (MB/s) of the BLAKE3 hashing implementation when processing random data of various sizes.
+
+## Features
+
+- **Multiple data sizes**: Tests from 1 KB to 100 MB
+- **Three hashing methods**:
+  - **Single-shot**: Direct `blake3(data)` calls
+  - **Streaming**: Using `createHasher()` with single update
+  - **Chunked**: Simulating large file processing with 64KB chunks
+- **Automatic iteration adjustment**: More iterations for smaller data sizes
+- **Warm-up runs**: Ensures consistent performance measurements
+- **Detailed reporting**: Shows time, throughput, and summary
+
+## Usage
+
+### Run the benchmark:
+
+```bash
+pnpm run bench
+```
+
+### From Node.js:
+
+```javascript
+import { runBenchmark } from "./tests/bench.js";
+
+const results = runBenchmark();
+```
+
+### Individual size benchmark:
+
+```javascript
+import { benchmarkSize } from "./tests/bench.js";
+
+const result = benchmarkSize(1000 * 1000, 10); // 1MB, 10 iterations
+console.log(result);
+```
+
+## Output Format
+
+The benchmark provides:
+
+- **Per-size results**: Time and throughput for each data size
+- **Summary table**: Comparison across all sizes and methods
+- **Best performance**: Highlights the fastest method and size
+
+Example output:
+
+```
+  BLAKE3 Performance Benchmark
+============================================================
+
+📊 Benchmarking 1.0 KB data (100 iterations, 100.0 KB total)
+────────────────────────────────────────────────────────────
+🔹 Single-shot: 12.65ms (7.90 MB/s)
+🔹 Streaming:  11.94ms (8.37 MB/s)
+🔹 Chunked:    12.44ms (8.04 MB/s)
+
+📊 Benchmarking 64.0 KB data (100 iterations, 6.4 MB total)
+────────────────────────────────────────────────────────────
+🔹 Single-shot: 701.26ms (9.13 MB/s)
+🔹 Streaming:  688.19ms (9.30 MB/s)
+🔹 Chunked:    703.23ms (9.10 MB/s)
+
+📈 SUMMARY
+============================================================
+Data Size    | Single-shot | Streaming  | Chunked
+────────────────────────────────────────────────────────────
+1.0 KB       | 7.90 MB/s    | 8.37 MB/s    | 8.04 MB/s
+64.0 KB      | 9.13 MB/s    | 9.30 MB/s    | 9.10 MB/s
+
+🏆 BEST PERFORMANCE
+────────────────────────────────────────────────────────────
+Method: Streaming
+Data Size: 64.0 KB
+Throughput: 9.30 MB/s
+```
+
+## Throughput Units
+
+The benchmark uses decimal units (power of 1000) for consistency:
+
+- **MB/s**: Megabytes per second (1,000,000 bytes/second)
+- **GB/s**: Gigabytes per second (1,000,000,000 bytes/second)
+
+## Data Sizes Tested
+
+- **1 KB**: Small data performance
+- **64 KB**: Medium data performance
+- **1 MB**: Large data performance
+- **10 MB**: Very large data performance
+- **100 MB**: Massive data performance
+
+## Iteration Counts
+
+- **Small data** (< 1 MB): 100 iterations for statistical accuracy
+- **Medium data** (1-10 MB): 10 iterations for reasonable runtime
+- **Large data** (> 10 MB): 3 iterations to avoid excessive runtime
+
+## Notes
+
+- Random data is generated for each test to ensure realistic performance
+- Warm-up runs are performed before timing to ensure consistent results
+- All measurements use `performance.now()` for high-precision timing
+- The benchmark automatically adjusts iterations based on data size