diff --git a/docs-site/guide/bytecode.md b/docs-site/guide/bytecode.md index efc2cba3..a08dbc1b 100644 --- a/docs-site/guide/bytecode.md +++ b/docs-site/guide/bytecode.md @@ -54,6 +54,16 @@ pkg --no-bytecode --public-packages "*" --public index.js `--public` additionally exposes the **top-level project** sources (i.e. your own code) as plain text. +## Fallback to source on failure + +When bytecode generation fails for a specific file (e.g. during cross-compilation without QEMU), `pkg` logs a warning and **skips the file** — it won't be available at runtime. If you'd rather ship the affected files as plain source instead of skipping them, pass `--fallback-to-source`: + +```sh +pkg --fallback-to-source -t node22-linux-arm64 index.js +``` + +Files that compile successfully still ship as bytecode; only the ones that fail are included as plain JavaScript. A warning is emitted for each file that falls back. + ## SEA mode SEA mode **never uses bytecode**. Source is always plaintext in a SEA binary. This is a deliberate trade-off — see [SEA vs Standard](/guide/sea-vs-standard). diff --git a/docs-site/guide/getting-started.md b/docs-site/guide/getting-started.md index 1ecc0334..2a6b2982 100644 --- a/docs-site/guide/getting-started.md +++ b/docs-site/guide/getting-started.md @@ -111,24 +111,25 @@ pkg . ## CLI reference -| Flag | Short | Description | -| -------------------------- | ----- | ------------------------------------------------------------------------------------------------ | -| `--help` | `-h` | Show usage | -| `--version` | `-v` | Print pkg version | -| `--targets ` | `-t` | Comma-separated target list, e.g. `node22-linux-x64` — see [Targets](/guide/targets) | -| `--config ` | `-c` | Path to `package.json` or any JSON file with a top-level `pkg` config | -| `--output ` | `-o` | Output file name (single-target builds only) | -| `--out-path ` | | Output directory for multi-target builds | -| `--debug` | `-d` | Verbose packaging log — see [Output & debug](/guide/output) | -| `--build` | `-b` | Compile base binaries from source instead of downloading — see [Build](/guide/build) | -| `--public` | | Speed up packaging and disclose top-level sources | -| `--public-packages ` | | Force listed packages to be treated as public — see [Bytecode](/guide/bytecode) | -| `--no-bytecode` | | Skip V8 bytecode compilation, embed source as plain JS — see [Bytecode](/guide/bytecode) | -| `--no-native-build` | | Skip building native addons | -| `--no-dict ` | | Ignore bundled dictionaries for listed packages (`*` disables all) | -| `--options ` | | Bake V8 options into the executable — see [CLI options](/guide/options) | -| `--compress ` | `-C` | Compress the embedded filesystem with `Brotli` or `GZip` — see [Compression](/guide/compression) | -| `--sea` | | Use Node.js SEA instead of the patched base binary — see [SEA mode](/guide/sea-mode) | +| Flag | Short | Description | +| -------------------------- | ----- | -------------------------------------------------------------------------------------------------------------------- | +| `--help` | `-h` | Show usage | +| `--version` | `-v` | Print pkg version | +| `--targets ` | `-t` | Comma-separated target list, e.g. `node22-linux-x64` — see [Targets](/guide/targets) | +| `--config ` | `-c` | Path to `package.json` or any JSON file with a top-level `pkg` config | +| `--output ` | `-o` | Output file name (single-target builds only) | +| `--out-path ` | | Output directory for multi-target builds | +| `--debug` | `-d` | Verbose packaging log — see [Output & debug](/guide/output) | +| `--build` | `-b` | Compile base binaries from source instead of downloading — see [Build](/guide/build) | +| `--public` | | Speed up packaging and disclose top-level sources | +| `--public-packages ` | | Force listed packages to be treated as public — see [Bytecode](/guide/bytecode) | +| `--no-bytecode` | | Skip V8 bytecode compilation, embed source as plain JS — see [Bytecode](/guide/bytecode) | +| `--fallback-to-source` | | Ship files as plain source when bytecode generation fails instead of skipping them — see [Bytecode](/guide/bytecode) | +| `--no-native-build` | | Skip building native addons | +| `--no-dict ` | | Ignore bundled dictionaries for listed packages (`*` disables all) | +| `--options ` | | Bake V8 options into the executable — see [CLI options](/guide/options) | +| `--compress ` | `-C` | Compress the embedded filesystem with `Brotli` or `GZip` — see [Compression](/guide/compression) | +| `--sea` | | Use Node.js SEA instead of the patched base binary — see [SEA mode](/guide/sea-mode) | Run `pkg --help` at any time for the live list of options. diff --git a/docs-site/guide/targets.md b/docs-site/guide/targets.md index 2d0ffe6b..ee3cce10 100644 --- a/docs-site/guide/targets.md +++ b/docs-site/guide/targets.md @@ -83,7 +83,8 @@ Tracked in [#87](https://github.com/yao-pkg/pkg/issues/87) and [#181](https://gi 1. **Switch to SEA** — `pkg . --sea`. Avoids the V8 bytecode step entirely. 2. **Disable bytecode** — `pkg . --no-bytecode --public-packages "*" --public`. Keeps Standard mode, stores source as plaintext. -3. **Target Node 24** — the regression is gone on `node24-*` targets. +3. **Fallback to source** — `pkg . --fallback-to-source`. Keeps bytecode for files that compile successfully and ships the rest as plain source. See [Bytecode → Fallback to source](/guide/bytecode#fallback-to-source-on-failure). +4. **Target Node 24** — the regression is gone on `node24-*` targets. ::: @@ -95,6 +96,7 @@ Regardless of the bug above, the V8 bytecode fabricator in Standard mode needs t - **macOS** — you can build `x64` on `arm64` with Rosetta 2, but not the opposite - **Windows** — you can build `x64` on `arm64` with x64 emulation, but not the opposite - Or disable bytecode generation entirely with `--no-bytecode --public-packages "*" --public` +- Or use `--fallback-to-source` to ship only the failing files as plain source while keeping bytecode for the rest Enhanced SEA doesn't have this limitation when the host and target share the same Node major: pkg uses `process.execPath` to generate the SEA blob, so no target-arch interpreter is needed. Cross-major SEA builds (e.g. building `node22-*` targets on a Node 24 host) still require an interpreter for the downloaded target binary. diff --git a/lib/help.ts b/lib/help.ts index b0aa3580..88312e22 100644 --- a/lib/help.ts +++ b/lib/help.ts @@ -19,6 +19,7 @@ export default function help() { --public-packages force specified packages to be considered public --no-bytecode skip bytecode generation and include source files as plain js --no-native-build skip native addons build + --fallback-to-source if bytecode generation fails for a file, ship it as plain source instead of skipping it --no-dict comma-separated list of packages names to ignore dictionaries. Use --no-dict * to disable all dictionaries -C, --compress [default=None] compression algorithm = Brotli or GZip --sea (Experimental) compile give file using node's SEA feature. Requires node v20.0.0 or higher and only single file is supported diff --git a/lib/index.ts b/lib/index.ts index bac3dd34..c38d3444 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -236,6 +236,7 @@ export async function exec(argv2: string[]) { 'native-build', 'd', 'debug', + 'fallback-to-source', 'h', 'help', 'public', @@ -692,6 +693,7 @@ export async function exec(argv2: string[]) { symLinks, doCompress, nativeBuild, + fallbackToSource: argv['fallback-to-source'], }); if (target.platform !== 'win' && target.output) { diff --git a/lib/packer.ts b/lib/packer.ts index 66908280..598d6881 100644 --- a/lib/packer.ts +++ b/lib/packer.ts @@ -65,6 +65,7 @@ interface PackerOptions { export interface Stripe { snap: string; + skip?: boolean; store: number; file?: string; buffer?: Buffer; diff --git a/lib/producer.ts b/lib/producer.ts index ef999a5e..379bb433 100644 --- a/lib/producer.ts +++ b/lib/producer.ts @@ -304,6 +304,7 @@ interface ProducerOptions { symLinks: SymLinks; doCompress: CompressType; nativeBuild: boolean; + fallbackToSource?: boolean; } /** @@ -368,6 +369,7 @@ export default function producer({ symLinks, doCompress, nativeBuild, + fallbackToSource, }: ProducerOptions) { return new Promise((resolve, reject) => { if (!Buffer.alloc) { @@ -443,7 +445,7 @@ export default function producer({ } if (count === 2) { - if (prevStripe) { + if (prevStripe && !prevStripe.skip) { const { store } = prevStripe; let { snap } = prevStripe; snap = snapshotify(snap, slash); @@ -464,25 +466,6 @@ export default function producer({ const snap = snapshotify(stripe.snap, slash); const sourceBuffer = stripe.buffer; - // Fall back to shipping source for this file (as if it had - // been packed with --no-bytecode). The previous behaviour was - // to emit an empty stripe, which left the VFS entry with - // neither STORE_BLOB nor STORE_CONTENT and blew up at runtime - // with "Error: UNEXPECTED-20" (#87, #181). - const fallbackToContent = (reason: string) => { - log.warn( - `Failed to generate V8 bytecode for ${ - stripe.file ?? snap - }. Shipping source instead. Cause: ${reason}`, - ); - stripe.store = STORE_CONTENT; - stripe.buffer = sourceBuffer; - return cb( - null, - pipeMayCompressToNewMeter(intoStream(sourceBuffer)), - ); - }; - return fabricateTwice( bakes, target.fabricator, @@ -490,7 +473,26 @@ export default function producer({ sourceBuffer, (error, buffer) => { if (error) { - return fallbackToContent(error.message); + const file = stripe.file ?? snap; + + if (fallbackToSource) { + log.warn( + `Failed to generate V8 bytecode for ${file}. Shipping source instead. Cause: ${error.message}`, + ); + stripe.store = STORE_CONTENT; + stripe.buffer = sourceBuffer; + return cb( + null, + pipeMayCompressToNewMeter(intoStream(sourceBuffer)), + ); + } + + log.warn( + `Failed to generate V8 bytecode for ${file}. Cause: ${error.message}. ` + + `Use --fallback-to-source to include the file as plain source instead.`, + ); + stripe.skip = true; + return cb(null, intoStream(Buffer.alloc(0))); } cb( diff --git a/prelude/bootstrap.js b/prelude/bootstrap.js index 6d6c343a..4935b296 100644 --- a/prelude/bootstrap.js +++ b/prelude/bootstrap.js @@ -955,19 +955,15 @@ function payloadFileSync(pointer) { if (entityBlob) { return cb2(null, Buffer.from('source-code-not-available')); } - // why return empty buffer? - // otherwise this error will arise: - // Error: UNEXPECTED-20 - // at readFileFromSnapshot (e:0) - // at Object.fs.readFileSync (e:0) - // at Object.Module._extensions..js (module.js:421:20) - // at Module.load (module.js:357:32) - // at Function.Module._load (module.js:314:12) - // at Function.Module.runMain (e:0) - // at startup (node.js:140:18) - // at node.js:1001:3 - - return cb2(new Error('UNEXPECTED-20')); + return cb2( + new Error( + '[pkg] UNEXPECTED-20: no source or bytecode for ' + + path_ + + '. This usually means V8 bytecode generation failed during ' + + 'packaging (e.g. cross-compilation without QEMU). Rebuild with ' + + '--fallback-to-source, --no-bytecode, or --sea to fix this.', + ), + ); } fs.readFileSync = function readFileSync(path_, options_) {