diff --git a/src/node/internal/events.ts b/src/node/internal/events.ts index 3c9130017bd..95e581dde16 100644 --- a/src/node/internal/events.ts +++ b/src/node/internal/events.ts @@ -146,7 +146,10 @@ export class EventEmitterAsyncResource } } -export function addAbortListener(signal: AbortSignal, listener: any) { +export function addAbortListener( + signal: AbortSignal | undefined, + listener: any +) { if (signal === undefined) { throw new ERR_INVALID_ARG_TYPE('signal', 'AbortSignal', signal); } diff --git a/src/node/internal/internal_http_client.ts b/src/node/internal/internal_http_client.ts index d3d79b049ef..d84ad28a103 100644 --- a/src/node/internal/internal_http_client.ts +++ b/src/node/internal/internal_http_client.ts @@ -43,7 +43,7 @@ import { import { OutgoingMessage } from 'node-internal:internal_http_outgoing'; import { Agent, globalAgent } from 'node-internal:internal_http_agent'; import type { IncomingMessageCallback } from 'node-internal:internal_http_util'; -import type { Socket } from 'net'; +import type { Socket } from 'node:net'; const INVALID_PATH_REGEX = /[^\u0021-\u00ff]/; @@ -168,7 +168,7 @@ export class ClientRequest extends OutgoingMessage implements _ClientRequest { const signal = options.signal; if (signal) { - addAbortSignal(signal, this); + addAbortSignal(signal, this as unknown as Writable); } let method = options.method; const methodIsString = typeof method === 'string'; diff --git a/src/node/internal/process.d.ts b/src/node/internal/process.d.ts index 4c587dbdc66..00aed95170a 100644 --- a/src/node/internal/process.d.ts +++ b/src/node/internal/process.d.ts @@ -8,7 +8,9 @@ export const platform: string; declare global { const Cloudflare: { - readonly compatibilityFlags: Record; + readonly compatibilityFlags: Record & { + enable_streams_nodejs_v24_compat: boolean; + }; }; } diff --git a/src/node/internal/public_process.ts b/src/node/internal/public_process.ts index ecdc356dc24..7651a171089 100644 --- a/src/node/internal/public_process.ts +++ b/src/node/internal/public_process.ts @@ -61,13 +61,12 @@ function chunkToBuffer( // For stdout, we emulate `nohup node foo.js` class SyncWriteStream extends Writable { fd: number; - override readable: boolean; + override readable: boolean = false; _type = 'fs'; _isStdio = true; constructor(fd: number) { super({ autoDestroy: true }); this.fd = fd; - this.readable = false; } override _write( chunk: string | Buffer | ArrayBufferView | DataView, diff --git a/src/node/internal/streams_add_abort_signal.ts b/src/node/internal/streams_add_abort_signal.ts index 94456674c84..de8edcbb43a 100644 --- a/src/node/internal/streams_add_abort_signal.ts +++ b/src/node/internal/streams_add_abort_signal.ts @@ -23,41 +23,85 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. -import { validateAbortSignal } from 'node-internal:validators'; -import { isNodeStream } from 'node-internal:streams_util'; -import { eos } from 'node-internal:streams_end_of_stream'; import { AbortError, ERR_INVALID_ARG_TYPE, } from 'node-internal:internal_errors'; -import { addAbortListener } from 'node-internal:events'; +import { + isNodeStream, + isWebStream, + kControllerErrorFunction, +} from 'node-internal:streams_util'; + +import { eos } from 'node-internal:streams_end_of_stream'; import type { Readable } from 'node-internal:streams_readable'; import type { Writable } from 'node-internal:streams_writable'; import type { Transform } from 'node-internal:streams_transform'; +import { addAbortListener } from 'node-internal:events'; + +// This method is inlined here for readable-stream +// It also does not allow for signal to not exist on the stream +// https://github.com/nodejs/node/pull/36061#discussion_r533718029 +function validateAbortSignal( + signal: unknown, + name: string +): asserts signal is AbortSignal { + if (signal == null || typeof signal !== 'object' || !('aborted' in signal)) { + throw new ERR_INVALID_ARG_TYPE(name, 'AbortSignal', signal); + } +} -type NodeStream = Readable | Writable | Transform; +type StreamType = + | Readable + | Writable + | Transform + | ReadableStream + | WritableStream; -export function addAbortSignal void }>( +export function addAbortSignal( signal: unknown, stream: T ): T { validateAbortSignal(signal, 'signal'); - if (!isNodeStream(stream)) { - throw new ERR_INVALID_ARG_TYPE('stream', 'stream.Stream', stream); - } - const onAbort = (): void => { - stream.destroy( - new AbortError(undefined, { - cause: signal.reason, - }) + if (!isNodeStream(stream) && !isWebStream(stream)) { + throw new ERR_INVALID_ARG_TYPE( + 'stream', + ['ReadableStream', 'WritableStream', 'Stream'], + stream ); - }; + } + return addAbortSignalNoValidate(signal, stream); +} + +export function addAbortSignalNoValidate( + signal: AbortSignal | null | undefined, + stream: T +): T { + if (signal == null || typeof signal !== 'object' || !('aborted' in signal)) { + return stream; + } + const onAbort = isNodeStream(stream) + ? (): void => { + stream.destroy(new AbortError(undefined, { cause: signal.reason })); + } + : (): void => { + ( + stream as ReadableStream & { + [kControllerErrorFunction]: (err: Error) => void; + } + )[kControllerErrorFunction]( + new AbortError(undefined, { cause: signal.reason }) + ); + }; if (signal.aborted) { onAbort(); } else { const disposable = addAbortListener(signal, onAbort); - eos(stream as NodeStream, disposable[Symbol.dispose]); + eos( + stream as Readable | Writable | Transform, + disposable[Symbol.dispose] as () => void + ); } return stream; } diff --git a/src/node/internal/streams_destroy.ts b/src/node/internal/streams_destroy.ts index d4887053a24..102b011bf1d 100644 --- a/src/node/internal/streams_destroy.ts +++ b/src/node/internal/streams_destroy.ts @@ -23,21 +23,34 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. +/* eslint-disable @typescript-eslint/no-redundant-type-constituents, @typescript-eslint/no-unsafe-call */ + import { nextTick } from 'node-internal:internal_process'; + +import type { Writable, WritableState } from 'node-internal:streams_writable'; +import type { Readable, ReadableState } from 'node-internal:streams_readable'; + import { AbortError, aggregateTwoErrors, ERR_MULTIPLE_CALLBACK, } from 'node-internal:internal_errors'; import { + kIsDestroyed, isDestroyed, isFinished, isServerRequest, + kState, + kErrorEmitted, + kEmitClose, + kClosed, + kCloseEmitted, + kConstructed, kDestroyed, + kAutoDestroy, + kErrored, } from 'node-internal:streams_util'; -import { isRequest } from 'node-internal:streams_end_of_stream'; -import type { Writable, WritableState } from 'node-internal:streams_writable'; -import type { Readable, ReadableState } from 'node-internal:streams_readable'; +import type { OutgoingMessage } from 'node-internal:internal_http_outgoing'; const kConstruct = Symbol('kConstruct'); const kDestroy = Symbol('kDestroy'); @@ -60,17 +73,22 @@ function checkError( } } +// Backwards compat. cb() is undocumented and unused in core but +// unfortunately might be used by modules. export function destroy( this: Readable | Writable, - err: Error, - cb: VoidFunction + err?: Error, + cb?: VoidFunction // @ts-expect-error TS2526 Returning this is not allowed. ): this { const r = this._readableState; const w = this._writableState; // With duplex streams we use the writable side for state. const s = w || r; - if ((w && w.destroyed) || (r && r.destroyed)) { + if ( + (w && (w[kState] & kDestroyed) !== 0) || + (r && (r[kState] & kDestroyed) !== 0) + ) { if (typeof cb === 'function') { cb(); } @@ -81,14 +99,15 @@ export function destroy( // to make it re-entrance safe in case destroy() is called within callbacks checkError(err, w, r); if (w) { - w.destroyed = true; + w[kState] |= kDestroyed; } if (r) { - r.destroyed = true; + r[kState] |= kDestroyed; } // If still constructing then defer calling _destroy. - if (!s?.constructed) { + // @ts-expect-error TS18048 `s` will always be defined here. + if ((s[kState] & kConstructed) === 0) { this.once(kDestroy, function (this: Readable | Writable, er: Error) { _destroy(this, aggregateTwoErrors(er, err), cb); }); @@ -100,27 +119,31 @@ export function destroy( function _destroy( self: Readable | Writable, - err: Error | null, - cb: (err?: Error | null) => void + err?: Error, + cb?: (err?: Error | null) => void ): void { let called = false; + function onDestroy(err?: Error | null): void { if (called) { return; } called = true; + const r = self._readableState; const w = self._writableState; checkError(err, w, r); if (w) { - w.closed = true; + w[kState] |= kClosed; } if (r) { - r.closed = true; + r[kState] |= kClosed; } + if (typeof cb === 'function') { cb(err); } + if (err) { nextTick(emitErrorCloseNT, self, err); } else { @@ -143,12 +166,15 @@ function emitCloseNT(self: Readable | Writable): void { const r = self._readableState; const w = self._writableState; if (w) { - w.closeEmitted = true; + w[kState] |= kCloseEmitted; } if (r) { - r.closeEmitted = true; + r[kState] |= kCloseEmitted; } - if ((w && w.emitClose) || (r && r.emitClose)) { + if ( + (w && (w[kState] & kEmitClose) !== 0) || + (r && (r[kState] & kEmitClose) !== 0) + ) { self.emit('close'); } } @@ -156,14 +182,18 @@ function emitCloseNT(self: Readable | Writable): void { function emitErrorNT(self: Readable | Writable, err: Error): void { const r = self._readableState; const w = self._writableState; - if ((w && w.errorEmitted) || (r && r.errorEmitted)) { + if ( + (w && (w[kState] & kErrorEmitted) !== 0) || + (r && (r[kState] & kErrorEmitted) !== 0) + ) { return; } + if (w) { - w.errorEmitted = true; + w[kState] |= kErrorEmitted; } if (r) { - r.errorEmitted = true; + r[kState] |= kErrorEmitted; } self.emit('error', err); } @@ -201,7 +231,8 @@ export function errorOrDestroy( stream: Readable | Writable, err?: Error, sync: boolean = false -): void { + // @ts-expect-error TS2526 Apparently `this` is disallowed. +): this | undefined { // We have tests that rely on errors being emitted // in the same tick, so changing this is semver major. // For now when you opt-in to autoDestroy we allow @@ -210,18 +241,26 @@ export function errorOrDestroy( const r = stream._readableState; const w = stream._writableState; - if ((w && w.destroyed) || (r && r.destroyed)) { - return; + if ( + (w && (w[kState] ? (w[kState] & kDestroyed) !== 0 : w.destroyed)) || + (r && (r[kState] ? (r[kState] & kDestroyed) !== 0 : r.destroyed)) + ) { + // @ts-expect-error TS2683 This should be somehow type-defined. + return this; } - if ((r && r.autoDestroy) || (w && w.autoDestroy)) stream.destroy(err); - else if (err) { + if ( + (r && (r[kState] & kAutoDestroy) !== 0) || + (w && (w[kState] & kAutoDestroy) !== 0) + ) { + stream.destroy(err); + } else if (err) { // Avoid V8 leak, https://github.com/nodejs/node/pull/34103#issuecomment-652002364 err.stack; // eslint-disable-line @typescript-eslint/no-unused-expressions - if (w && !w.errored) { + if (w && (w[kState] & kErrored) === 0) { w.errored = err; } - if (r && !r.errored) { + if (r && (r[kState] & kErrored) === 0) { r.errored = err; } if (sync) { @@ -230,6 +269,8 @@ export function errorOrDestroy( emitErrorNT(stream, err); } } + + return undefined; } export function construct(stream: Readable | Writable, cb: VoidFunction): void { @@ -238,64 +279,78 @@ export function construct(stream: Readable | Writable, cb: VoidFunction): void { } const r = stream._readableState; const w = stream._writableState; + if (r) { - r.constructed = false; + r[kState] &= ~kConstructed; } if (w) { - w.constructed = false; + w[kState] &= ~kConstructed; } + stream.once(kConstruct, cb); + if (stream.listenerCount(kConstruct) > 1) { // Duplex return; } + nextTick(constructNT, stream); } -function constructNT(stream: Readable | Writable): void { +function constructNT(this: unknown, stream: Readable | Writable): void { let called = false; - function onConstruct(err: Error | null | undefined): void { + + function onConstruct(err?: Error): void { if (called) { - errorOrDestroy( - stream, - err !== null && err !== undefined ? err : new ERR_MULTIPLE_CALLBACK() - ); + errorOrDestroy(stream, err ?? new ERR_MULTIPLE_CALLBACK()); return; } called = true; + const r = stream._readableState; const w = stream._writableState; const s = w || r; + if (r) { - r.constructed = true; + r[kState] |= kConstructed; } if (w) { - w.constructed = true; + w[kState] |= kConstructed; } + if (s?.destroyed) { stream.emit(kDestroy, err); } else if (err) { errorOrDestroy(stream, err, true); } else { - nextTick(emitConstructNT, stream); + stream.emit(kConstruct); } } + try { - stream._construct?.(onConstruct); + stream._construct?.((err) => { + nextTick(onConstruct, err); + }); } catch (err) { - onConstruct(err as Error); + nextTick(onConstruct, err); } } -function emitConstructNT(stream: Readable | Writable): void { - stream.emit(kConstruct); +function isRequest(stream: unknown): stream is OutgoingMessage { + return ( + stream != null && + typeof stream === 'object' && + 'setHeader' in stream && + 'abort' in stream && + typeof stream.abort === 'function' + ); } function emitCloseLegacy(stream: Readable | Writable): void { stream.emit('close'); } -function emitErrorCloseLegacy(stream: Readable | Writable, err: Error): void { +function emitErrorCloseLegacy(stream: Readable | Writable, err?: Error): void { stream.emit('error', err); nextTick(emitCloseLegacy, stream); } @@ -308,6 +363,7 @@ export function destroyer( if (!stream || isDestroyed(stream)) { return; } + if (!err && !isFinished(stream)) { err = new AbortError(); } @@ -318,20 +374,22 @@ export function destroyer( stream.socket = null; stream.destroy(err); } else if (isRequest(stream)) { + // @ts-expect-error TS2339 - abort exists on OutgoingMessage but not in types stream.abort(); } else if (isRequest(stream.req)) { + // @ts-expect-error TS2339 - abort exists on req but not in all types stream.req.abort(); } else if (typeof stream.destroy === 'function') { stream.destroy(err); } else if ('close' in stream && typeof stream.close === 'function') { // TODO: Don't lose err? - stream.close(); // eslint-disable-line @typescript-eslint/no-unsafe-call + stream.close(); } else if (err) { nextTick(emitErrorCloseLegacy, stream, err); } else { nextTick(emitCloseLegacy, stream); } if (!stream.destroyed) { - stream[kDestroyed] = true; + stream[kIsDestroyed] = true; } } diff --git a/src/node/internal/streams_duplex.js b/src/node/internal/streams_duplex.js index a60864b30d8..a0a01e2361b 100644 --- a/src/node/internal/streams_duplex.js +++ b/src/node/internal/streams_duplex.js @@ -38,11 +38,8 @@ import { import { ok as assert } from 'node-internal:internal_assert'; import { Stream } from 'node-internal:streams_legacy'; import { nextTick } from 'node-internal:internal_process'; - import { validateBoolean, validateObject } from 'node-internal:validators'; - import { normalizeEncoding } from 'node-internal:internal_utils'; - import { addAbortSignal } from 'node-internal:streams_add_abort_signal'; import { @@ -57,13 +54,11 @@ import { isDuplexNodeStream, kOnConstructed, } from 'node-internal:streams_util'; - -import { eos } from 'node-internal:streams_end_of_stream'; - import { - destroyer, construct as destroyConstruct, + destroyer, } from 'node-internal:streams_destroy'; +import { eos } from 'node-internal:streams_end_of_stream'; import { AbortError, @@ -87,8 +82,7 @@ Object.setPrototypeOf(Duplex, Readable); // Allow the keys array to be GC'ed. for (let i = 0; i < keys.length; i++) { const method = keys[i]; - if (!Duplex.prototype[method]) - Duplex.prototype[method] = Writable.prototype[method]; + Duplex.prototype[method] ||= Writable.prototype[method]; } } @@ -170,39 +164,50 @@ Duplex.prototype.destroy = Writable.prototype.destroy; Object.defineProperties(Duplex.prototype, { writable: { + __proto__: null, ...Object.getOwnPropertyDescriptor(Writable.prototype, 'writable'), }, writableHighWaterMark: { + __proto__: null, ...Object.getOwnPropertyDescriptor( Writable.prototype, 'writableHighWaterMark' ), }, writableObjectMode: { + __proto__: null, ...Object.getOwnPropertyDescriptor( Writable.prototype, 'writableObjectMode' ), }, writableBuffer: { + __proto__: null, ...Object.getOwnPropertyDescriptor(Writable.prototype, 'writableBuffer'), }, writableLength: { + __proto__: null, ...Object.getOwnPropertyDescriptor(Writable.prototype, 'writableLength'), }, writableFinished: { + __proto__: null, ...Object.getOwnPropertyDescriptor(Writable.prototype, 'writableFinished'), }, writableCorked: { + __proto__: null, ...Object.getOwnPropertyDescriptor(Writable.prototype, 'writableCorked'), }, writableEnded: { + __proto__: null, ...Object.getOwnPropertyDescriptor(Writable.prototype, 'writableEnded'), }, writableNeedDrain: { + __proto__: null, ...Object.getOwnPropertyDescriptor(Writable.prototype, 'writableNeedDrain'), }, + destroyed: { + __proto__: null, get() { if ( this._readableState === undefined || @@ -243,10 +248,6 @@ Duplex.fromWeb = fromWeb; Duplex.toWeb = toWeb; Duplex.from = from; -export function isDuplexInstance(obj) { - return obj instanceof Duplex; -} - // ====================================================================================== function isBlob(b) { @@ -800,7 +801,7 @@ export function newStreamDuplexFromReadableWritablePair( writer.ready.then(() => { return Promise.all( chunks.map((data) => { - return writer.write(data); + return writer.write(data.chunk); }) ).then(done, done); }, done); diff --git a/src/node/internal/streams_end_of_stream.ts b/src/node/internal/streams_end_of_stream.ts index 46f2de3206c..d6de8eb2d7c 100644 --- a/src/node/internal/streams_end_of_stream.ts +++ b/src/node/internal/streams_end_of_stream.ts @@ -23,53 +23,56 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. -/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unnecessary-condition, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access */ +// Ported from https://github.com/mafintosh/end-of-stream with +// permission from the author, Mathias Buus (@mafintosh). -import type { EventEmitter } from 'node:events'; +/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unnecessary-condition, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access */ import { Readable } from 'node-internal:streams_readable'; import { Writable } from 'node-internal:streams_writable'; -import { Transform } from 'node-internal:streams_transform'; -import { - validateObject, - validateFunction, - validateAbortSignal, -} from 'node-internal:validators'; -import { once } from 'node-internal:internal_http_util'; +import type { Transform } from 'node-internal:streams_transform'; +import { nextTick } from 'node-internal:internal_process'; +import type { EventEmitter } from 'node:events'; import { AbortError, ERR_INVALID_ARG_TYPE, ERR_STREAM_PREMATURE_CLOSE, } from 'node-internal:internal_errors'; +import { once } from 'node-internal:internal_http_util'; +import { + validateAbortSignal, + validateFunction, + validateObject, + validateBoolean, +} from 'node-internal:validators'; + import { isClosed, isReadable, isReadableNodeStream, + isReadableStream, isReadableFinished, isReadableErrored, isWritable, isWritableNodeStream, + isWritableStream, isWritableFinished, isWritableErrored, isNodeStream, - willEmitClose, - nop, + willEmitClose as _willEmitClose, + kIsClosedPromise, } from 'node-internal:streams_util'; -import { nextTick } from 'node-internal:internal_process'; +import { addAbortListener } from 'node-internal:events'; +import type Stream from 'node:stream'; -// TODO(later): We do not current implement Node.js' Request object. Might never? // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-parameters -export function isRequest( - stream: any -): stream is T & { abort: VoidFunction } { - return ( - stream != null && - 'setHeader' in stream && - typeof stream.abort === 'function' - ); +function isRequest(stream: any): stream is T { + return 'setHeader' in stream && typeof stream.abort === 'function'; } -export type EOSOptions = { +export const nop = (): void => {}; + +type EOSOptions = { cleanup?: boolean; error?: boolean; readable?: boolean; @@ -90,36 +93,42 @@ export function eos( options: EOSOptions | Callback, callback?: Callback ): Callback { - let _options$readable, _options$writable; - let opts: EOSOptions; if (arguments.length === 2) { - callback = options as Callback; - opts = {} as EOSOptions; + // @ts-expect-error TS2322 Supports overloads + callback = options; + options = {} as EOSOptions; } else if (options == null) { - opts = {} as EOSOptions; + options = {} as EOSOptions; } else { - validateObject(options as EOSOptions, 'options'); - opts = options as EOSOptions; + validateObject(options, 'options'); } validateFunction(callback, 'callback'); - validateAbortSignal(opts.signal, 'options.signal'); + validateAbortSignal((options as EOSOptions).signal, 'options.signal'); + + // Avoid AsyncResource.bind() because it calls Object.defineProperties which + // is a bottleneck here. callback = once(callback) as Callback; - const readable = - (_options$readable = opts.readable) !== null && - _options$readable !== undefined - ? _options$readable - : isReadableNodeStream(stream); - const writable = - (_options$writable = opts.writable) !== null && - _options$writable !== undefined - ? _options$writable - : isWritableNodeStream(stream); + + if (isReadableStream(stream) || isWritableStream(stream)) { + return eosWeb(stream, options as EOSOptions, callback); + } + if (!isNodeStream(stream)) { - // TODO: Webstreams. - throw new ERR_INVALID_ARG_TYPE('stream', 'Stream', stream); + throw new ERR_INVALID_ARG_TYPE( + 'stream', + ['ReadableStream', 'WritableStream', 'Stream'], + stream + ); } + + const readable = + (options as EOSOptions).readable ?? isReadableNodeStream(stream); + const writable = + (options as EOSOptions).writable ?? isWritableNodeStream(stream); + const wState = stream._writableState; const rState = stream._readableState; + const onlegacyfinish = (): void => { if (!stream.writable) { onfinish(); @@ -129,10 +138,11 @@ export function eos( // TODO (ronag): Improve soft detection to include core modules and // common ecosystem modules that do properly emit 'close' but fail // this generic check. - let _willEmitClose = - willEmitClose(stream) && + let willEmitClose = + _willEmitClose(stream) && isReadableNodeStream(stream) === readable && isWritableNodeStream(stream) === writable; + let writableFinished = isWritableFinished(stream, false); const onfinish = (): void => { writableFinished = true; @@ -140,41 +150,52 @@ export function eos( // means that user space is doing something differently and // we cannot trust willEmitClose. if (stream.destroyed) { - _willEmitClose = false; + willEmitClose = false; } - if (_willEmitClose && (!stream.readable || readable)) { + + if (willEmitClose && (!stream.readable || readable)) { return; } + if (!readable || readableFinished) { callback?.call(stream); } }; - let readableFinished = isReadableFinished(stream as Readable, false); + + let readableFinished = isReadableFinished(stream, false); const onend = (): void => { readableFinished = true; // Stream should not be destroyed here. If it is that // means that user space is doing something differently and // we cannot trust willEmitClose. if (stream.destroyed) { - _willEmitClose = false; + willEmitClose = false; } - if (_willEmitClose && (!stream.writable || writable)) { + + if (willEmitClose && (!stream.writable || writable)) { return; } + if (!writable || writableFinished) { callback?.call(stream); } }; - const onerror = (err: any): void => { + + const onerror = (err: Error): void => { callback?.call(stream, err); }; + let closed = isClosed(stream); + const onclose = (): void => { closed = true; + const errored = isWritableErrored(stream) || isReadableErrored(stream); + if (errored && typeof errored !== 'boolean') { return callback?.call(stream, errored); } + if (readable && !readableFinished && isReadableNodeStream(stream, true)) { if (!isReadableFinished(stream, false)) return callback?.call(stream, new ERR_STREAM_PREMATURE_CLOSE()); @@ -183,14 +204,30 @@ export function eos( if (!isWritableFinished(stream, false)) return callback?.call(stream, new ERR_STREAM_PREMATURE_CLOSE()); } + + callback?.call(stream); + }; + + const onclosed = (): void => { + closed = true; + + const errored = isWritableErrored(stream) || isReadableErrored(stream); + + if (errored && typeof errored !== 'boolean') { + callback?.call(stream, errored); + return; + } + callback?.call(stream); }; + const onrequest = (): void => { - stream.req.on('finish', onfinish); + stream.req?.on('finish', onfinish); }; + if (isRequest(stream)) { stream.on('complete', onfinish); - if (!_willEmitClose) { + if (!willEmitClose) { stream.on('abort', onclose); } if (stream.req) { @@ -200,51 +237,55 @@ export function eos( } } else if (writable && !wState) { // legacy streams - stream.on('end', onlegacyfinish); - stream.on('close', onlegacyfinish); + (stream as Stream).on('end', onlegacyfinish).on('close', onlegacyfinish); } // Not all streams will emit 'close' after 'aborted'. - if (!_willEmitClose && typeof stream.aborted === 'boolean') { + if ( + !willEmitClose && + 'aborted' in stream && + typeof stream.aborted === 'boolean' + ) { stream.on('aborted', onclose); } + stream.on('end', onend); stream.on('finish', onfinish); - if (opts.error !== false) { + if ((options as EOSOptions).error !== false) { stream.on('error', onerror); } stream.on('close', onclose); + if (closed) { nextTick(onclose); - } else if ( - (wState !== null && wState !== undefined && wState.errorEmitted) || - (rState !== null && rState !== undefined && rState.errorEmitted) - ) { - if (!_willEmitClose) { - nextTick(onclose); + } else if (wState?.errorEmitted || rState?.errorEmitted) { + if (!willEmitClose) { + nextTick(onclosed); } } else if ( !readable && - (!_willEmitClose || isReadable(stream)) && - (writableFinished || isWritable(stream) === false) + (!willEmitClose || isReadable(stream)) && + (writableFinished || isWritable(stream) === false) && + (wState == null || wState.pendingcb === undefined || wState.pendingcb === 0) ) { - nextTick(onclose); + nextTick(onclosed); } else if ( !writable && - (!_willEmitClose || isWritable(stream)) && + (!willEmitClose || isWritable(stream)) && (readableFinished || isReadable(stream) === false) ) { - nextTick(onclose); + nextTick(onclosed); } else if (rState && stream.req && stream.aborted) { - nextTick(onclose); + nextTick(onclosed); } + const cleanup = (): void => { callback = nop; stream.removeListener('aborted', onclose); stream.removeListener('complete', onfinish); stream.removeListener('abort', onclose); stream.removeListener('request', onrequest); - if (stream.req) stream.req.removeListener('finish', onfinish); + stream.req?.removeListener('finish', onfinish); stream.removeListener('end', onlegacyfinish); stream.removeListener('close', onlegacyfinish); stream.removeListener('finish', onfinish); @@ -252,7 +293,8 @@ export function eos( stream.removeListener('error', onerror); stream.removeListener('close', onclose); }; - if (opts.signal && !closed) { + + if ((options as EOSOptions).signal && !closed) { const abort = (): void => { // Keep it because cleanup removes it. const endCallback = callback; @@ -260,32 +302,83 @@ export function eos( endCallback?.call( stream, new AbortError(undefined, { - cause: opts.signal?.reason, + cause: (options as EOSOptions).signal?.reason, }) ); }; - if (opts.signal.aborted) { + if ((options as EOSOptions).signal?.aborted) { nextTick(abort); } else { + const disposable = addAbortListener( + (options as EOSOptions).signal, + abort + ); const originalCallback = callback; - callback = once((...args) => { - opts.signal?.removeEventListener('abort', abort); + callback = once((...args: unknown[]): void => { + disposable[Symbol.dispose](); originalCallback.apply(stream, args); }); - opts.signal.addEventListener('abort', abort); } } + return cleanup; } +function eosWeb( + stream: ReadableStream | WritableStream, + options: { signal?: AbortSignal }, + callback: (...args: unknown[]) => void +): () => void { + let isAborted = false; + let abort = nop; + if (options.signal) { + abort = (): void => { + isAborted = true; + callback.call( + stream, + new AbortError(undefined, { cause: options.signal?.reason }) + ); + }; + if (options.signal.aborted) { + nextTick(abort); + } else { + const disposable = addAbortListener(options.signal, abort); + const originalCallback = callback; + callback = once((...args: unknown[]): void => { + disposable[Symbol.dispose](); + originalCallback.apply(stream, args); + }); + } + } + const resolverFn = (...args: unknown[]): void => { + if (!isAborted) { + nextTick(() => { + callback.apply(stream, args); + }); + } + }; + // @ts-expect-error TS7053 Symbols are not defined in types yet. + stream[kIsClosedPromise].promise.then(resolverFn, resolverFn); + return nop; +} + export function finished( stream: Readable | Writable, opts: EOSOptions = {} ): Promise { - return new Promise((resolve, reject) => { - eos(stream, opts, (err) => { + let autoCleanup = false; + if (opts.cleanup) { + validateBoolean(opts.cleanup, 'cleanup'); + autoCleanup = opts.cleanup; + } + return new Promise((resolve, reject) => { + const cleanup = eos(stream, opts, (err: unknown) => { + if (autoCleanup) { + cleanup(); + } if (err) { - reject(err as Error); + // eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors + reject(err); } else { resolve(); } diff --git a/src/node/internal/streams_legacy.d.ts b/src/node/internal/streams_legacy.d.ts index 7b2608c9146..20ebd547146 100644 --- a/src/node/internal/streams_legacy.d.ts +++ b/src/node/internal/streams_legacy.d.ts @@ -3,6 +3,7 @@ // https://opensource.org/licenses/Apache-2.0 import NodeJSStream from 'node:stream'; +import EventEmitter from 'node:events'; export declare class Stream extends NodeJSStream.Stream { static isWritable: typeof NodeJSStream.isWritable; @@ -20,4 +21,18 @@ export declare class Stream extends NodeJSStream.Stream { static _uint8ArrayToBuffer(value: Uint8Array): NodeJS.Buffer; static Stream: typeof Stream; + + aborted?: boolean; + readable: boolean; + writable: boolean; + destroyed: boolean; + req: EventEmitter; + + _writableState: { + errorEmitted: boolean; + pendingcb?: number; + }; + _readableState: { + errorEmitted: boolean; + }; } diff --git a/src/node/internal/streams_pipeline.js b/src/node/internal/streams_pipeline.js index b7da4cfa959..f866bd33625 100644 --- a/src/node/internal/streams_pipeline.js +++ b/src/node/internal/streams_pipeline.js @@ -27,7 +27,6 @@ /* eslint-disable */ import { - once, isIterable, isReadable, isReadableNodeStream, @@ -35,6 +34,8 @@ import { } from 'node-internal:streams_util'; import { eos } from 'node-internal:streams_end_of_stream'; import { destroyer as destroyerImpl } from 'node-internal:streams_destroy'; +import { once } from 'node-internal:internal_http_util'; + import { nextTick } from 'node-internal:internal_process'; import { PassThrough } from 'node-internal:streams_transform'; import { Duplex } from 'node-internal:streams_duplex'; diff --git a/src/node/internal/streams_promises.ts b/src/node/internal/streams_promises.ts index 0a3a0de983f..3144b394c53 100644 --- a/src/node/internal/streams_promises.ts +++ b/src/node/internal/streams_promises.ts @@ -25,6 +25,7 @@ import { isIterable, isNodeStream } from 'node-internal:streams_util'; import { finished } from 'node-internal:streams_end_of_stream'; + import { pipelineImpl as pl } from 'node-internal:streams_pipeline'; export { finished }; diff --git a/src/node/internal/streams_readable.d.ts b/src/node/internal/streams_readable.d.ts index 70664bdd419..47fe19e71c6 100644 --- a/src/node/internal/streams_readable.d.ts +++ b/src/node/internal/streams_readable.d.ts @@ -3,10 +3,13 @@ // https://opensource.org/licenses/Apache-2.0 import { Readable as _Readable, Duplex } from 'node:stream'; +import type EventEmitter from 'node:events'; import { - kDestroyed, - kIsReadable, + kState, kIsWritable, + kIsReadable, + kIsClosedPromise, + kIsDestroyed, } from 'node-internal:streams_util'; export declare class ReadableState { @@ -27,13 +30,13 @@ export declare class ReadableState { readingMore?: boolean; readableDidRead?: boolean; readableAborted?: boolean; + [kState]: number; } export declare class Readable extends _Readable { _readableState: ReadableState; _writableState: undefined; _closed: boolean; - errored?: Error | null; writableErrored: boolean; readableErrored: boolean; @@ -43,11 +46,13 @@ export declare class Readable extends _Readable { writable?: boolean; aborted?: boolean; req?: EventEmitter; + writableEnded?: boolean; - [kDestroyed]: boolean; - [kIsReadable]: boolean; [kIsWritable]: boolean; + [kIsReadable]: boolean; + [kIsClosedPromise]: Promise; + [kIsDestroyed]: boolean; } export const from: typeof Duplex.from; diff --git a/src/node/internal/streams_readable.js b/src/node/internal/streams_readable.js index 4b92805fdd7..5a6bd107320 100644 --- a/src/node/internal/streams_readable.js +++ b/src/node/internal/streams_readable.js @@ -23,37 +23,40 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. -/* TODO: the following is adopted code, enabling linting one day */ -/* eslint-disable */ - -import { addAbortSignal } from 'node-internal:streams_add_abort_signal'; - import { - nop, - kPaused, - BufferList, + kState, + // bitfields + kObjectMode, + kErrorEmitted, + kAutoDestroy, + kEmitClose, + kDestroyed, + kClosed, + kCloseEmitted, + kErrored, + kConstructed, + kOnConstructed, isDestroyed, isReadable, - kOnConstructed, + isReadableStream, + handleKnownInternalErrors, } from 'node-internal:streams_util'; -import { eos, finished } from 'node-internal:streams_end_of_stream'; +import { nextTick } from 'node-internal:internal_process'; import { - construct, destroy, - destroyer, - errorOrDestroy, undestroy, + errorOrDestroy, + destroyer, + construct, } from 'node-internal:streams_destroy'; +import { eos, finished, nop } from 'node-internal:streams_end_of_stream'; import { - getDefaultHighWaterMark, getHighWaterMark, + getDefaultHighWaterMark, } from 'node-internal:streams_state'; -import { nextTick } from 'node-internal:internal_process'; - +import { addAbortSignal } from 'node-internal:streams_add_abort_signal'; import { EventEmitter } from 'node-internal:events'; - import { Stream } from 'node-internal:streams_legacy'; - import { Buffer } from 'node-internal:internal_buffer'; import { @@ -61,13 +64,13 @@ import { aggregateTwoErrors, ERR_INVALID_ARG_TYPE, ERR_INVALID_ARG_VALUE, - ERR_STREAM_PREMATURE_CLOSE, ERR_METHOD_NOT_IMPLEMENTED, ERR_MISSING_ARGS, ERR_OUT_OF_RANGE, ERR_STREAM_PUSH_AFTER_EOF, ERR_STREAM_UNSHIFT_AFTER_END_EVENT, ERR_STREAM_NULL_VALUES, + ERR_UNKNOWN_ENCODING, } from 'node-internal:internal_errors'; import { @@ -79,107 +82,240 @@ import { import { StringDecoder } from 'node-internal:internal_stringdecoder'; +const streamsNodejsV24Compat = + Cloudflare.compatibilityFlags.enable_streams_nodejs_v24_compat; // eslint-disable-line no-undef + +const kErroredValue = Symbol('kErroredValue'); +const kDefaultEncodingValue = Symbol('kDefaultEncodingValue'); +const kDecoderValue = Symbol('kDecoderValue'); +const kEncodingValue = Symbol('kEncodingValue'); + +// Bitfield flag constants for ReadableState. Each constant uses left-shift (<<) to set a specific +// bit position, allowing multiple boolean flags to be stored efficiently in a single integer (kState). +// For example, `1 << 9` creates a value with only bit 9 set (value: 512). +const kEnded = 1 << 9; +const kEndEmitted = 1 << 10; +const kReading = 1 << 11; +const kSync = 1 << 12; +const kNeedReadable = 1 << 13; +const kEmittedReadable = 1 << 14; +const kReadableListening = 1 << 15; +const kResumeScheduled = 1 << 16; +const kMultiAwaitDrain = 1 << 17; +const kReadingMore = 1 << 18; +const kDataEmitted = 1 << 19; +const kDefaultUTF8Encoding = 1 << 20; +const kDecoder = 1 << 21; +const kEncoding = 1 << 22; +const kHasFlowing = 1 << 23; +const kFlowing = 1 << 24; +const kHasPaused = 1 << 25; +const kPaused = 1 << 26; +const kDataListening = 1 << 27; + // ====================================================================================== // ReadableState -export function ReadableState(options, stream, isDuplex) { - // Duplex streams are both readable and writable, but share - // the same options object. - // However, some cases require setting options to different - // values for the readable and the writable sides of the duplex stream. - // These options can be provided separately as readableXXX and writableXXX. - - // Object stream flag. Used to make read(n) ignore n and to - // make all the buffer merging and length checks go away. - this.objectMode = !!options?.objectMode; - if (isDuplex) - this.objectMode = this.objectMode || !!options?.readableObjectMode; - - // The point at which it stops calling _read() to fill the buffer - // Note: 0 is a valid value, means "don't call _read preemptively ever" - this.highWaterMark = options - ? getHighWaterMark(this, options, 'readableHighWaterMark', isDuplex) - : getDefaultHighWaterMark(false); - - // A linked list is used to store data chunks instead of an array because the - // linked list can remove elements from the beginning faster than - // array.shift(). - this.buffer = new BufferList(); - this.length = 0; - this.pipes = []; - this.flowing = null; - this.ended = false; - this.endEmitted = false; - this.reading = false; - +// TODO(benjamingr) it is likely slower to do it this way than with free functions +function makeBitMapDescriptor(bit) { + return { + enumerable: false, + get() { + return (this[kState] & bit) !== 0; + }, + set(value) { + if (value) this[kState] |= bit; + else this[kState] &= ~bit; + }, + }; +} +Object.defineProperties(ReadableState.prototype, { + objectMode: makeBitMapDescriptor(kObjectMode), + ended: makeBitMapDescriptor(kEnded), + endEmitted: makeBitMapDescriptor(kEndEmitted), + reading: makeBitMapDescriptor(kReading), // Stream is still being constructed and cannot be // destroyed until construction finished or failed. // Async construction is opt in, therefore we start as // constructed. - this.constructed = true; - + constructed: makeBitMapDescriptor(kConstructed), // A flag to be able to tell if the event 'readable'/'data' is emitted // immediately, or on a later tick. We set this to true at first, because // any actions that shouldn't happen until "later" should generally also // not happen before the first read call. - this.sync = true; - + sync: makeBitMapDescriptor(kSync), // Whenever we return null, then we set a flag to say // that we're awaiting a 'readable' event emission. - this.needReadable = false; - this.emittedReadable = false; - this.readableListening = false; - this.resumeScheduled = false; - this[kPaused] = null; - + needReadable: makeBitMapDescriptor(kNeedReadable), + emittedReadable: makeBitMapDescriptor(kEmittedReadable), + readableListening: makeBitMapDescriptor(kReadableListening), + resumeScheduled: makeBitMapDescriptor(kResumeScheduled), // True if the error was already emitted and should not be thrown again. - this.errorEmitted = false; - - // Should close be emitted on destroy. Defaults to true. - this.emitClose = !options || options.emitClose !== false; - - // Should .destroy() be called after 'end' (and potentially 'finish'). - this.autoDestroy = !options || options.autoDestroy !== false; - + errorEmitted: makeBitMapDescriptor(kErrorEmitted), + emitClose: makeBitMapDescriptor(kEmitClose), + autoDestroy: makeBitMapDescriptor(kAutoDestroy), // Has it been destroyed. - this.destroyed = false; + destroyed: makeBitMapDescriptor(kDestroyed), + // Indicates whether the stream has finished destroying. + closed: makeBitMapDescriptor(kClosed), + // True if close has been emitted or would have been emitted + // depending on emitClose. + closeEmitted: makeBitMapDescriptor(kCloseEmitted), + multiAwaitDrain: makeBitMapDescriptor(kMultiAwaitDrain), + // If true, a maybeReadMore has been scheduled. + readingMore: makeBitMapDescriptor(kReadingMore), + dataEmitted: makeBitMapDescriptor(kDataEmitted), // Indicates whether the stream has errored. When true no further // _read calls, 'data' or 'readable' events should occur. This is needed // since when autoDestroy is disabled we need a way to tell whether the // stream has failed. - this.errored = null; + errored: { + __proto__: null, + enumerable: false, + get() { + return (this[kState] & kErrored) !== 0 ? this[kErroredValue] : null; + }, + set(value) { + if (value) { + this[kErroredValue] = value; + this[kState] |= kErrored; + } else { + this[kState] &= ~kErrored; + } + }, + }, - // Indicates whether the stream has finished destroying. - this.closed = false; + defaultEncoding: { + __proto__: null, + enumerable: false, + get() { + return (this[kState] & kDefaultUTF8Encoding) !== 0 + ? 'utf8' + : this[kDefaultEncodingValue]; + }, + set(value) { + if (value === 'utf8' || value === 'utf-8') { + this[kState] |= kDefaultUTF8Encoding; + } else { + this[kState] &= ~kDefaultUTF8Encoding; + this[kDefaultEncodingValue] = value; + } + }, + }, - // True if close has been emitted or would have been emitted - // depending on emitClose. - this.closeEmitted = false; + decoder: { + __proto__: null, + enumerable: false, + get() { + return (this[kState] & kDecoder) !== 0 ? this[kDecoderValue] : null; + }, + set(value) { + if (value) { + this[kDecoderValue] = value; + this[kState] |= kDecoder; + } else { + this[kState] &= ~kDecoder; + } + }, + }, + + encoding: { + __proto__: null, + enumerable: false, + get() { + return (this[kState] & kEncoding) !== 0 ? this[kEncodingValue] : null; + }, + set(value) { + if (value) { + this[kEncodingValue] = value; + this[kState] |= kEncoding; + } else { + this[kState] &= ~kEncoding; + } + }, + }, + + flowing: { + __proto__: null, + enumerable: false, + get() { + return (this[kState] & kHasFlowing) !== 0 + ? (this[kState] & kFlowing) !== 0 + : null; + }, + set(value) { + if (value == null) { + this[kState] &= ~(kHasFlowing | kFlowing); + } else if (value) { + this[kState] |= kHasFlowing | kFlowing; + } else { + this[kState] |= kHasFlowing; + this[kState] &= ~kFlowing; + } + }, + }, +}); + +export function ReadableState(options, _stream, isDuplex) { + // Bit map field to store ReadableState more efficiently with 1 bit per field + // instead of a V8 slot per field. + this[kState] = kEmitClose | kAutoDestroy | kConstructed | kSync; + + // Object stream flag. Used to make read(n) ignore n and to + // make all the buffer merging and length checks go away. + if (options?.objectMode) this[kState] |= kObjectMode; + + if (isDuplex && options?.readableObjectMode) this[kState] |= kObjectMode; + + // The point at which it stops calling _read() to fill the buffer + // Note: 0 is a valid value, means "don't call _read preemptively ever" + this.highWaterMark = options + ? getHighWaterMark(this, options, 'readableHighWaterMark', isDuplex) + : getDefaultHighWaterMark(false); + + this.buffer = []; + this.bufferIndex = 0; + this.length = 0; + this.pipes = []; + + // Should close be emitted on destroy. Defaults to true. + if (options && options.emitClose === false) this[kState] &= ~kEmitClose; + + // Should .destroy() be called after 'end' (and potentially 'finish'). + if (options && options.autoDestroy === false) this[kState] &= ~kAutoDestroy; // Crypto is kind of old and crusty. Historically, its default string // encoding is 'binary' so we have to make this configurable. // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options?.defaultEncoding || 'utf8'; + const defaultEncoding = options?.defaultEncoding; + if ( + defaultEncoding == null || + defaultEncoding === 'utf8' || + defaultEncoding === 'utf-8' + ) { + this[kState] |= kDefaultUTF8Encoding; + } else if (Buffer.isEncoding(defaultEncoding)) { + this.defaultEncoding = defaultEncoding; + } else if (streamsNodejsV24Compat) { + // This is a semver-major change. Ref: https://github.com/nodejs/node/pull/46430 + throw new ERR_UNKNOWN_ENCODING(defaultEncoding); + } else { + this.defaultEncoding = defaultEncoding; + } // Ref the piped dest which we need a drain event on it // type: null | Writable | Set. this.awaitDrainWriters = null; - this.multiAwaitDrain = false; - // If true, a maybeReadMore has been scheduled. - this.readingMore = false; - this.dataEmitted = false; - this.decoder = null; - this.encoding = null; - if (options && options.encoding) { + if (options?.encoding) { this.decoder = new StringDecoder(options.encoding); this.encoding = options.encoding; } } ReadableState.prototype[kOnConstructed] = function onConstructed(stream) { - if (this.needReadable) { + if ((this[kState] & kNeedReadable) !== 0) { maybeReadMore(stream, this); } }; @@ -195,20 +331,41 @@ Object.setPrototypeOf(Readable, Stream); export function Readable(options) { if (!(this instanceof Readable)) return new Readable(options); + this._events ??= { + close: undefined, + error: undefined, + data: undefined, + end: undefined, + readable: undefined, + // Skip uncommon events... + // pause: undefined, + // resume: undefined, + // pipe: undefined, + // unpipe: undefined, + // [destroyImpl.kConstruct]: undefined, + // [destroyImpl.kDestroy]: undefined, + }; + this._readableState = new ReadableState(options, this, false); + if (options) { if (typeof options.read === 'function') this._read = options.read; + if (typeof options.destroy === 'function') this._destroy = options.destroy; + if (typeof options.construct === 'function') this._construct = options.construct; + if (options.signal) addAbortSignal(options.signal, this); } + Stream.call(this, options); - construct(this, () => { - if (this._readableState.needReadable) { - maybeReadMore(this, this._readableState); - } - }); + + if (this._construct != null) { + construct(this, () => { + this._readableState[kOnConstructed](this); + }); + } } Readable.prototype.destroy = destroy; Readable.prototype._undestroy = undestroy; @@ -220,134 +377,252 @@ Readable.prototype[EventEmitter.captureRejectionSymbol] = function (err) { this.destroy(err); }; +Readable.prototype[Symbol.asyncDispose] = async function () { + let error; + if (!this.destroyed) { + error = this.readableEnded ? null : new AbortError(); + this.destroy(error); + } + await new Promise((resolve, reject) => + eos(this, (err) => (err && err !== error ? reject(err) : resolve(null))) + ); +}; + // Manually shove something into the read() buffer. // This returns true if the highWaterMark has not been hit yet, // similar to how Writable.write() returns true if you should // write() some more. Readable.prototype.push = function (chunk, encoding) { - return readableAddChunk(this, chunk, encoding, false); + const state = this._readableState; + return (state[kState] & kObjectMode) === 0 + ? readableAddChunkPushByteMode(this, state, chunk, encoding) + : readableAddChunkPushObjectMode(this, state, chunk, encoding); }; // Unshift should *always* be something directly out of read(). Readable.prototype.unshift = function (chunk, encoding) { - return readableAddChunk(this, chunk, encoding, true); + const state = this._readableState; + return (state[kState] & kObjectMode) === 0 + ? readableAddChunkUnshiftByteMode(this, state, chunk, encoding) + : readableAddChunkUnshiftObjectMode(this, state, chunk); }; -function readableAddChunk(stream, chunk, encoding, addToFront) { - const state = stream._readableState; - let err; - if (!state.objectMode) { - if (typeof chunk === 'string') { - encoding ||= state.defaultEncoding; - if (state.encoding !== encoding) { - if (addToFront && state.encoding) { - // When unshifting, if state.encoding is set, we have to save - // the string in the BufferList with the state encoding. - chunk = Buffer.from(chunk, encoding).toString(state.encoding); - } else { - chunk = Buffer.from(chunk, encoding); - encoding = ''; - } +function readableAddChunkUnshiftByteMode(stream, state, chunk, encoding) { + if (chunk === null) { + state[kState] &= ~kReading; + onEofChunk(stream, state); + + return false; + } + + if (typeof chunk === 'string') { + encoding ||= state.defaultEncoding; + if (state.encoding !== encoding) { + if (state.encoding) { + // When unshifting, if state.encoding is set, we have to save + // the string in the BufferList with the state encoding. + chunk = Buffer.from(chunk, encoding).toString(state.encoding); + } else { + chunk = Buffer.from(chunk, encoding); } - } else if (chunk instanceof Buffer) { - encoding = ''; - } else if (Stream._isUint8Array(chunk)) { - chunk = Stream._uint8ArrayToBuffer(chunk); - encoding = ''; - } else if (chunk != null) { - err = new ERR_INVALID_ARG_TYPE( + } + } else if (Stream._isArrayBufferView(chunk)) { + chunk = Stream._uint8ArrayToBuffer(chunk); + } else if (chunk !== undefined && !(chunk instanceof Buffer)) { + errorOrDestroy( + stream, + new ERR_INVALID_ARG_TYPE( 'chunk', - ['string', 'Buffer', 'Uint8Array'], + ['string', 'Buffer', 'TypedArray', 'DataView'], chunk - ); - } + ) + ); + return false; } - if (err) { - errorOrDestroy(stream, err); - } else if (chunk === null) { - state.reading = false; + + if (!(chunk && chunk.length > 0)) { + return canPushMore(state); + } + + return readableAddChunkUnshiftValue(stream, state, chunk); +} + +function readableAddChunkUnshiftObjectMode(stream, state, chunk) { + if (chunk === null) { + state[kState] &= ~kReading; onEofChunk(stream, state); - } else if (state.objectMode || (chunk && chunk.length > 0)) { - if (addToFront) { - if (state.endEmitted) - errorOrDestroy(stream, new ERR_STREAM_UNSHIFT_AFTER_END_EVENT()); - else if (state.destroyed || state.errored) return false; - else addChunk(stream, state, chunk, true); - } else if (state.ended) { - errorOrDestroy(stream, new ERR_STREAM_PUSH_AFTER_EOF()); - } else if (state.destroyed || state.errored) { - return false; - } else { - state.reading = false; - if (state.decoder && !encoding) { - chunk = state.decoder.write(chunk); - if (state.objectMode || chunk.length !== 0) - addChunk(stream, state, chunk, false); - else maybeReadMore(stream, state); - } else { - addChunk(stream, state, chunk, false); - } + + return false; + } + + return readableAddChunkUnshiftValue(stream, state, chunk); +} + +function readableAddChunkUnshiftValue(stream, state, chunk) { + if ((state[kState] & kEndEmitted) !== 0) + errorOrDestroy(stream, new ERR_STREAM_UNSHIFT_AFTER_END_EVENT()); + else if ((state[kState] & (kDestroyed | kErrored)) !== 0) return false; + else addChunk(stream, state, chunk, true); + + return canPushMore(state); +} + +function readableAddChunkPushByteMode(stream, state, chunk, encoding) { + if (chunk === null) { + state[kState] &= ~kReading; + onEofChunk(stream, state); + return false; + } + + if (typeof chunk === 'string') { + encoding ||= state.defaultEncoding; + if (state.encoding !== encoding) { + chunk = Buffer.from(chunk, encoding); + encoding = ''; } - } else if (!addToFront) { - state.reading = false; + } else if (chunk instanceof Buffer) { + encoding = ''; + } else if (Stream._isArrayBufferView(chunk)) { + chunk = Stream._uint8ArrayToBuffer(chunk); + encoding = ''; + } else if (chunk !== undefined) { + errorOrDestroy( + stream, + new ERR_INVALID_ARG_TYPE( + 'chunk', + ['string', 'Buffer', 'TypedArray', 'DataView'], + chunk + ) + ); + return false; + } + + if (!chunk || chunk.length <= 0) { + state[kState] &= ~kReading; maybeReadMore(stream, state); + + return canPushMore(state); + } + + if ((state[kState] & kEnded) !== 0) { + errorOrDestroy(stream, new ERR_STREAM_PUSH_AFTER_EOF()); + return false; + } + + if ((state[kState] & (kDestroyed | kErrored)) !== 0) { + return false; + } + + state[kState] &= ~kReading; + if ((state[kState] & kDecoder) !== 0 && !encoding) { + chunk = state[kDecoderValue].write(chunk); + if (chunk.length === 0) { + maybeReadMore(stream, state); + return canPushMore(state); + } + } + + addChunk(stream, state, chunk, false); + return canPushMore(state); +} + +function readableAddChunkPushObjectMode(stream, state, chunk, encoding) { + if (chunk === null) { + state[kState] &= ~kReading; + onEofChunk(stream, state); + return false; + } + + if ((state[kState] & kEnded) !== 0) { + errorOrDestroy(stream, new ERR_STREAM_PUSH_AFTER_EOF()); + return false; + } + + if ((state[kState] & (kDestroyed | kErrored)) !== 0) { + return false; + } + + state[kState] &= ~kReading; + + if ((state[kState] & kDecoder) !== 0 && !encoding) { + chunk = state[kDecoderValue].write(chunk); } + addChunk(stream, state, chunk, false); + return canPushMore(state); +} + +function canPushMore(state) { // We can push more data if we are below the highWaterMark. // Also, if we have no data yet, we can stand some more bytes. // This is to work around cases where hwm=0, such as the repl. return ( - !state.ended && (state.length < state.highWaterMark || state.length === 0) + (state[kState] & kEnded) === 0 && + (state.length < state.highWaterMark || state.length === 0) ); } function addChunk(stream, state, chunk, addToFront) { if ( - state.flowing && - state.length === 0 && - !state.sync && - stream.listenerCount('data') > 0 + (state[kState] & (kFlowing | kSync | kDataListening)) === + (kFlowing | kDataListening) && + state.length === 0 ) { // Use the guard to avoid creating `Set()` repeatedly // when we have multiple pipes. - if (state.multiAwaitDrain) { + if ((state[kState] & kMultiAwaitDrain) !== 0) { state.awaitDrainWriters.clear(); } else { state.awaitDrainWriters = null; - state.multiAwaitDrain = false; } - state.dataEmitted = true; + + state[kState] |= kDataEmitted; stream.emit('data', chunk); } else { // Update the buffer info. - state.length += state.objectMode ? 1 : chunk.length; - if (addToFront) state.buffer.unshift(chunk); - else state.buffer.push(chunk); - if (state.needReadable) emitReadable(stream); + state.length += (state[kState] & kObjectMode) !== 0 ? 1 : chunk.length; + if (addToFront) { + if (state.bufferIndex > 0) { + state.buffer[--state.bufferIndex] = chunk; + } else { + state.buffer.unshift(chunk); // Slow path + } + } else { + state.buffer.push(chunk); + } + + if ((state[kState] & kNeedReadable) !== 0) emitReadable(stream); } maybeReadMore(stream, state); } Readable.prototype.isPaused = function () { const state = this._readableState; - return state[kPaused] === true || state.flowing === false; + return ( + (state[kState] & kPaused) !== 0 || + (state[kState] & (kHasFlowing | kFlowing)) === kHasFlowing + ); }; // Backwards compatibility. Readable.prototype.setEncoding = function (enc) { + const state = this._readableState; + const decoder = new StringDecoder(enc); - this._readableState.decoder = decoder; + state.decoder = decoder; // If setEncoding(null), decoder.encoding equals utf8. - this._readableState.encoding = decoder.encoding; - const buffer = this._readableState.buffer; + state.encoding = state.decoder.encoding; + // Iterate over current buffer to convert already stored Buffers: let content = ''; - for (const data of buffer) { + for (const data of state.buffer.slice(state.bufferIndex)) { content += decoder.write(data); } - buffer.clear(); - if (content !== '') buffer.push(content); - this._readableState.length = content.length; + state.buffer.length = 0; + state.bufferIndex = 0; + + if (content !== '') state.buffer.push(content); + state.length = content.length; return this; }; @@ -373,15 +648,17 @@ function computeNewHighWaterMark(n) { // This function is designed to be inlinable, so please take care when making // changes to the function body. function howMuchToRead(n, state) { - if (n <= 0 || (state.length === 0 && state.ended)) return 0; - if (state.objectMode) return 1; + if (n <= 0 || (state.length === 0 && (state[kState] & kEnded) !== 0)) + return 0; + if ((state[kState] & kObjectMode) !== 0) return 1; if (Number.isNaN(n)) { // Only flow one buffer at a time. - if (state.flowing && state.length) return state.buffer.first().length; + if ((state[kState] & kFlowing) !== 0 && state.length) + return state.buffer[state.bufferIndex].length; return state.length; } if (n <= state.length) return n; - return state.ended ? state.length : 0; + return (state[kState] & kEnded) !== 0 ? state.length : 0; } // You can override either this method, or the async _read(n) below. @@ -391,34 +668,36 @@ Readable.prototype.read = function (n) { if (n === undefined) { n = NaN; } else if (!Number.isInteger(n)) { - n = Number.parseInt(`${n}`, 10); + n = Number.parseInt(n, 10); } const state = this._readableState; const nOrig = n; // If we're asking for more than the current hwm, then raise the hwm. if (n > state.highWaterMark) state.highWaterMark = computeNewHighWaterMark(n); - if (n !== 0) state.emittedReadable = false; + + if (n !== 0) state[kState] &= ~kEmittedReadable; // If we're doing read(0) to trigger a readable event, but we // already have a bunch of data in the buffer, then just trigger // the 'readable' event and move on. if ( n === 0 && - state.needReadable && + (state[kState] & kNeedReadable) !== 0 && ((state.highWaterMark !== 0 ? state.length >= state.highWaterMark : state.length > 0) || - state.ended) + (state[kState] & kEnded) !== 0) ) { - if (state.length === 0 && state.ended) endReadable(this); + if (state.length === 0 && (state[kState] & kEnded) !== 0) endReadable(this); else emitReadable(this); return null; } + n = howMuchToRead(n, state); // If we've ended, and we're now clear, then finish it up. - if (n === 0 && state.ended) { + if (n === 0 && (state[kState] & kEnded) !== 0) { if (state.length === 0) endReadable(this); return null; } @@ -446,7 +725,7 @@ Readable.prototype.read = function (n) { // 3. Actually pull the requested chunks out of the buffer and return. // if we need a readable event, then we need to do some reading. - let doRead = state.needReadable; + let doRead = (state[kState] & kNeedReadable) !== 0; // If we currently have less than the highWaterMark, then also read some. if (state.length === 0 || state.length - n < state.highWaterMark) { @@ -457,18 +736,15 @@ Readable.prototype.read = function (n) { // reading, then it's unnecessary, if we're constructing we have to wait, // and if we're destroyed or errored, then it's not allowed, if ( - state.ended || - state.reading || - state.destroyed || - state.errored || - !state.constructed + (state[kState] & + (kReading | kEnded | kDestroyed | kErrored | kConstructed)) !== + kConstructed ) { doRead = false; } else if (doRead) { - state.reading = true; - state.sync = true; + state[kState] |= kReading | kSync; // If the length is currently zero, then we *need* a readable event. - if (state.length === 0) state.needReadable = true; + if (state.length === 0) state[kState] |= kNeedReadable; // Call internal read method try { @@ -476,60 +752,68 @@ Readable.prototype.read = function (n) { } catch (err) { errorOrDestroy(this, err); } - state.sync = false; + state[kState] &= ~kSync; + // If _read pushed data synchronously, then `reading` will be false, // and we need to re-evaluate how much data we can return to the user. - if (!state.reading) n = howMuchToRead(nOrig, state); + if ((state[kState] & kReading) === 0) n = howMuchToRead(nOrig, state); } + let ret; if (n > 0) ret = fromList(n, state); else ret = null; + if (ret === null) { - state.needReadable = state.length <= state.highWaterMark; + state[kState] |= state.length <= state.highWaterMark ? kNeedReadable : 0; n = 0; } else { state.length -= n; - if (state.multiAwaitDrain) { + if ((state[kState] & kMultiAwaitDrain) !== 0) { state.awaitDrainWriters.clear(); } else { state.awaitDrainWriters = null; - state.multiAwaitDrain = false; } } + if (state.length === 0) { // If we have nothing in the buffer, then we want to know // as soon as we *do* get something into the buffer. - if (!state.ended) state.needReadable = true; + if ((state[kState] & kEnded) === 0) state[kState] |= kNeedReadable; // If we tried to read() past the EOF, then emit end on the next tick. - if (nOrig !== n && state.ended) endReadable(this); + if (nOrig !== n && (state[kState] & kEnded) !== 0) endReadable(this); } - if (ret !== null && !state.errorEmitted && !state.closeEmitted) { - state.dataEmitted = true; + + if (ret !== null && (state[kState] & (kErrorEmitted | kCloseEmitted)) === 0) { + state[kState] |= kDataEmitted; this.emit('data', ret); } + return ret; }; function onEofChunk(stream, state) { - if (state.ended) return; - if (state.decoder) { - const chunk = state.decoder.end(); - if (chunk && chunk.length) { + if ((state[kState] & kEnded) !== 0) return; + const decoder = + (state[kState] & kDecoder) !== 0 ? state[kDecoderValue] : null; + if (decoder) { + const chunk = decoder.end(); + if (chunk?.length) { state.buffer.push(chunk); - state.length += state.objectMode ? 1 : chunk.length; + state.length += (state[kState] & kObjectMode) !== 0 ? 1 : chunk.length; } } - state.ended = true; - if (state.sync) { + state[kState] |= kEnded; + + if ((state[kState] & kSync) !== 0) { // If we are sync, wait until next tick to emit the data. // Otherwise we risk emitting data in the flow() // the readable code triggers during a read() call. emitReadable(stream); } else { // Emit 'readable' now to make sure it gets picked up. - state.needReadable = false; - state.emittedReadable = true; + state[kState] &= ~kNeedReadable; + state[kState] |= kEmittedReadable; // We have to emit readable now that we are EOF. Modules // in the ecosystem (e.g. dicer) rely on this event being sync. emitReadable_(stream); @@ -541,18 +825,21 @@ function onEofChunk(stream, state) { // a nextTick recursion warning, but that's not so bad. function emitReadable(stream) { const state = stream._readableState; - state.needReadable = false; - if (!state.emittedReadable) { - state.emittedReadable = true; + state[kState] &= ~kNeedReadable; + if ((state[kState] & kEmittedReadable) === 0) { + state[kState] |= kEmittedReadable; nextTick(emitReadable_, stream); } } function emitReadable_(stream) { const state = stream._readableState; - if (!state.destroyed && !state.errored && (state.length || state.ended)) { + if ( + (state[kState] & (kDestroyed | kErrored)) === 0 && + (state.length || (state[kState] & kEnded) !== 0) + ) { stream.emit('readable'); - state.emittedReadable = false; + state[kState] &= ~kEmittedReadable; } // The stream needs another readable event if: @@ -561,8 +848,11 @@ function emitReadable_(stream) { // 2. It is not ended. // 3. It is below the highWaterMark, so we can schedule // another readable later. - state.needReadable = - !state.flowing && !state.ended && state.length <= state.highWaterMark; + state[kState] |= + (state[kState] & (kFlowing | kEnded)) === 0 && + state.length <= state.highWaterMark + ? kNeedReadable + : 0; flow(stream); } @@ -573,8 +863,8 @@ function emitReadable_(stream) { // However, if we're not ended, or reading, and the length < hwm, // then go ahead and try to read some more preemptively. function maybeReadMore(stream, state) { - if (!state.readingMore && state.constructed) { - state.readingMore = true; + if ((state[kState] & (kReadingMore | kConstructed)) === kConstructed) { + state[kState] |= kReadingMore; nextTick(maybeReadMore_, stream, state); } } @@ -604,10 +894,9 @@ function maybeReadMore_(stream, state) { // read()s. The execution ends in this method again after the _read() ends // up calling push() with more data. while ( - !state.reading && - !state.ended && + (state[kState] & (kReading | kEnded)) === 0 && (state.length < state.highWaterMark || - (state.flowing && state.length === 0)) + ((state[kState] & kFlowing) !== 0 && state.length === 0)) ) { const len = state.length; stream.read(0); @@ -615,7 +904,7 @@ function maybeReadMore_(stream, state) { // Didn't get any data, stop spinning. break; } - state.readingMore = false; + state[kState] &= ~kReadingMore; } // Abstract method. to be overridden in specific implementation classes. @@ -627,21 +916,26 @@ Readable.prototype._read = function (_size) { }; Readable.prototype.pipe = function (dest, pipeOpts) { - const src = this; + const src = this; // eslint-disable-line @typescript-eslint/no-this-alias const state = this._readableState; + if (state.pipes.length === 1) { - if (!state.multiAwaitDrain) { - state.multiAwaitDrain = true; + if ((state[kState] & kMultiAwaitDrain) === 0) { + state[kState] |= kMultiAwaitDrain; state.awaitDrainWriters = new Set( state.awaitDrainWriters ? [state.awaitDrainWriters] : [] ); } } + state.pipes.push(dest); + const doEnd = !pipeOpts || pipeOpts.end !== false; + const endFn = doEnd ? onend : unpipe; - if (state.endEmitted) nextTick(endFn); + if ((state[kState] & kEndEmitted) !== 0) nextTick(endFn); else src.once('end', endFn); + dest.on('unpipe', onunpipe); function onunpipe(readable, unpipeInfo) { if (readable === src) { @@ -651,10 +945,13 @@ Readable.prototype.pipe = function (dest, pipeOpts) { } } } + function onend() { dest.end(); } + let ondrain; + let cleanedUp = false; function cleanup() { // Cleanup event handlers once the pipe is broken. @@ -668,6 +965,7 @@ Readable.prototype.pipe = function (dest, pipeOpts) { src.removeListener('end', onend); src.removeListener('end', unpipe); src.removeListener('data', ondata); + cleanedUp = true; // If the reader is waiting for a drain event from this @@ -682,6 +980,7 @@ Readable.prototype.pipe = function (dest, pipeOpts) { ) ondrain(); } + function pause() { // If the user unpiped during `dest.write()`, it is possible // to get stuck in a permanently paused state if that write @@ -690,7 +989,7 @@ Readable.prototype.pipe = function (dest, pipeOpts) { if (!cleanedUp) { if (state.pipes.length === 1 && state.pipes[0] === dest) { state.awaitDrainWriters = dest; - state.multiAwaitDrain = false; + state[kState] &= ~kMultiAwaitDrain; } else if (state.pipes.length > 1 && state.pipes.includes(dest)) { state.awaitDrainWriters.add(dest); } @@ -705,11 +1004,24 @@ Readable.prototype.pipe = function (dest, pipeOpts) { dest.on('drain', ondrain); } } + src.on('data', ondata); function ondata(chunk) { - const ret = dest.write(chunk); - if (ret === false) { - pause(); + // This is a semver-major change. Ref: https://github.com/nodejs/node/pull/55270 + if (streamsNodejsV24Compat) { + try { + const ret = dest.write(chunk); + if (ret === false) { + pause(); + } + } catch (error) { + dest.destroy(error); + } + } else { + const ret = dest.write(chunk); + if (ret === false) { + pause(); + } } } @@ -743,6 +1055,7 @@ Readable.prototype.pipe = function (dest, pipeOpts) { unpipe(); } dest.once('finish', onfinish); + function unpipe() { src.unpipe(dest); } @@ -753,12 +1066,11 @@ Readable.prototype.pipe = function (dest, pipeOpts) { // Start the flow if it hasn't been started already. if (dest.writableNeedDrain === true) { - if (state.flowing) { - pause(); - } - } else if (!state.flowing) { + pause(); + } else if ((state[kState] & kFlowing) === 0) { src.resume(); } + return dest; }; @@ -771,12 +1083,13 @@ function pipeOnDrain(src, dest) { // so we use the real dest here. if (state.awaitDrainWriters === dest) { state.awaitDrainWriters = null; - } else if (state.multiAwaitDrain) { + } else if ((state[kState] & kMultiAwaitDrain) !== 0) { state.awaitDrainWriters.delete(dest); } + if ( (!state.awaitDrainWriters || state.awaitDrainWriters.size === 0) && - src.listenerCount('data') + (state[kState] & kDataListening) !== 0 ) { src.resume(); } @@ -785,27 +1098,26 @@ function pipeOnDrain(src, dest) { Readable.prototype.unpipe = function (dest) { const state = this._readableState; - const unpipeInfo = { - hasUnpiped: false, - }; + const unpipeInfo = { hasUnpiped: false }; // If we're not piping anywhere, then do nothing. if (state.pipes.length === 0) return this; + if (!dest) { // remove all. const dests = state.pipes; state.pipes = []; this.pause(); + for (let i = 0; i < dests.length; i++) - dests[i].emit('unpipe', this, { - hasUnpiped: false, - }); + dests[i].emit('unpipe', this, { hasUnpiped: false }); return this; } // Try to find the right one. const index = state.pipes.indexOf(dest); if (index === -1) return this; + state.pipes.splice(index, 1); if (state.pipes.length === 0) this.pause(); dest.emit('unpipe', this, unpipeInfo); @@ -817,30 +1129,40 @@ Readable.prototype.unpipe = function (dest) { Readable.prototype.on = function (ev, fn) { const res = Stream.prototype.on.call(this, ev, fn); const state = this._readableState; + if (ev === 'data') { + state[kState] |= kDataListening; + // Update readableListening so that resume() may be a no-op // a few lines down. This is needed to support once('readable'). - state.readableListening = this.listenerCount('readable') > 0; + state[kState] |= + this.listenerCount('readable') > 0 ? kReadableListening : 0; // Try start flowing on next tick if stream isn't explicitly paused. - if (state.flowing !== false) this.resume(); + if ((state[kState] & (kHasFlowing | kFlowing)) !== kHasFlowing) { + this.resume(); + } } else if (ev === 'readable') { - if (!state.endEmitted && !state.readableListening) { - state.readableListening = state.needReadable = true; - state.flowing = false; - state.emittedReadable = false; + if ((state[kState] & (kEndEmitted | kReadableListening)) === 0) { + state[kState] |= kReadableListening | kNeedReadable | kHasFlowing; + state[kState] &= ~(kFlowing | kEmittedReadable); if (state.length) { emitReadable(this); - } else if (!state.reading) { + } else if ((state[kState] & kReading) === 0) { nextTick(nReadingNextTick, this); } } } + return res; }; Readable.prototype.addListener = Readable.prototype.on; + Readable.prototype.removeListener = function (ev, fn) { + const state = this._readableState; + const res = Stream.prototype.removeListener.call(this, ev, fn); + if (ev === 'readable') { // We need to check if there is someone still listening to // readable and reset the state. However this needs to happen @@ -849,12 +1171,17 @@ Readable.prototype.removeListener = function (ev, fn) { // resume within the same tick will have no // effect. nextTick(updateReadableListening, this); + } else if (ev === 'data' && this.listenerCount('data') === 0) { + state[kState] &= ~kDataListening; } + return res; }; Readable.prototype.off = Readable.prototype.removeListener; + Readable.prototype.removeAllListeners = function (ev) { const res = Stream.prototype.removeAllListeners.apply(this, arguments); + if (ev === 'readable' || ev === undefined) { // We need to check if there is someone still listening to // readable and reset the state. However this needs to happen @@ -864,22 +1191,32 @@ Readable.prototype.removeAllListeners = function (ev) { // effect. nextTick(updateReadableListening, this); } + return res; }; function updateReadableListening(self) { const state = self._readableState; - state.readableListening = self.listenerCount('readable') > 0; - if (state.resumeScheduled && state[kPaused] === false) { + + if (self.listenerCount('readable') > 0) { + state[kState] |= kReadableListening; + } else { + state[kState] &= ~kReadableListening; + } + + if ( + (state[kState] & (kHasPaused | kPaused | kResumeScheduled)) === + (kHasPaused | kResumeScheduled) + ) { // Flowing needs to be set to true now, otherwise // the upcoming resume will not flow. - state.flowing = true; + state[kState] |= kHasFlowing | kFlowing; // Crude way to check if we should resume. - } else if (self.listenerCount('data') > 0) { + } else if ((state[kState] & kDataListening) !== 0) { self.resume(); - } else if (!state.readableListening) { - state.flowing = null; + } else if ((state[kState] & kReadableListening) === 0) { + state[kState] &= ~(kHasFlowing | kFlowing); } } @@ -891,46 +1228,55 @@ function nReadingNextTick(self) { // If the user uses them, then switch into old mode. Readable.prototype.resume = function () { const state = this._readableState; - if (!state.flowing) { + if ((state[kState] & kFlowing) === 0) { // We flow only if there is no one listening // for readable, but we still have to call // resume(). - state.flowing = !state.readableListening; + state[kState] |= kHasFlowing; + if ((state[kState] & kReadableListening) === 0) { + state[kState] |= kFlowing; + } else { + state[kState] &= ~kFlowing; + } resume(this, state); } - state[kPaused] = false; + state[kState] |= kHasPaused; + state[kState] &= ~kPaused; return this; }; function resume(stream, state) { - if (!state.resumeScheduled) { - state.resumeScheduled = true; + if ((state[kState] & kResumeScheduled) === 0) { + state[kState] |= kResumeScheduled; nextTick(resume_, stream, state); } } function resume_(stream, state) { - if (!state.reading) { + if ((state[kState] & kReading) === 0) { stream.read(0); } - state.resumeScheduled = false; + + state[kState] &= ~kResumeScheduled; stream.emit('resume'); flow(stream); - if (state.flowing && !state.reading) stream.read(0); + if ((state[kState] & (kFlowing | kReading)) === kFlowing) stream.read(0); } Readable.prototype.pause = function () { - if (this._readableState.flowing !== false) { - this._readableState.flowing = false; + const state = this._readableState; + if ((state[kState] & (kHasFlowing | kFlowing)) !== kHasFlowing) { + state[kState] |= kHasFlowing; + state[kState] &= ~kFlowing; this.emit('pause'); } - this._readableState[kPaused] = true; + state[kState] |= kHasPaused | kPaused; return this; }; function flow(stream) { const state = stream._readableState; - while (state.flowing && stream.read() !== null); + while ((state[kState] & kFlowing) !== 0 && stream.read() !== null); } // Wrap an old-style stream as the async data source. @@ -942,6 +1288,7 @@ Readable.prototype.wrap = function (stream) { // TODO (ronag): Should this.destroy(err) emit // 'error' on the wrapped stream? Would require // a static factory method, e.g. Readable.wrap(stream). + stream.on('data', (chunk) => { if (!this.push(chunk) && stream.pause) { paused = true; @@ -952,15 +1299,19 @@ Readable.prototype.wrap = function (stream) { stream.on('end', () => { this.push(null); }); + stream.on('error', (err) => { errorOrDestroy(this, err); }); + stream.on('close', () => { this.destroy(); }); + stream.on('destroy', () => { this.destroy(); }); + this._read = () => { if (paused && stream.resume) { paused = false; @@ -976,6 +1327,7 @@ Readable.prototype.wrap = function (stream) { this[i] = stream[i].bind(stream); } } + return this; }; @@ -985,17 +1337,16 @@ Readable.prototype[Symbol.asyncIterator] = function () { Readable.prototype.iterator = function (options) { if (options !== undefined) { - validateObject(options, 'options', options); + validateObject(options, 'options'); } return streamToAsyncIterator(this, options); }; function streamToAsyncIterator(stream, options) { if (typeof stream.read !== 'function') { - stream = Readable.wrap(stream, { - objectMode: true, - }); + stream = Readable.wrap(stream, { objectMode: true }); } + const iter = createAsyncIterator(stream, options); iter.stream = stream; return iter; @@ -1003,6 +1354,7 @@ function streamToAsyncIterator(stream, options) { async function* createAsyncIterator(stream, options) { let callback = nop; + function next(resolve) { if (this === stream) { callback(); @@ -1011,19 +1363,16 @@ async function* createAsyncIterator(stream, options) { callback = resolve; } } + stream.on('readable', next); + let error; - const cleanup = eos( - stream, - { - writable: false, - }, - (err) => { - error = err ? aggregateTwoErrors(error, err) : null; - callback(); - callback = nop; - } - ); + const cleanup = eos(stream, { writable: false }, (err) => { + error = err ? aggregateTwoErrors(error, err) : null; + callback(); + callback = nop; + }); + try { while (true) { const chunk = stream.destroyed ? null : stream.read(); @@ -1042,10 +1391,7 @@ async function* createAsyncIterator(stream, options) { throw error; } finally { if ( - (error || - (options === null || options === undefined - ? undefined - : options.destroyOnReturn) !== false) && + (error || options?.destroyOnReturn !== false) && (error === undefined || stream._readableState.autoDestroy) ) { destroyer(stream, null); @@ -1061,6 +1407,7 @@ async function* createAsyncIterator(stream, options) { // userland will fail. Object.defineProperties(Readable.prototype, { readable: { + __proto__: null, get() { const r = this._readableState; // r.readable === false means that this is part of a Duplex stream @@ -1082,38 +1429,48 @@ Object.defineProperties(Readable.prototype, { } }, }, + readableDidRead: { + __proto__: null, enumerable: false, get: function () { - return !!this._readableState?.dataEmitted; + return this._readableState.dataEmitted; }, }, + readableAborted: { + __proto__: null, enumerable: false, get: function () { return !!( - this._readableState?.readable !== false && - (this._readableState?.destroyed || this._readableState?.errored) && - !this._readableState?.endEmitted + this._readableState.readable !== false && + (this._readableState.destroyed || this._readableState.errored) && + !this._readableState.endEmitted ); }, }, + readableHighWaterMark: { + __proto__: null, enumerable: false, get: function () { - return this._readableState?.highWaterMark; + return this._readableState.highWaterMark; }, }, + readableBuffer: { + __proto__: null, enumerable: false, get: function () { return this._readableState?.buffer; }, }, + readableFlowing: { + __proto__: null, enumerable: false, get: function () { - return !!this._readableState?.flowing; + return this._readableState.flowing; }, set: function (state) { if (this._readableState) { @@ -1121,39 +1478,51 @@ Object.defineProperties(Readable.prototype, { } }, }, + readableLength: { + __proto__: null, enumerable: false, get() { - return this._readableState?.length | 0; + return this._readableState.length; }, }, + readableObjectMode: { + __proto__: null, enumerable: false, get() { return this._readableState ? this._readableState.objectMode : false; }, }, + readableEncoding: { + __proto__: null, enumerable: false, get() { - return this._readableState?.encoding || null; + return this._readableState ? this._readableState.encoding : null; }, }, + errored: { + __proto__: null, enumerable: false, get() { - return this._readableState?.errored || null; + return this._readableState ? this._readableState.errored : null; }, }, + closed: { + __proto__: null, get() { - return !!this._readableState?.closed; + return this._readableState ? this._readableState.closed : false; }, }, + destroyed: { + __proto__: null, enumerable: false, get() { - return !!this._readableState?.destroyed; + return this._readableState ? this._readableState.destroyed : false; }, set(value) { // We ignore the value if the stream @@ -1167,10 +1536,12 @@ Object.defineProperties(Readable.prototype, { this._readableState.destroyed = value; }, }, + readableEnded: { + __proto__: null, enumerable: false, get() { - return !!this._readableState?.endEmitted; + return this._readableState ? this._readableState.endEmitted : false; }, }, }); @@ -1178,17 +1549,25 @@ Object.defineProperties(Readable.prototype, { Object.defineProperties(ReadableState.prototype, { // Legacy getter for `pipesCount`. pipesCount: { + __proto__: null, get() { return this.pipes.length; }, }, + // Legacy property for `paused`. paused: { + __proto__: null, get() { - return this[kPaused] !== false; + return (this[kState] & kPaused) !== 0; }, set(value) { - this[kPaused] = !!value; + this[kState] |= kHasPaused; + if (value) { + this[kState] |= kPaused; + } else { + this[kState] &= ~kPaused; + } }, }, }); @@ -1203,25 +1582,111 @@ Readable._fromList = fromList; function fromList(n, state) { // nothing buffered. if (state.length === 0) return null; + + let idx = state.bufferIndex; let ret; - if (state.objectMode) ret = state.buffer.shift(); - else if (!n || n >= state.length) { + + const buf = state.buffer; + const len = buf.length; + + if ((state[kState] & kObjectMode) !== 0) { + ret = buf[idx]; + buf[idx++] = null; + } else if (!n || n >= state.length) { // Read it all, truncate the list. - if (state.decoder) ret = state.buffer.join(''); - else if (state.buffer.length === 1) ret = state.buffer.first(); - else ret = state.buffer.concat(state.length); - state.buffer.clear(); + if ((state[kState] & kDecoder) !== 0) { + ret = ''; + while (idx < len) { + ret += buf[idx]; + buf[idx++] = null; + } + } else if (len - idx === 0) { + ret = Buffer.alloc(0); + } else if (len - idx === 1) { + ret = buf[idx]; + buf[idx++] = null; + } else { + ret = Buffer.allocUnsafe(state.length); + + let i = 0; + while (idx < len) { + ret.set(buf[idx], i); + i += buf[idx].length; + buf[idx++] = null; + } + } + } else if (n < buf[idx].length) { + // `slice` is the same for buffers and strings. + ret = buf[idx].slice(0, n); + buf[idx] = buf[idx].slice(n); + } else if (n === buf[idx].length) { + // First chunk is a perfect match. + ret = buf[idx]; + buf[idx++] = null; + } else if ((state[kState] & kDecoder) !== 0) { + ret = ''; + while (idx < len) { + const str = buf[idx]; + if (n > str.length) { + ret += str; + n -= str.length; + buf[idx++] = null; + } else { + if (n === buf.length) { + ret += str; + buf[idx++] = null; + } else { + ret += str.slice(0, n); + buf[idx] = str.slice(n); + } + break; + } + } } else { - // read part of list. - ret = state.buffer.consume(n, !!state.decoder); + ret = Buffer.allocUnsafe(n); + + const retLen = n; + while (idx < len) { + const data = buf[idx]; + if (n > data.length) { + ret.set(data, retLen - n); + n -= data.length; + buf[idx++] = null; + } else { + if (n === data.length) { + ret.set(data, retLen - n); + buf[idx++] = null; + } else { + ret.set(Buffer.from(data.buffer, data.byteOffset, n), retLen - n); + buf[idx] = Buffer.from( + data.buffer, + data.byteOffset + n, + data.length - n + ); + } + break; + } + } } + + if (idx === len) { + state.buffer.length = 0; + state.bufferIndex = 0; + } else if (idx > 1024) { + state.buffer.splice(0, idx); + state.bufferIndex = 0; + } else { + state.bufferIndex = idx; + } + return ret; } function endReadable(stream) { const state = stream._readableState; - if (!state.endEmitted) { - state.ended = true; + + if ((state[kState] & kEndEmitted) === 0) { + state[kState] |= kEnded; nextTick(endReadableNT, state, stream); } } @@ -1229,13 +1694,12 @@ function endReadable(stream) { function endReadableNT(state, stream) { // Check that we didn't get one last unshift. if ( - !state.errored && - !state.closeEmitted && - !state.endEmitted && + (state[kState] & (kErrored | kCloseEmitted | kEndEmitted)) === 0 && state.length === 0 ) { - state.endEmitted = true; + state[kState] |= kEndEmitted; stream.emit('end'); + if (stream.writable && stream.allowHalfOpen === false) { nextTick(endWritableNT, stream); } else if (state.autoDestroy) { @@ -1248,6 +1712,7 @@ function endReadableNT(state, stream) { // We don't expect the writable to ever 'finish' // if writable is explicitly set to false. (wState.finished || wState.writable === false)); + if (autoDestroy) { stream.destroy(); } @@ -1413,8 +1878,8 @@ function map(fn, options) { validateInteger(concurrency, 'concurrency', 1); return async function* map() { let _options$signal, _options$signal2; - const ac = new AbortController(); - const stream = this; + const ac = new globalThis.AbortController(); + const stream = this; // eslint-disable-line @typescript-eslint/no-this-alias const queue = []; const signal = ac.signal; const signalOpt = { @@ -1422,15 +1887,15 @@ function map(fn, options) { }; const abort = () => ac.abort(); if ( - options !== null && - options !== undefined && + options != null && (_options$signal = options.signal) !== null && _options$signal !== undefined && _options$signal.aborted ) { abort(); } - options === null || options === undefined + // eslint-disable-next-line @typescript-eslint/no-unused-expressions + options == null ? undefined : (_options$signal2 = options.signal) === null || _options$signal2 === undefined @@ -1490,7 +1955,8 @@ function map(fn, options) { next(); next = null; } - options === null || options === undefined + // eslint-disable-next-line @typescript-eslint/no-unused-expressions + options == null ? undefined : (_options$signal3 = options.signal) === null || _options$signal3 === undefined @@ -1537,10 +2003,7 @@ function asIndexedPairs(options) { if (options != null) { validateObject(options, 'options', options); } - if ( - (options === null || options === undefined ? undefined : options.signal) != - null - ) { + if ((options == null ? undefined : options.signal) != null) { validateAbortSignal(options.signal, 'options.signal'); } return async function* asIndexedPairs() { @@ -1657,7 +2120,7 @@ async function reduce(reducer, initialValue, options) { await finished(this.destroy(err)); throw err; } - const ac = new AbortController(); + const ac = new globalThis.AbortController(); const signal = ac.signal; if (options?.signal) { const opts = { @@ -1879,7 +2342,7 @@ export function newReadableStreamFromStreamReadable( } if (isDestroyed(streamReadable) || !isReadable(streamReadable)) { - const readable = new ReadableStream(); + const readable = new globalThis.ReadableStream(); readable.cancel(); return readable; } @@ -1894,19 +2357,16 @@ export function newReadableStreamFromStreamReadable( if (objectMode) { // When running in objectMode explicitly but no strategy, we just fall // back to CountQueuingStrategy - return new CountQueuingStrategy({ highWaterMark }); + return new globalThis.CountQueuingStrategy({ highWaterMark }); } - // When not running in objectMode explicitly, we just fall - // back to a minimal strategy that just specifies the highWaterMark - // and no size algorithm. Using a ByteLengthQueuingStrategy here - // is unnecessary. - return { highWaterMark }; + return new globalThis.ByteLengthQueuingStrategy({ highWaterMark }); }; const strategy = evaluateStrategyOrFallback(options?.strategy); let controller; + let wasCanceled = false; function onData(chunk) { // Copy the Buffer to detach it from the pool. @@ -1918,22 +2378,23 @@ export function newReadableStreamFromStreamReadable( streamReadable.pause(); const cleanup = eos(streamReadable, (error) => { - if (error?.code === 'ERR_STREAM_PREMATURE_CLOSE') { - const err = new AbortError(undefined, { cause: error }); - error = err; - } + error = handleKnownInternalErrors(error); cleanup(); // This is a protection against non-standard, legacy streams // that happen to emit an error event again after finished is called. streamReadable.on('error', () => {}); if (error) return controller.error(error); + // Was already canceled + if (wasCanceled) { + return; + } controller.close(); }); streamReadable.on('data', onData); - return new ReadableStream( + return new globalThis.ReadableStream( { start(c) { controller = c; @@ -1944,8 +2405,8 @@ export function newReadableStreamFromStreamReadable( }, cancel(reason) { - ERR_STREAM_PREMATURE_CLOSE; - destroy.call(streamReadable, reason); + wasCanceled = true; + destroy(streamReadable, reason); }, type: createTypeBytes ? 'bytes' : undefined, }, @@ -1967,7 +2428,7 @@ export function newStreamReadableFromReadableStream( readableStream, options = {} ) { - if (!(readableStream instanceof ReadableStream)) { + if (!isReadableStream(readableStream)) { throw new ERR_INVALID_ARG_TYPE( 'readableStream', 'ReadableStream', @@ -1979,7 +2440,7 @@ export function newStreamReadableFromReadableStream( const { highWaterMark, encoding, objectMode = false, signal } = options; if (encoding !== undefined && !Buffer.isEncoding(encoding)) - throw new ERR_INVALID_ARG_VALUE(encoding, 'options.encoding'); + throw new ERR_INVALID_ARG_VALUE('options.encoding', encoding); validateBoolean(objectMode, 'options.objectMode'); const reader = readableStream.getReader(); @@ -2001,9 +2462,7 @@ export function newStreamReadableFromReadableStream( readable.push(chunk.value); } }, - (error) => { - destroy.call(readable, error); - } + (error) => destroy.call(readable, error) ); }, diff --git a/src/node/internal/streams_state.ts b/src/node/internal/streams_state.ts index a3dca080bc9..a0826177e69 100644 --- a/src/node/internal/streams_state.ts +++ b/src/node/internal/streams_state.ts @@ -23,18 +23,15 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. -import { ERR_INVALID_ARG_VALUE } from 'node-internal:internal_errors'; import { validateInteger } from 'node-internal:validators'; +import { ERR_INVALID_ARG_VALUE } from 'node-internal:internal_errors'; let defaultHighWaterMarkBytes = 64 * 1024; let defaultHighWaterMarkObjectMode = 16; -export type HighWaterMarkFromOptions = { - highWaterMark?: number; - [key: string]: number | undefined; -}; +export type HighWaterMarkFromOptions = { highWaterMark?: number }; -export function highWaterMarkFrom( +function highWaterMarkFrom( options: HighWaterMarkFromOptions, isDuplex: boolean, duplexKey: string @@ -42,18 +39,12 @@ export function highWaterMarkFrom( return options.highWaterMark != null ? options.highWaterMark : isDuplex - ? (options[duplexKey] as number) + ? // @ts-expect-error TS7053 Fix this soon. + (options[duplexKey] as number) : null; } -// By default Node.js uses: -// - Object mode: 16 -// - Non-object mode: -// - Windows: 16 * 1024 -// - Non-windows: 64 * 1024 -// Ref: https://github.com/nodejs/node/blob/d9fe28bd6b7836accff5a174ef76f7340bf5e600/lib/internal/streams/state.js#L12 -// We always return 64 * 1024 to be in par with production. -export function getDefaultHighWaterMark(objectMode: boolean = false): number { +export function getDefaultHighWaterMark(objectMode?: boolean): number { return objectMode ? defaultHighWaterMarkObjectMode : defaultHighWaterMarkBytes; @@ -81,7 +72,7 @@ export function getHighWaterMark( if (hwm != null) { if (!Number.isInteger(hwm) || hwm < 0) { const name = isDuplex ? `options.${duplexKey}` : 'options.highWaterMark'; - throw new ERR_INVALID_ARG_VALUE(name, hwm, name); + throw new ERR_INVALID_ARG_VALUE(name, hwm); } return Math.floor(hwm); } diff --git a/src/node/internal/streams_transform.d.ts b/src/node/internal/streams_transform.d.ts index 9890346871d..f78c2d5ef68 100644 --- a/src/node/internal/streams_transform.d.ts +++ b/src/node/internal/streams_transform.d.ts @@ -2,21 +2,21 @@ // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 +import type EventEmitter from 'node:events'; +import { Transform as _Transform } from 'node:stream'; +import type { ReadableState } from 'node-internal:streams_readable'; +import type { WritableState } from 'node-internal:streams_writable'; import { kIsWritable, - kDestroyed, kIsReadable, + kIsDestroyed, } from 'node-internal:streams_util'; -import { Transform as _Transform } from 'node:stream'; -import type EventEmitter from 'node:events'; -import type { ReadableState } from 'node-internal:streams_readable'; -import type { WritableState } from 'node-internal:streams_writable'; - export { Readable, Writable, Duplex, + Transform as _Transform, Stream, TransformOptions, TransformCallback, @@ -27,10 +27,9 @@ export { export declare class Transform extends _Transform { _writableState: WritableState; _readableState: ReadableState; - _closed: boolean; - writableErrored: boolean; readableErrored: boolean; + _closed: boolean; readable: boolean; writable: boolean; errored?: Error | null; @@ -42,5 +41,5 @@ export declare class Transform extends _Transform { [kIsWritable]: boolean; [kIsReadable]: boolean; - [kDestroyed]: boolean; + [kIsDestroyed]: boolean; } diff --git a/src/node/internal/streams_transform.js b/src/node/internal/streams_transform.js index 4917cf39944..0e7874eecfb 100644 --- a/src/node/internal/streams_transform.js +++ b/src/node/internal/streams_transform.js @@ -68,11 +68,15 @@ 'use strict'; import { ERR_METHOD_NOT_IMPLEMENTED } from 'node-internal:internal_errors'; +import { nextTick } from 'node-internal:internal_process'; import { Duplex } from 'node-internal:streams_duplex'; import { getHighWaterMark } from 'node-internal:streams_state'; +const streamsNodejsV24Compat = + Cloudflare.compatibilityFlags.enable_streams_nodejs_v24_compat; // eslint-disable-line no-undef + Object.setPrototypeOf(Transform.prototype, Duplex.prototype); Object.setPrototypeOf(Transform, Duplex); @@ -85,7 +89,7 @@ export function Transform(options) { // applied but would be semver-major. Or even better; // make Transform a Readable with the Writable interface. const readableHighWaterMark = options - ? getHighWaterMark(options, 'readableHighWaterMark', true) + ? getHighWaterMark(this, options, 'readableHighWaterMark', true) : null; if (readableHighWaterMark === 0) { // A Duplex will buffer both on the writable and readable side while @@ -95,11 +99,7 @@ export function Transform(options) { ...options, highWaterMark: null, readableHighWaterMark, - // TODO (ronag): 0 is not optimal since we have - // a "bug" where we check needDrain before calling _write and not after. - // Refs: https://github.com/nodejs/node/pull/32887 - // Refs: https://github.com/nodejs/node/pull/35941 - writableHighWaterMark: options?.writableHighWaterMark || 0, + writableHighWaterMark: options.writableHighWaterMark || 0, }; } Duplex.call(this, options); @@ -172,7 +172,13 @@ Transform.prototype._write = function (chunk, encoding, callback) { if (val != null) { this.push(val); } - if ( + // This is a semver-major change. Ref: https://github.com/nodejs/node/commit/557044af407376aff28a0a0800f3053bb58e9239 + if (streamsNodejsV24Compat && rState.ended) { + // If user has called this.push(null) we have to delay the callback to properly propagate the new + // state. + nextTick(callback); + return; + } else if ( wState.ended || // Backwards compat. length === rState.length || diff --git a/src/node/internal/streams_util.ts b/src/node/internal/streams_util.ts index ae75ad53c13..5b66fdf54e4 100644 --- a/src/node/internal/streams_util.ts +++ b/src/node/internal/streams_util.ts @@ -23,74 +23,71 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. -/* eslint-disable @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-assignment,@typescript-eslint/restrict-plus-operands,@typescript-eslint/no-unsafe-return,@typescript-eslint/no-unnecessary-condition,@typescript-eslint/no-unnecessary-type-conversion,@typescript-eslint/no-unnecessary-boolean-literal-compare,no-var */ +/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unnecessary-condition */ -import { Buffer } from 'node-internal:internal_buffer'; +import { AbortError } from 'node-internal:internal_errors'; +import { constants } from 'node-internal:internal_zlib_constants'; -import type { Readable } from 'node-internal:streams_readable'; import type { Writable } from 'node-internal:streams_writable'; +import type { Readable } from 'node-internal:streams_readable'; import type { Transform } from 'node-internal:streams_transform'; -import type { Duplex } from 'node-internal:streams_duplex'; import type { OutgoingMessage } from 'node-internal:internal_http_outgoing'; import type { ServerResponse } from 'node-internal:internal_http_server'; +import type { IncomingMessage } from 'node-internal:internal_http_incoming'; -export const kDestroyed = Symbol('kDestroyed'); -export const kIsErrored = Symbol('kIsErrored'); -export const kIsReadable = Symbol('kIsReadable'); -export const kIsDisturbed = Symbol('kIsDisturbed'); -export const kPaused = Symbol('kPaused'); -export const kOnFinished = Symbol('kOnFinished'); - -type NodeStream = Readable | Writable | Transform | Duplex; - +// We need to use Symbol.for to make these globally available +// for interoperability with readable-stream, i.e. readable-stream +// and node core needs to be able to read/write private state +// from each other for proper interoperability. export const kIsDestroyed = Symbol.for('nodejs.stream.destroyed'); +export const kIsErrored = Symbol.for('nodejs.stream.errored'); +export const kIsReadable = Symbol.for('nodejs.stream.readable'); export const kIsWritable = Symbol.for('nodejs.stream.writable'); +export const kIsDisturbed = Symbol.for('nodejs.stream.disturbed'); + export const kOnConstructed = Symbol('kOnConstructed'); +export const kIsClosedPromise = Symbol.for('nodejs.webstream.isClosedPromise'); +export const kControllerErrorFunction = Symbol.for( + 'nodejs.webstream.controllerErrorFunction' +); + +export const kState = Symbol('kState'); +export const kObjectMode = 1 << 0; +export const kErrorEmitted = 1 << 1; +export const kAutoDestroy = 1 << 2; +export const kEmitClose = 1 << 3; +export const kDestroyed = 1 << 4; +export const kClosed = 1 << 5; +export const kCloseEmitted = 1 << 6; +export const kErrored = 1 << 7; +export const kConstructed = 1 << 8; + export function isReadableNodeStream( - // eslint-disable-next-line @typescript-eslint/no-explicit-any obj: any, strict: boolean = false -): obj is Readable { - let _obj$_readableState; +): boolean { return !!( - ( - obj && - typeof obj.pipe === 'function' && - typeof obj.on === 'function' && - (!strict || - (typeof obj.pause === 'function' && - typeof obj.resume === 'function')) && - (!obj._writableState || - ((_obj$_readableState = obj._readableState) === null || - _obj$_readableState === undefined - ? undefined - : _obj$_readableState.readable) !== false) && - // Duplex - (!obj._writableState || obj._readableState) - ) // Writable has .pipe. + obj && + typeof obj.pipe === 'function' && + typeof obj.on === 'function' && + (!strict || + (typeof obj.pause === 'function' && typeof obj.resume === 'function')) && + (!obj._writableState || obj._readableState?.readable !== false) && // Duplex + (!obj._writableState || obj._readableState) // Writable has .pipe. ); } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export function isWritableNodeStream(obj: any): obj is Writable { - let _obj$_writableState; +export function isWritableNodeStream(obj: any): boolean { return !!( - ( - obj && - typeof obj.write === 'function' && - typeof obj.on === 'function' && - (!obj._readableState || - ((_obj$_writableState = obj._writableState) === null || - _obj$_writableState === undefined - ? undefined - : _obj$_writableState.writable) !== false) - ) // Duplex + obj && + typeof obj.write === 'function' && + typeof obj.on === 'function' && + (!obj._readableState || obj._writableState?.writable !== false) // Duplex ); } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export function isDuplexNodeStream(obj: any): obj is Duplex { +export function isDuplexNodeStream(obj: any): boolean { return !!( obj && typeof obj.pipe === 'function' && @@ -100,18 +97,17 @@ export function isDuplexNodeStream(obj: any): obj is Duplex { ); } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export function isNodeStream(obj: any): obj is NodeStream { +export function isNodeStream(obj: any): obj is Readable | Writable | Transform { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return return ( obj && - (obj._readableState || - obj._writableState || + (obj._readableState != null || + obj._writableState != null || (typeof obj.write === 'function' && typeof obj.on === 'function') || (typeof obj.pipe === 'function' && typeof obj.on === 'function')) ); } -// eslint-disable-next-line @typescript-eslint/no-explicit-any export function isReadableStream(obj: any): obj is ReadableStream { return !!( obj && @@ -122,7 +118,6 @@ export function isReadableStream(obj: any): obj is ReadableStream { ); } -// eslint-disable-next-line @typescript-eslint/no-explicit-any export function isWritableStream(obj: any): obj is WritableStream { return !!( obj && @@ -132,7 +127,23 @@ export function isWritableStream(obj: any): obj is WritableStream { ); } -// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function isTransformStream(obj: any): obj is TransformStream { + return !!( + obj && + !isNodeStream(obj) && + typeof obj.readable === 'object' && + typeof obj.writable === 'object' + ); +} + +export function isWebStream( + obj: any +): obj is ReadableStream | WritableStream | TransformStream { + return ( + isReadableStream(obj) || isWritableStream(obj) || isTransformStream(obj) + ); +} + export function isIterable(obj: any, isAsync?: true | false): boolean { if (obj == null) return false; if (isAsync === true) return typeof obj[Symbol.asyncIterator] === 'function'; @@ -143,69 +154,76 @@ export function isIterable(obj: any, isAsync?: true | false): boolean { ); } -export function isDestroyed(stream: unknown): boolean | null { +export function isDestroyed(stream: any): boolean | null { if (!isNodeStream(stream)) return null; const wState = stream._writableState; const rState = stream._readableState; const state = wState || rState; - return !!(stream.destroyed || stream[kDestroyed] || state?.destroyed); + return !!(stream.destroyed || stream[kIsDestroyed] || state?.destroyed); } -export function isWritableEnded(stream: Writable): boolean | null { +// Have been end():d. +export function isWritableEnded( + stream: Writable | Readable | Transform +): boolean | null { if (!isWritableNodeStream(stream)) return null; if (stream.writableEnded === true) return true; const wState = stream._writableState; - if (wState != null && wState.errored) return false; - if (typeof (wState == null ? undefined : wState.ended) !== 'boolean') - return null; - return wState.ended ?? false; + if (wState?.errored) return false; + if (typeof wState?.ended !== 'boolean') return null; + return wState.ended; } +// Have emitted 'finish'. export function isWritableFinished( - stream: NodeStream, - strict = false + stream: Writable | Readable | Transform, + strict?: true | false | null ): boolean | null { if (!isWritableNodeStream(stream)) return null; if (stream.writableFinished === true) return true; const wState = stream._writableState; - if (wState != null && wState.errored) return false; - if (typeof (wState == null ? undefined : wState.finished) !== 'boolean') - return null; + if (wState?.errored) return false; + if (typeof wState?.finished !== 'boolean') return null; + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-conversion return !!( wState.finished || (strict === false && wState.ended === true && wState.length === 0) ); } -export function isReadableEnded(stream: NodeStream): boolean | null { +// Have been push(null):d. +export function isReadableEnded( + stream: Readable | Writable | Transform +): boolean | null { if (!isReadableNodeStream(stream)) return null; if (stream.readableEnded === true) return true; const rState = stream._readableState; if (!rState || rState.errored) return false; - if (typeof (rState == null ? undefined : rState.ended) !== 'boolean') - return null; - return rState.ended ?? false; + if (typeof rState?.ended !== 'boolean') return null; + return rState.ended; } +// Have emitted 'end'. export function isReadableFinished( - stream: Readable, - strict = false + stream: Readable | Writable | Transform, + strict?: boolean ): boolean | null { if (!isReadableNodeStream(stream)) return null; const rState = stream._readableState; - if (rState != null && rState.errored) return false; - if (typeof (rState == null ? undefined : rState.endEmitted) !== 'boolean') - return null; + if (rState?.errored) return false; + if (typeof rState?.endEmitted !== 'boolean') return null; + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-conversion return !!( rState.endEmitted || (strict === false && rState.ended === true && rState.length === 0) ); } -export function isReadable(stream: NodeStream): boolean | null { +export function isReadable( + stream: Readable | Writable | Transform +): boolean | null { if (stream && stream[kIsReadable] != null) return stream[kIsReadable]; - if (typeof (stream == null ? undefined : stream.readable) !== 'boolean') - return null; + if (typeof stream?.readable !== 'boolean') return null; if (isDestroyed(stream)) return false; return ( isReadableNodeStream(stream) && @@ -214,9 +232,11 @@ export function isReadable(stream: NodeStream): boolean | null { ); } -export function isWritable(stream: NodeStream): boolean | null { - if (typeof (stream == null ? undefined : stream.writable) !== 'boolean') - return null; +export function isWritable( + stream: Readable | Writable | Transform +): boolean | null { + if (stream && stream[kIsWritable] != null) return stream[kIsWritable]; + if (typeof stream?.writable !== 'boolean') return null; if (isDestroyed(stream)) return false; return ( isWritableNodeStream(stream) && stream.writable && !isWritableEnded(stream) @@ -224,378 +244,169 @@ export function isWritable(stream: NodeStream): boolean | null { } export function isFinished( - stream: unknown, - opts: { readable?: boolean; writable?: boolean } = {} + stream: Readable | Writable | Transform, + opts?: { readable?: boolean; writable?: boolean } ): boolean | null { if (!isNodeStream(stream)) { return null; } + if (isDestroyed(stream)) { return true; } - if ( - (opts == null ? undefined : opts.readable) !== false && - isReadable(stream) - ) { + + if (opts?.readable !== false && isReadable(stream)) { return false; } - if ( - (opts == null ? undefined : opts.writable) !== false && - isWritable(stream) - ) { + + if (opts?.writable !== false && isWritable(stream)) { return false; } + return true; } -export function isWritableErrored(stream: unknown): Error | boolean | null { - let _stream$_writableStat, _stream$_writableStat2; +export function isWritableErrored( + stream: Writable | Readable | Transform +): Error | boolean | null { if (!isNodeStream(stream)) { return null; } + if (stream.writableErrored) { return stream.writableErrored; } - return (_stream$_writableStat = - (_stream$_writableStat2 = stream._writableState) === null || - _stream$_writableStat2 === undefined - ? undefined - : _stream$_writableStat2.errored) !== null && - _stream$_writableStat !== undefined - ? _stream$_writableStat - : null; + + return stream._writableState?.errored ?? null; } -export function isReadableErrored(stream: unknown): Error | boolean | null { - let _stream$_readableStat, _stream$_readableStat2; +export function isReadableErrored( + stream: Readable | Writable | Transform +): Error | boolean | null { if (!isNodeStream(stream)) { return null; } + if (stream.readableErrored) { return stream.readableErrored; } - return (_stream$_readableStat = - (_stream$_readableStat2 = stream._readableState) === null || - _stream$_readableStat2 === undefined - ? undefined - : _stream$_readableStat2.errored) !== null && - _stream$_readableStat !== undefined - ? _stream$_readableStat - : null; + + return stream._readableState?.errored ?? null; } -export function isClosed(stream: unknown): boolean | null | undefined { +export function isClosed( + stream: Readable | Writable | Transform +): boolean | null { if (!isNodeStream(stream)) { return null; } + if (typeof stream.closed === 'boolean') { return stream.closed; } + const wState = stream._writableState; const rState = stream._readableState; + if ( - typeof (wState === null || wState === undefined - ? undefined - : wState.closed) === 'boolean' || - typeof (rState === null || rState === undefined - ? undefined - : rState.closed) === 'boolean' + typeof wState?.closed === 'boolean' || + typeof rState?.closed === 'boolean' ) { - return ( - (wState === null || wState === undefined ? undefined : wState.closed) || - (rState === null || rState === undefined ? undefined : rState.closed) - ); + return (wState?.closed || rState?.closed) ?? null; } + if (typeof stream._closed === 'boolean' && isOutgoingMessage(stream)) { return stream._closed; } + return null; } -// TODO(later): We do not actually support OutgoingMessage yet. Might not ever? -// Keeping this here tho just to keep things simple. export function isOutgoingMessage(stream: unknown): stream is OutgoingMessage { return ( stream != null && typeof stream === 'object' && '_closed' in stream && - '_defaultKeepAlive' in stream && - '_removedConnection' in stream && - '_removedContLen' in stream && typeof stream._closed === 'boolean' && + '_defaultKeepAlive' in stream && typeof stream._defaultKeepAlive === 'boolean' && + '_removedConnection' in stream && typeof stream._removedConnection === 'boolean' && + '_removedContLen' in stream && typeof stream._removedContLen === 'boolean' ); } -// TODO(later): We do not actually support Server Response yet. Might not ever? -// Keeping this here tho just to keep things simple. +// This function includes the following check that we don't include, because +// our ServerResponse implementation does not implement it. +// `typeof stream._sent100 === 'boolean'` export function isServerResponse(stream: unknown): stream is ServerResponse { - return ( - stream != null && - typeof stream === 'object' && - '_sent100' in stream && - typeof stream._sent100 === 'boolean' && - isOutgoingMessage(stream) - ); + return isOutgoingMessage(stream); } -// TODO(later): We do not actually support Server Request yet. Might not ever? -// Keeping this here tho just to keep things simple. -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export function isServerRequest(stream: any): boolean { - let _stream$req; +export function isServerRequest(stream: any): stream is IncomingMessage { return ( typeof stream._consuming === 'boolean' && typeof stream._dumped === 'boolean' && - ((_stream$req = stream.req) === null || _stream$req === undefined - ? undefined - : _stream$req.upgradeOrConnect) === undefined + stream.req?.upgradeOrConnect === undefined ); } -export function willEmitClose(stream: unknown): boolean | null { +export function willEmitClose(stream: any): boolean | null { if (!isNodeStream(stream)) return null; + const wState = stream._writableState; const rState = stream._readableState; const state = wState || rState; + return ( (!state && isServerResponse(stream)) || - !!(state && state.autoDestroy && state.emitClose && state.closed === false) + !!(state?.autoDestroy && state.emitClose && state.closed === false) ); } -// eslint-disable-next-line @typescript-eslint/no-explicit-any export function isDisturbed(stream: any): boolean { - let _stream$kIsDisturbed; return !!( stream && - ((_stream$kIsDisturbed = stream[kIsDisturbed]) !== null && - _stream$kIsDisturbed !== undefined - ? _stream$kIsDisturbed - : stream.readableDidRead || stream.readableAborted) + (stream[kIsDisturbed] ?? (stream.readableDidRead || stream.readableAborted)) ); } -// eslint-disable-next-line @typescript-eslint/no-explicit-any export function isErrored(stream: any): boolean { - var _ref, - _ref2, - _ref3, - _ref4, - _ref5, - _stream$kIsErrored, - _stream$_readableStat3, - _stream$_writableStat3, - _stream$_readableStat4, - _stream$_writableStat4; return !!( stream && - ((_ref = - (_ref2 = - (_ref3 = - (_ref4 = - (_ref5 = - (_stream$kIsErrored = stream[kIsErrored]) !== null && - _stream$kIsErrored !== undefined - ? _stream$kIsErrored - : stream.readableErrored) !== null && _ref5 !== undefined - ? _ref5 - : stream.writableErrored) !== null && _ref4 !== undefined - ? _ref4 - : (_stream$_readableStat3 = stream._readableState) === null || - _stream$_readableStat3 === undefined - ? undefined - : _stream$_readableStat3.errorEmitted) !== null && - _ref3 !== undefined - ? _ref3 - : (_stream$_writableStat3 = stream._writableState) === null || - _stream$_writableStat3 === undefined - ? undefined - : _stream$_writableStat3.errorEmitted) !== null && - _ref2 !== undefined - ? _ref2 - : (_stream$_readableStat4 = stream._readableState) === null || - _stream$_readableStat4 === undefined - ? undefined - : _stream$_readableStat4.errored) !== null && _ref !== undefined - ? _ref - : (_stream$_writableStat4 = stream._writableState) === null || - _stream$_writableStat4 === undefined - ? undefined - : _stream$_writableStat4.errored) + (stream[kIsErrored] ?? + stream.readableErrored ?? + stream.writableErrored ?? + stream._readableState?.errorEmitted ?? + stream._writableState?.errorEmitted ?? + stream._readableState?.errored ?? + stream._writableState?.errored) ); } -export const nop = (): void => {}; - -type CallbackFunction = (...args: unknown[]) => void; - -export function once(callback: T): T { - let called = false; - return function (this: unknown, ...args: unknown[]) { - if (called) { - return; +const ZLIB_FAILURES = new Set([ + ...Object.entries(constants).flatMap(([code, value]) => + value < 0 ? code : [] + ), + 'Z_NEED_DICT', +]); + +export function handleKnownInternalErrors( + cause?: Error & { code?: string } +): (Error & { code?: string }) | undefined { + switch (true) { + case cause?.code === 'ERR_STREAM_PREMATURE_CLOSE': { + return new AbortError(undefined, { cause }); } - called = true; - callback.apply(this, args); - } as T; -} - -// ====================================================================================== -// BufferList - -export type BufferListEntry = { - data: Buffer; - next: BufferListEntry | null; -}; - -export class BufferList { - head: BufferListEntry | null = null; - tail: BufferListEntry | null = null; - length = 0; - - push(v: Buffer): void { - const entry = { - data: v, - next: null, - }; - if (this.length > 0 && this.tail != null) this.tail.next = entry; - else this.head = entry; - this.tail = entry; - ++this.length; - } - unshift(v: Buffer): void { - const entry = { - data: v, - next: this.head, - }; - if (this.length === 0) this.tail = entry; - this.head = entry; - ++this.length; - } - shift(): Buffer | undefined { - if (this.length === 0) return; - const ret = this.head?.data as Buffer; - if (this.length === 1) this.head = this.tail = null; - else this.head = this.head?.next ?? null; - --this.length; - return ret; - } - - clear(): void { - this.head = this.tail = null; - this.length = 0; - } - - join(s: Buffer): string { - if (this.length === 0) return ''; - let p = this.head; - let ret = '' + p?.data; - while ((p = p?.next ?? null) != null) ret += s + String(p.data); - return ret; - } - - concat(n: number): Buffer { - if (this.length === 0) return Buffer.alloc(0); - const ret = Buffer.allocUnsafe(n >>> 0); - let p = this.head; - let i = 0; - while (p) { - ret.set(p.data, i); - i += p.data.length; - p = p.next; - } - return ret; - } - - consume(n: number, hasStrings: boolean = false): Buffer | string | undefined { - const data = this.head?.data as Buffer; - if (n < data.length) { - // `slice` is the same for buffers and strings. - const slice = data.slice(0, n); - if (this.head != null) { - this.head.data = data.slice(n) as Buffer; - } - return slice as Buffer; + case ZLIB_FAILURES.has(cause?.code ?? ''): { + const error = new TypeError(undefined, { cause }) as Error & { + code?: string; + }; + error.code = cause?.code as string; + return error; } - if (n === data.length) { - // First chunk is a perfect match. - return this.shift(); - } - // Result spans more than one buffer. - return hasStrings ? this._getString(n) : this._getBuffer(n); - } - - first(): Buffer | null { - return this.head?.data ?? null; - } - - *[Symbol.iterator](): Generator { - for (let p = this.head; p; p = p.next) { - yield p.data; - } - } - - _getString(n: number): string { - let ret = ''; - let p = this.head; - let c = 0; - do { - const str = p?.data as Buffer; - if (n > str.length) { - ret += str; - n -= str.length; - } else { - if (n === str.length) { - ret += str; - ++c; - if (p?.next) this.head = p.next; - else this.head = this.tail = null; - } else { - ret += str.slice(0, n); - this.head = p; - if (p != null) { - p.data = str.slice(n) as Buffer; - } - } - break; - } - ++c; - } while ((p = p?.next ?? null) != null); - this.length -= c; - return ret; - } - - _getBuffer(n: number): Buffer { - const ret = Buffer.allocUnsafe(n); - const retLen = n; - let p = this.head; - let c = 0; - do { - const buf = p?.data as Buffer; - if (n > buf.length) { - ret.set(buf, retLen - n); - n -= buf.length; - } else { - if (n === buf.length) { - ret.set(buf, retLen - n); - ++c; - if (p?.next) this.head = p.next; - else this.head = this.tail = null; - } else { - ret.set(new Uint8Array(buf.buffer, buf.byteOffset, n), retLen - n); - this.head = p; - if (p != null) { - p.data = buf.slice(n) as Buffer; - } - } - break; - } - ++c; - } while ((p = p?.next ?? null) != null); - this.length -= c; - return ret; + default: + return cause; } } diff --git a/src/node/internal/streams_writable.d.ts b/src/node/internal/streams_writable.d.ts index 1c43e1b4fd9..30df5ac5d59 100644 --- a/src/node/internal/streams_writable.d.ts +++ b/src/node/internal/streams_writable.d.ts @@ -5,9 +5,11 @@ import type { OutgoingMessage } from 'node:http'; import { Writable as _Writable, Duplex } from 'node:stream'; import { - kDestroyed, - kIsReadable, + kState, kIsWritable, + kIsReadable, + kIsClosedPromise, + kIsDestroyed, } from 'node-internal:streams_util'; export declare class WritableState { @@ -32,6 +34,7 @@ export declare class WritableState { prefinished?: boolean; finalCalled?: boolean; writelen?: number; + [kState]: number; } export declare class Writable extends _Writable { @@ -54,6 +57,8 @@ export declare class Writable extends _Writable { [kDestroyed]: boolean; [kIsReadable]: boolean; [kIsWritable]: boolean; + [kIsDestroyed]: boolean; + [kIsClosedPromise]: Promise; } export const fromWeb: typeof Duplex.fromWeb; diff --git a/src/node/internal/streams_writable.js b/src/node/internal/streams_writable.js index 3d431791407..0cbb5f4b98a 100644 --- a/src/node/internal/streams_writable.js +++ b/src/node/internal/streams_writable.js @@ -22,37 +22,35 @@ // DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. -/* eslint-disable */ import { EventEmitter } from 'node-internal:events'; - +import * as destroyImpl from 'node-internal:streams_destroy'; import { Stream } from 'node-internal:streams_legacy'; - import { Buffer } from 'node-internal:internal_buffer'; import { nextTick } from 'node-internal:internal_process'; import { normalizeEncoding } from 'node-internal:internal_utils'; import { validateBoolean, validateObject } from 'node-internal:validators'; - -import { addAbortSignal } from 'node-internal:streams_add_abort_signal'; - import { - nop, - kOnFinished, - isWritableEnded, - isWritable, - isDestroyed, + kState, + // bitfields + kObjectMode, + kErrorEmitted, + kAutoDestroy, + kEmitClose, + kDestroyed, + kClosed, + kCloseEmitted, + kErrored, + kConstructed, kOnConstructed, + isDestroyed, + isWritable, + isWritableEnded, + isWritableStream, + handleKnownInternalErrors, } from 'node-internal:streams_util'; - -import { eos } from 'node-internal:streams_end_of_stream'; - -import { - construct, - destroy, - undestroy, - errorOrDestroy, -} from 'node-internal:streams_destroy'; - +import { finished, eos, nop } from 'node-internal:streams_end_of_stream'; +import { addAbortSignal } from 'node-internal:streams_add_abort_signal'; import { getHighWaterMark, getDefaultHighWaterMark, @@ -72,153 +70,339 @@ import { ERR_STREAM_PREMATURE_CLOSE, } from 'node-internal:internal_errors'; -const encoder = new TextEncoder(); - -// ====================================================================================== -// WritableState - -export function WritableState(options, stream, isDuplex) { - // Duplex streams are both readable and writable, but share - // the same options object. - // However, some cases require setting options to different - // values for the readable and the writable sides of the duplex stream, - // e.g. options.readableObjectMode vs. options.writableObjectMode, etc. - +const streamsNodejsV24Compat = + Cloudflare.compatibilityFlags.enable_streams_nodejs_v24_compat; // eslint-disable-line no-undef + +const encoder = new globalThis.TextEncoder(); + +const kOnFinishedValue = Symbol('kOnFinishedValue'); +const kErroredValue = Symbol('kErroredValue'); +const kDefaultEncodingValue = Symbol('kDefaultEncodingValue'); +const kWriteCbValue = Symbol('kWriteCbValue'); +const kAfterWriteTickInfoValue = Symbol('kAfterWriteTickInfoValue'); +const kBufferedValue = Symbol('kBufferedValue'); + +// Bitfield flag constants for WritableState. Each constant uses left-shift (<<) to set a specific +// bit position, allowing multiple boolean flags to be stored efficiently in a single integer (kState). +// For example, `1 << 9` creates a value with only bit 9 set (value: 512). +const kSync = 1 << 9; +const kFinalCalled = 1 << 10; +const kNeedDrain = 1 << 11; +const kEnding = 1 << 12; +const kFinished = 1 << 13; +const kDecodeStrings = 1 << 14; +const kWriting = 1 << 15; +const kBufferProcessing = 1 << 16; +const kPrefinished = 1 << 17; +const kAllBuffers = 1 << 18; +const kAllNoop = 1 << 19; +const kOnFinished = 1 << 20; +const kHasWritable = 1 << 21; +const kWritable = 1 << 22; +const kCorked = 1 << 23; +const kDefaultUTF8Encoding = 1 << 24; +const kWriteCb = 1 << 25; +const kExpectWriteCb = 1 << 26; +const kAfterWriteTickInfo = 1 << 27; +const kAfterWritePending = 1 << 28; +const kBuffered = 1 << 29; +const kEnded = 1 << 30; + +// TODO(benjamingr) it is likely slower to do it this way than with free functions +function makeBitMapDescriptor(bit) { + return { + // This is not a breaking change according to Node.js but better safe than sorry. + // Ref: https://github.com/nodejs/node/pull/49834 + enumerable: !streamsNodejsV24Compat, + get() { + return (this[kState] & bit) !== 0; + }, + set(value) { + if (value) this[kState] |= bit; + else this[kState] &= ~bit; + }, + }; +} +Object.defineProperties(WritableState.prototype, { // Object stream flag to indicate whether or not this stream // contains buffers or objects. - this.objectMode = !!options?.objectMode; - if (isDuplex) - this.objectMode = this.objectMode || !!options?.writableObjectMode; - - // The point at which write() starts returning false - // Note: 0 is a valid value, means that we always return false if - // the entire buffer is not flushed immediately on write(). - this.highWaterMark = options - ? getHighWaterMark(this, options, 'writableHighWaterMark', isDuplex) - : getDefaultHighWaterMark(false); + objectMode: makeBitMapDescriptor(kObjectMode), // if _final has been called. - this.finalCalled = false; + finalCalled: makeBitMapDescriptor(kFinalCalled), // drain event flag. - this.needDrain = false; + needDrain: makeBitMapDescriptor(kNeedDrain), + // At the start of calling end() - this.ending = false; + ending: makeBitMapDescriptor(kEnding), + // When end() has been called, and returned. - this.ended = false; + ended: makeBitMapDescriptor(kEnded), + // When 'finish' is emitted. - this.finished = false; + finished: makeBitMapDescriptor(kFinished), - // Has it been destroyed - this.destroyed = false; + // Has it been destroyed. + destroyed: makeBitMapDescriptor(kDestroyed), // Should we decode strings into buffers before passing to _write? // this is here so that some node-core streams can optimize string // handling at a lower level. - const noDecode = !!(options?.decodeStrings === false); - this.decodeStrings = !noDecode; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options?.defaultEncoding || 'utf8'; - - // Not an actual buffer we keep track of, but a measurement - // of how much we're waiting to get pushed to some underlying - // socket or file. - this.length = 0; + decodeStrings: makeBitMapDescriptor(kDecodeStrings), // A flag to see when we're in the middle of a write. - this.writing = false; - - // When true all writes will be buffered until .uncork() call. - this.corked = 0; + writing: makeBitMapDescriptor(kWriting), // A flag to be able to tell if the onwrite cb is called immediately, // or on a later tick. We set this to true at first, because any // actions that shouldn't happen until "later" should generally also // not happen before the first write call. - this.sync = true; + sync: makeBitMapDescriptor(kSync), // A flag to know if we're processing previously buffered items, which // may call the _write() callback in the same tick, so that we don't // end up in an overlapped onwrite situation. - this.bufferProcessing = false; - - // The callback that's passed to _write(chunk, cb). - this.onwrite = (err) => onwrite.call(undefined, stream, err); - - // The callback that the user supplies to write(chunk, encoding, cb). - this.writecb = null; - - // The amount that is being written when _write is called. - this.writelen = 0; - - // Storage for data passed to the afterWrite() callback in case of - // synchronous _write() completion. - this.afterWriteTickInfo = null; - resetBuffer(this); - - // Number of pending user-supplied write callbacks - // this must be 0 before 'finish' can be emitted. - this.pendingcb = 0; + bufferProcessing: makeBitMapDescriptor(kBufferProcessing), // Stream is still being constructed and cannot be // destroyed until construction finished or failed. // Async construction is opt in, therefore we start as // constructed. - this.constructed = true; + constructed: makeBitMapDescriptor(kConstructed), // Emit prefinish if the only thing we're waiting for is _write cbs // This is relevant for synchronous Transform streams. - this.prefinished = false; + prefinished: makeBitMapDescriptor(kPrefinished), // True if the error was already emitted and should not be thrown again. - this.errorEmitted = false; + errorEmitted: makeBitMapDescriptor(kErrorEmitted), // Should close be emitted on destroy. Defaults to true. - this.emitClose = !options || options.emitClose !== false; + emitClose: makeBitMapDescriptor(kEmitClose), // Should .destroy() be called after 'finish' (and potentially 'end'). - this.autoDestroy = !options || options.autoDestroy !== false; + autoDestroy: makeBitMapDescriptor(kAutoDestroy), + + // Indicates whether the stream has finished destroying. + closed: makeBitMapDescriptor(kClosed), + + // True if close has been emitted or would have been emitted + // depending on emitClose. + closeEmitted: makeBitMapDescriptor(kCloseEmitted), + + allBuffers: makeBitMapDescriptor(kAllBuffers), + allNoop: makeBitMapDescriptor(kAllNoop), // Indicates whether the stream has errored. When true all write() calls // should return false. This is needed since when autoDestroy // is disabled we need a way to tell whether the stream has failed. - this.errored = null; + // This is/should be a cold path. + errored: { + __proto__: null, + enumerable: false, + get() { + return (this[kState] & kErrored) !== 0 ? this[kErroredValue] : null; + }, + set(value) { + if (value) { + this[kErroredValue] = value; + this[kState] |= kErrored; + } else { + this[kState] &= ~kErrored; + } + }, + }, - // Indicates whether the stream has finished destroying. - this.closed = false; + writable: { + __proto__: null, + enumerable: false, + get() { + return (this[kState] & kHasWritable) !== 0 + ? (this[kState] & kWritable) !== 0 + : undefined; + }, + set(value) { + if (value == null) { + this[kState] &= ~(kHasWritable | kWritable); + } else if (value) { + this[kState] |= kHasWritable | kWritable; + } else { + this[kState] |= kHasWritable; + this[kState] &= ~kWritable; + } + }, + }, - // True if close has been emitted or would have been emitted - // depending on emitClose. - this.closeEmitted = false; - this[kOnFinished] = []; + defaultEncoding: { + __proto__: null, + enumerable: false, + get() { + return (this[kState] & kDefaultUTF8Encoding) !== 0 + ? 'utf8' + : this[kDefaultEncodingValue]; + }, + set(value) { + if (value === 'utf8' || value === 'utf-8') { + this[kState] |= kDefaultUTF8Encoding; + } else { + this[kState] &= ~kDefaultUTF8Encoding; + this[kDefaultEncodingValue] = value; + } + }, + }, + + // The callback that the user supplies to write(chunk, encoding, cb). + writecb: { + __proto__: null, + enumerable: false, + get() { + return (this[kState] & kWriteCb) !== 0 ? this[kWriteCbValue] : nop; + }, + set(value) { + this[kWriteCbValue] = value; + if (value) { + this[kState] |= kWriteCb; + } else { + this[kState] &= ~kWriteCb; + } + }, + }, + + // Storage for data passed to the afterWrite() callback in case of + // synchronous _write() completion. + afterWriteTickInfo: { + __proto__: null, + enumerable: false, + get() { + return (this[kState] & kAfterWriteTickInfo) !== 0 + ? this[kAfterWriteTickInfoValue] + : null; + }, + set(value) { + this[kAfterWriteTickInfoValue] = value; + if (value) { + this[kState] |= kAfterWriteTickInfo; + } else { + this[kState] &= ~kAfterWriteTickInfo; + } + }, + }, + + buffered: { + __proto__: null, + enumerable: false, + get() { + return (this[kState] & kBuffered) !== 0 ? this[kBufferedValue] : []; + }, + set(value) { + this[kBufferedValue] = value; + if (value) { + this[kState] |= kBuffered; + } else { + this[kState] &= ~kBuffered; + } + }, + }, +}); + +// ====================================================================================== +// WritableState + +export function WritableState(options, stream, isDuplex) { + // Bit map field to store WritableState more efficiently with 1 bit per field + // instead of a V8 slot per field. + this[kState] = kSync | kConstructed | kEmitClose | kAutoDestroy; + + if (options?.objectMode) this[kState] |= kObjectMode; + + if (isDuplex && options?.writableObjectMode) this[kState] |= kObjectMode; + + // The point at which write() starts returning false + // Note: 0 is a valid value, means that we always return false if + // the entire buffer is not flushed immediately on write(). + this.highWaterMark = options + ? getHighWaterMark(this, options, 'writableHighWaterMark', isDuplex) + : getDefaultHighWaterMark(false); + + if (!options || options.decodeStrings !== false) + this[kState] |= kDecodeStrings; + + // Should close be emitted on destroy. Defaults to true. + if (options && options.emitClose === false) this[kState] &= ~kEmitClose; + + // Should .destroy() be called after 'end' (and potentially 'finish'). + if (options && options.autoDestroy === false) this[kState] &= ~kAutoDestroy; + + // Crypto is kind of old and crusty. Historically, its default string + // encoding is 'binary' so we have to make this configurable. + // Everything else in the universe uses 'utf8', though. + const defaultEncoding = options ? options.defaultEncoding : null; + if ( + defaultEncoding == null || + defaultEncoding === 'utf8' || + defaultEncoding === 'utf-8' + ) { + this[kState] |= kDefaultUTF8Encoding; + } else if (Buffer.isEncoding(defaultEncoding)) { + this[kState] &= ~kDefaultUTF8Encoding; + this[kDefaultEncodingValue] = defaultEncoding; + } else if (streamsNodejsV24Compat) { + // This is a semver-major change. Ref: https://github.com/nodejs/node/pull/46322 + throw new ERR_UNKNOWN_ENCODING(defaultEncoding); + } else { + this[kDefaultEncodingValue] = defaultEncoding; + } + + // Not an actual buffer we keep track of, but a measurement + // of how much we're waiting to get pushed to some underlying + // socket or file. + this.length = 0; + + // When true all writes will be buffered until .uncork() call. + this.corked = 0; + + // The callback that's passed to _write(chunk, cb). + this.onwrite = onwrite.bind(undefined, stream); + + // The amount that is being written when _write is called. + this.writelen = 0; + + resetBuffer(this); + + // Number of pending user-supplied write callbacks + // this must be 0 before 'finish' can be emitted. + this.pendingcb = 0; } function resetBuffer(state) { - state.buffered = []; + state[kBufferedValue] = null; state.bufferedIndex = 0; - state.allBuffers = true; - state.allNoop = true; + state[kState] |= kAllBuffers | kAllNoop; + state[kState] &= ~kBuffered; } WritableState.prototype.getBuffer = function getBuffer() { - return this.buffered.slice(this.bufferedIndex); + return (this[kState] & kBuffered) === 0 + ? [] + : this.buffered.slice(this.bufferedIndex); }; Object.defineProperty(WritableState.prototype, 'bufferedRequestCount', { + __proto__: null, get() { - return this.buffered.length - this.bufferedIndex; + return (this[kState] & kBuffered) === 0 + ? 0 + : this[kBufferedValue].length - this.bufferedIndex; }, }); WritableState.prototype[kOnConstructed] = function onConstructed(stream) { - if (!this.writing) { + if ((this[kState] & kWriting) === 0) { clearBuffer(stream, this); } - if (!this.ending) { + if ((this[kState] & kEnding) !== 0) { finishMaybe(stream, this); } }; @@ -234,160 +418,222 @@ Object.setPrototypeOf(Writable, Stream); export function Writable(options) { if (!(this instanceof Writable)) return new Writable(options); + this._events ??= { + close: undefined, + error: undefined, + prefinish: undefined, + finish: undefined, + drain: undefined, + // Skip uncommon events... + // [destroyImpl.kConstruct]: undefined, + // [destroyImpl.kDestroy]: undefined, + }; + this._writableState = new WritableState(options, this, false); + if (options) { if (typeof options.write === 'function') this._write = options.write; + if (typeof options.writev === 'function') this._writev = options.writev; + if (typeof options.destroy === 'function') this._destroy = options.destroy; + if (typeof options.final === 'function') this._final = options.final; + if (typeof options.construct === 'function') this._construct = options.construct; + if (options.signal) addAbortSignal(options.signal, this); } + Stream.call(this, options); - construct(this, () => { - const state = this._writableState; - if (!state.writing) { - clearBuffer(this, state); - } - finishMaybe(this, state); - }); + + if (this._construct != null) { + destroyImpl.construct(this, () => { + this._writableState[kOnConstructed](this); + }); + } } Object.defineProperty(Writable, Symbol.hasInstance, { + __proto__: null, value: function (object) { if (Function.prototype[Symbol.hasInstance].call(this, object)) return true; if (this !== Writable) return false; - return object?._writableState instanceof WritableState; + + return object && object._writableState instanceof WritableState; }, }); // Otherwise people can pipe Writable streams, which is just wrong. -Writable.prototype.pipe = function (_1, _2) { - errorOrDestroy(this, new ERR_STREAM_CANNOT_PIPE()); +Writable.prototype.pipe = function () { + destroyImpl.errorOrDestroy(this, new ERR_STREAM_CANNOT_PIPE()); }; function _write(stream, chunk, encoding, cb) { const state = stream._writableState; - if (typeof encoding === 'function') { - cb = encoding; - encoding = state.defaultEncoding; - } else { - if (!encoding) encoding = state.defaultEncoding; - else if (encoding !== 'buffer' && !Buffer.isEncoding(encoding)) { - throw new ERR_UNKNOWN_ENCODING(encoding); - } - if (typeof cb !== 'function') cb = nop; + + if (cb == null || typeof cb !== 'function') { + cb = nop; } + if (chunk === null) { throw new ERR_STREAM_NULL_VALUES(); - } else if (!state.objectMode) { + } + + if ((state[kState] & kObjectMode) === 0) { + if (!encoding) { + encoding = + (state[kState] & kDefaultUTF8Encoding) !== 0 + ? 'utf8' + : state.defaultEncoding; + } else if (encoding !== 'buffer' && !Buffer.isEncoding(encoding)) { + throw new ERR_UNKNOWN_ENCODING(encoding); + } + if (typeof chunk === 'string') { - if (state.decodeStrings !== false) { + if ((state[kState] & kDecodeStrings) !== 0) { chunk = Buffer.from(chunk, encoding); encoding = 'buffer'; } } else if (chunk instanceof Buffer) { encoding = 'buffer'; - } else if (Stream._isUint8Array(chunk)) { + } else if (Stream._isArrayBufferView(chunk)) { chunk = Stream._uint8ArrayToBuffer(chunk); encoding = 'buffer'; } else { throw new ERR_INVALID_ARG_TYPE( 'chunk', - ['string', 'Buffer', 'Uint8Array'], + ['string', 'Buffer', 'TypedArray', 'DataView'], chunk ); } } + let err; - if (state.ending) { + if ((state[kState] & kEnding) !== 0) { err = new ERR_STREAM_WRITE_AFTER_END(); - } else if (state.destroyed) { + } else if ((state[kState] & kDestroyed) !== 0) { err = new ERR_STREAM_DESTROYED('write'); } + if (err) { nextTick(cb, err); - errorOrDestroy(stream, err, true); + destroyImpl.errorOrDestroy(stream, err, true); return err; } + state.pendingcb++; return writeOrBuffer(stream, state, chunk, encoding, cb); } -function write(chunk, encoding, cb) { - return _write(this, chunk, encoding, cb) === true; -} +Writable.prototype.write = function (chunk, encoding, cb) { + if (encoding != null && typeof encoding === 'function') { + cb = encoding; + encoding = null; + } -Writable.prototype.write = write; + return _write(this, chunk, encoding, cb) === true; +}; Writable.prototype.cork = function () { - this._writableState.corked++; + const state = this._writableState; + + state[kState] |= kCorked; + state.corked++; }; Writable.prototype.uncork = function () { const state = this._writableState; + if (state.corked) { state.corked--; - if (!state.writing) clearBuffer(this, state); + + if (!state.corked) { + state[kState] &= ~kCorked; + } + + if ((state[kState] & kWriting) === 0) clearBuffer(this, state); } }; -function setDefaultEncoding(encoding) { +Writable.prototype.setDefaultEncoding = function setDefaultEncoding(encoding) { + // node::ParseEncoding() requires lower case. if (typeof encoding === 'string') encoding = encoding.toLowerCase(); if (!Buffer.isEncoding(encoding)) throw new ERR_UNKNOWN_ENCODING(encoding); this._writableState.defaultEncoding = encoding; return this; -} - -Writable.prototype.setDefaultEncoding = setDefaultEncoding; +}; // If we're already writing something, then just put this // in the queue, and wait our turn. Otherwise, call _write // If we return false, then we need a drain event, so set that flag. function writeOrBuffer(stream, state, chunk, encoding, callback) { - const len = state.objectMode ? 1 : chunk.length; + const len = (state[kState] & kObjectMode) !== 0 ? 1 : chunk.length; + state.length += len; // stream._write resets state.length const ret = state.length < state.highWaterMark || state.length === 0; - // We must ensure that previous needDrain will not be reset to false. - if (!ret) state.needDrain = true; - if (state.writing || state.corked || state.errored || !state.constructed) { - state.buffered.push({ - chunk, - encoding, - callback, - }); - if (state.allBuffers && encoding !== 'buffer') { - state.allBuffers = false; + + // This is a semver-major change. Ref: https://github.com/nodejs/node/commit/557044af407376aff28a0a0800f3053bb58e9239 + if (!streamsNodejsV24Compat) { + // We must ensure that previous needDrain will not be reset to false. + if (!ret) { + state[kState] |= kNeedDrain; } - if (state.allNoop && callback !== nop) { - state.allNoop = false; + } + + if ( + (state[kState] & (kWriting | kErrored | kCorked | kConstructed)) !== + kConstructed + ) { + if ((state[kState] & kBuffered) === 0) { + state[kState] |= kBuffered; + state[kBufferedValue] = []; + } + + state[kBufferedValue].push({ chunk, encoding, callback }); + if ((state[kState] & kAllBuffers) !== 0 && encoding !== 'buffer') { + state[kState] &= ~kAllBuffers; + } + if ((state[kState] & kAllNoop) !== 0 && callback !== nop) { + state[kState] &= ~kAllNoop; } } else { state.writelen = len; - state.writecb = callback; - state.writing = true; - state.sync = true; + if (callback !== nop) { + state.writecb = callback; + } + state[kState] |= kWriting | kSync | kExpectWriteCb; stream._write(chunk, encoding, state.onwrite); - state.sync = false; + state[kState] &= ~kSync; + } + + // This is a semver-major change. Ref: https://github.com/nodejs/node/commit/557044af407376aff28a0a0800f3053bb58e9239 + if (streamsNodejsV24Compat) { + // We must ensure that previous needDrain will not be reset to false. + if (!ret) { + state[kState] |= kNeedDrain; + } } // Return false if errored or destroyed in order to break // any synchronous while(stream.write(data)) loops. - return ret && !state.errored && !state.destroyed; + return ret && (state[kState] & (kDestroyed | kErrored)) === 0; } function doWrite(stream, state, writev, len, chunk, encoding, cb) { state.writelen = len; - state.writecb = cb; - state.writing = true; - state.sync = true; - if (state.destroyed) state.onwrite(new ERR_STREAM_DESTROYED('write')); + if (cb !== nop) { + state.writecb = cb; + } + state[kState] |= kWriting | kSync | kExpectWriteCb; + if ((state[kState] & kDestroyed) !== 0) + state.onwrite(new ERR_STREAM_DESTROYED('write')); else if (writev) stream._writev(chunk, state.onwrite); else stream._write(chunk, encoding, state.onwrite); - state.sync = false; + state[kState] &= ~kSync; } function onwriteError(stream, state, er, cb) { @@ -399,27 +645,32 @@ function onwriteError(stream, state, er, cb) { // writes. errorBuffer(state); // This can emit error, but error must always follow cb. - errorOrDestroy(stream, er); + destroyImpl.errorOrDestroy(stream, er); } function onwrite(stream, er) { const state = stream._writableState; - const sync = state.sync; - const cb = state.writecb; - if (typeof cb !== 'function') { - errorOrDestroy(stream, new ERR_MULTIPLE_CALLBACK()); + + if ((state[kState] & kExpectWriteCb) === 0) { + destroyImpl.errorOrDestroy(stream, new ERR_MULTIPLE_CALLBACK()); return; } - state.writing = false; + + const sync = (state[kState] & kSync) !== 0; + const cb = (state[kState] & kWriteCb) !== 0 ? state[kWriteCbValue] : nop; + state.writecb = null; + state[kState] &= ~(kWriting | kExpectWriteCb); state.length -= state.writelen; state.writelen = 0; + if (er) { // Avoid V8 leak, https://github.com/nodejs/node/pull/34103#issuecomment-652002364 - er.stack; + er.stack; // eslint-disable-line @typescript-eslint/no-unused-expressions - if (!state.errored) { - state.errored = er; + if ((state[kState] & kErrored) === 0) { + state[kErroredValue] = er; + state[kState] |= kErrored; } // In case of duplex streams we need to notify the readable side of the @@ -427,33 +678,51 @@ function onwrite(stream, er) { if (stream._readableState && !stream._readableState.errored) { stream._readableState.errored = er; } + if (sync) { nextTick(onwriteError, stream, state, er, cb); } else { onwriteError(stream, state, er, cb); } } else { - if (state.buffered.length > state.bufferedIndex) { + if ((state[kState] & kBuffered) !== 0) { clearBuffer(stream, state); } + if (sync) { + const needDrain = + (state[kState] & kNeedDrain) !== 0 && state.length === 0; + const needTick = + needDrain || state[kState] & (kDestroyed !== 0) || cb !== nop; + // It is a common case that the callback passed to .write() is always // the same. In that case, we do not schedule a new nextTick(), but // rather just increase a counter, to improve performance and avoid // memory allocations. - if ( - state.afterWriteTickInfo !== null && - state.afterWriteTickInfo.cb === cb + if (cb === nop) { + if ((state[kState] & kAfterWritePending) === 0 && needTick) { + nextTick(afterWrite, stream, state, 1, cb); + state[kState] |= kAfterWritePending; + } else { + state.pendingcb--; + if ((state[kState] & kEnding) !== 0) { + finishMaybe(stream, state, true); + } + } + } else if ( + (state[kState] & kAfterWriteTickInfo) !== 0 && + state[kAfterWriteTickInfoValue].cb === cb ) { - state.afterWriteTickInfo.count++; + state[kAfterWriteTickInfoValue].count++; + } else if (needTick) { + state[kAfterWriteTickInfoValue] = { count: 1, cb, stream, state }; + nextTick(afterWriteTick, state[kAfterWriteTickInfoValue]); + state[kState] |= kAfterWritePending | kAfterWriteTickInfo; } else { - state.afterWriteTickInfo = { - count: 1, - cb, - stream, - state, - }; - nextTick(afterWriteTick, state.afterWriteTickInfo); + state.pendingcb--; + if ((state[kState] & kEnding) !== 0) { + finishMaybe(stream, state, true); + } } } else { afterWrite(stream, state, 1, cb); @@ -462,87 +731,103 @@ function onwrite(stream, er) { } function afterWriteTick({ stream, state, count, cb }) { - state.afterWriteTickInfo = null; + state[kState] &= ~kAfterWriteTickInfo; + state[kAfterWriteTickInfoValue] = null; return afterWrite(stream, state, count, cb); } function afterWrite(stream, state, count, cb) { + state[kState] &= ~kAfterWritePending; + const needDrain = - !state.ending && !stream.destroyed && state.length === 0 && state.needDrain; + (state[kState] & (kEnding | kNeedDrain | kDestroyed)) === kNeedDrain && + state.length === 0; if (needDrain) { - state.needDrain = false; + state[kState] &= ~kNeedDrain; stream.emit('drain'); } + + // This is a semver-major change. Ref: https://github.com/nodejs/node/pull/44312/files + const callbackValue = streamsNodejsV24Compat ? null : undefined; while (count-- > 0) { state.pendingcb--; - cb(); + cb(callbackValue); } - if (state.destroyed) { + + if ((state[kState] & kDestroyed) !== 0) { errorBuffer(state); } - finishMaybe(stream, state); + + if ((state[kState] & kEnding) !== 0) { + finishMaybe(stream, state, true); + } } // If there's something in the buffer waiting, then invoke callbacks. function errorBuffer(state) { - if (state.writing) { + if ((state[kState] & kWriting) !== 0) { return; } - for (let n = state.bufferedIndex; n < state.buffered.length; ++n) { - let _state$errored; - const { chunk, callback } = state.buffered[n]; - const len = state.objectMode ? 1 : chunk.length; - state.length -= len; - callback( - (_state$errored = state.errored) !== null && _state$errored !== undefined - ? _state$errored - : new ERR_STREAM_DESTROYED('write') - ); - } - const onfinishCallbacks = state[kOnFinished].splice(0); - for (let i = 0; i < onfinishCallbacks.length; i++) { - let _state$errored2; - onfinishCallbacks[i]( - (_state$errored2 = state.errored) !== null && - _state$errored2 !== undefined - ? _state$errored2 - : new ERR_STREAM_DESTROYED('end') - ); + + if ((state[kState] & kBuffered) !== 0) { + for (let n = state.bufferedIndex; n < state.buffered.length; ++n) { + const { chunk, callback } = state[kBufferedValue][n]; + const len = (state[kState] & kObjectMode) !== 0 ? 1 : chunk.length; + state.length -= len; + callback(state.errored ?? new ERR_STREAM_DESTROYED('write')); + } } + + callFinishedCallbacks( + state, + state.errored ?? new ERR_STREAM_DESTROYED('end') + ); + resetBuffer(state); } // If there's something in the buffer waiting, then process it. function clearBuffer(stream, state) { if ( - state.corked || - state.bufferProcessing || - state.destroyed || - !state.constructed + (state[kState] & + (kDestroyed | kBufferProcessing | kCorked | kBuffered | kConstructed)) !== + (kBuffered | kConstructed) ) { return; } - const { buffered, bufferedIndex, objectMode } = state; + + const objectMode = (state[kState] & kObjectMode) !== 0; + const { [kBufferedValue]: buffered, bufferedIndex } = state; const bufferedLength = buffered.length - bufferedIndex; + if (!bufferedLength) { return; } + let i = bufferedIndex; - state.bufferProcessing = true; + + state[kState] |= kBufferProcessing; if (bufferedLength > 1 && stream._writev) { state.pendingcb -= bufferedLength - 1; - const callback = state.allNoop - ? nop - : (err) => { - for (let n = i; n < buffered.length; ++n) { - buffered[n].callback(err); - } - }; + + const callback = + (state[kState] & kAllNoop) !== 0 + ? nop + : (err) => { + for (let n = i; n < buffered.length; ++n) { + buffered[n].callback(err); + } + }; // Make a copy of `buffered` if it's going to be used by `callback` above, // since `doWrite` will mutate the array. - const chunks = state.allNoop && i === 0 ? buffered : buffered.slice(i); - chunks.allBuffers = state.allBuffers; + const chunks = + (state[kState] & kAllNoop) !== 0 && i === 0 + ? buffered + : buffered.slice(i); + chunks.allBuffers = (state[kState] & kAllBuffers) !== 0; + doWrite(stream, state, true, state.length, chunks, '', callback); + resetBuffer(state); } else { do { @@ -550,7 +835,8 @@ function clearBuffer(stream, state) { buffered[i++] = null; const len = objectMode ? 1 : chunk.length; doWrite(stream, state, false, len, chunk, encoding, callback); - } while (i < buffered.length && !state.writing); + } while (i < buffered.length && (state[kState] & kWriting) === 0); + if (i === buffered.length) { resetBuffer(state); } else if (i > 256) { @@ -560,20 +846,12 @@ function clearBuffer(stream, state) { state.bufferedIndex = i; } } - state.bufferProcessing = false; + state[kState] &= ~kBufferProcessing; } Writable.prototype._write = function (chunk, encoding, cb) { if (this._writev) { - this._writev( - [ - { - chunk, - encoding, - }, - ], - cb - ); + this._writev([{ chunk, encoding }], cb); } else { throw new ERR_METHOD_NOT_IMPLEMENTED('_write()'); } @@ -581,8 +859,9 @@ Writable.prototype._write = function (chunk, encoding, cb) { Writable.prototype._writev = null; -function end(chunk, encoding, cb) { +Writable.prototype.end = function (chunk, encoding, cb) { const state = this._writableState; + if (typeof chunk === 'function') { cb = chunk; chunk = null; @@ -591,8 +870,10 @@ function end(chunk, encoding, cb) { cb = encoding; encoding = null; } + let err; - if (chunk !== null && chunk !== undefined) { + + if (chunk != null) { const ret = _write(this, chunk, encoding); if (ret instanceof Error) { err = ret; @@ -600,116 +881,137 @@ function end(chunk, encoding, cb) { } // .end() fully uncorks. - if (state.corked) { + if ((state[kState] & kCorked) !== 0) { state.corked = 1; this.uncork(); } + if (err) { // Do nothing... - } else if (!state.errored && !state.ending) { + } else if ((state[kState] & (kEnding | kErrored)) === 0) { // This is forgiving in terms of unnecessary calls to end() and can hide // logic errors. However, usually such errors are harmless and causing a // hard error can be disproportionately destructive. It is not always // trivial for the user to determine whether end() needs to be called // or not. - state.ending = true; + state[kState] |= kEnding; finishMaybe(this, state, true); - state.ended = true; - } else if (state.finished) { + state[kState] |= kEnded; + } else if ((state[kState] & kFinished) !== 0) { err = new ERR_STREAM_ALREADY_FINISHED('end'); - } else if (state.destroyed) { + } else if ((state[kState] & kDestroyed) !== 0) { err = new ERR_STREAM_DESTROYED('end'); } + if (typeof cb === 'function') { - if (err || state.finished) { - nextTick(cb, err); + // This is a semver-major change. Ref: https://github.com/nodejs/node/pull/44312 + if (streamsNodejsV24Compat) { + if (err) { + nextTick(cb, err); + } else if ((state[kState] & kErrored) !== 0) { + nextTick(cb, state[kErroredValue]); + } else if ((state[kState] & kFinished) !== 0) { + nextTick(cb, null); + } else { + state[kState] |= kOnFinished; + state[kOnFinishedValue] ??= []; + state[kOnFinishedValue].push(cb); + } } else { - state[kOnFinished].push(cb); + if (err || (state[kState] & kFinished) !== 0) { + nextTick(cb, err); + } else if ((state[kState] & kErrored) !== 0) { + nextTick(cb, state[kErroredValue]); + } else { + state[kState] |= kOnFinished; + state[kOnFinishedValue] ??= []; + state[kOnFinishedValue].push(cb); + } } } - return this; -} -Writable.prototype.end = end; + return this; +}; function needFinish(state) { return ( - state.ending && - !state.destroyed && - state.constructed && - state.length === 0 && - !state.errored && - state.buffered.length === 0 && - !state.finished && - !state.writing && - !state.errorEmitted && - !state.closeEmitted + // State is ended && constructed but not destroyed, finished, writing, errorEmitted or closedEmitted + (state[kState] & + (kEnding | + kDestroyed | + kConstructed | + kFinished | + kWriting | + kErrorEmitted | + kCloseEmitted | + kErrored | + kBuffered)) === + (kEnding | kConstructed) && state.length === 0 ); } - -function callFinal(stream, state) { - let called = false; - function onFinish(err) { - if (called) { - errorOrDestroy(stream, err || new ERR_MULTIPLE_CALLBACK()); - return; - } - called = true; - state.pendingcb--; - if (err) { - const onfinishCallbacks = state[kOnFinished].splice(0); - for (let i = 0; i < onfinishCallbacks.length; i++) { - onfinishCallbacks[i](err); - } - errorOrDestroy(stream, err, state.sync); - } else if (needFinish(state)) { - state.prefinished = true; - stream.emit('prefinish'); - // Backwards compat. Don't check state.sync here. - // Some streams assume 'finish' will be emitted - // asynchronously relative to _final callback. - state.pendingcb++; - nextTick(finish, stream, state); - } +function onFinish(stream, state, err) { + if ((state[kState] & kPrefinished) !== 0) { + destroyImpl.errorOrDestroy(stream, err ?? new ERR_MULTIPLE_CALLBACK()); + return; } - state.sync = true; - state.pendingcb++; - try { - stream._final(onFinish); - } catch (err) { - onFinish(err); + state.pendingcb--; + if (err) { + callFinishedCallbacks(state, err); + destroyImpl.errorOrDestroy(stream, err, (state[kState] & kSync) !== 0); + } else if (needFinish(state)) { + state[kState] |= kPrefinished; + stream.emit('prefinish'); + // Backwards compat. Don't check state.sync here. + // Some streams assume 'finish' will be emitted + // asynchronously relative to _final callback. + state.pendingcb++; + nextTick(finish, stream, state); } - state.sync = false; } function prefinish(stream, state) { - if (!state.prefinished && !state.finalCalled) { - if (typeof stream._final === 'function' && !state.destroyed) { - state.finalCalled = true; - callFinal(stream, state); - } else { - state.prefinished = true; - stream.emit('prefinish'); + if ((state[kState] & (kPrefinished | kFinalCalled)) !== 0) { + return; + } + + if ( + typeof stream._final === 'function' && + (state[kState] & kDestroyed) === 0 + ) { + state[kState] |= kFinalCalled | kSync; + state.pendingcb++; + + try { + stream._final((err) => onFinish(stream, state, err)); + } catch (err) { + onFinish(stream, state, err); } + + state[kState] &= ~kSync; + } else { + state[kState] |= kFinalCalled | kPrefinished; + stream.emit('prefinish'); } } -function finishMaybe(stream, state, sync = false) { +function finishMaybe(stream, state, sync) { if (needFinish(state)) { prefinish(stream, state); if (state.pendingcb === 0) { if (sync) { state.pendingcb++; - nextTick(() => { - ((stream, state) => { + nextTick( + (stream, state) => { if (needFinish(state)) { finish(stream, state); } else { state.pendingcb--; } - })(stream, state); - }); + }, + stream, + state + ); } else if (needFinish(state)) { state.pendingcb++; finish(stream, state); @@ -720,13 +1022,13 @@ function finishMaybe(stream, state, sync = false) { function finish(stream, state) { state.pendingcb--; - state.finished = true; - const onfinishCallbacks = state[kOnFinished].splice(0); - for (let i = 0; i < onfinishCallbacks.length; i++) { - onfinishCallbacks[i](); - } + state[kState] |= kFinished; + + callFinishedCallbacks(state, null); + stream.emit('finish'); - if (state.autoDestroy) { + + if ((state[kState] & kAutoDestroy) !== 0) { // In case of duplex streams we need a way to detect // if the readable side is ready for autoDestroy as well. const rState = stream._readableState; @@ -742,30 +1044,48 @@ function finish(stream, state) { } } +function callFinishedCallbacks(state, err) { + if ((state[kState] & kOnFinished) === 0) { + return; + } + + const onfinishCallbacks = state[kOnFinishedValue]; + // This is a semver-major change. Ref: https://github.com/nodejs/node/pull/44312 + state[kOnFinishedValue] = streamsNodejsV24Compat ? null : undefined; + state[kState] &= ~kOnFinished; + for (let i = 0; i < onfinishCallbacks.length; i++) { + onfinishCallbacks[i](err); + } +} + Object.defineProperties(Writable.prototype, { closed: { + __proto__: null, get() { - return !!this._writableState?.closed; + return this._writableState + ? (this._writableState[kState] & kClosed) !== 0 + : false; }, }, + destroyed: { + __proto__: null, get() { - return !!this._writableState?.destroyed; + return this._writableState + ? (this._writableState[kState] & kDestroyed) !== 0 + : false; }, set(value) { // Backward compatibility, the user is explicitly managing destroyed. - if (this._writableState) { - this._writableState.destroyed = value; - } - }, - }, - errored: { - enumerable: false, - get() { - return this._writableState?.errored || null; + if (!this._writableState) return; + + if (value) this._writableState[kState] |= kDestroyed; + else this._writableState[kState] &= ~kDestroyed; }, }, + writable: { + __proto__: null, get() { const w = this._writableState; // w.writable === false means that this is part of a Duplex stream @@ -775,10 +1095,7 @@ Object.defineProperties(Writable.prototype, { return ( !!w && w.writable !== false && - !w.destroyed && - !w.errored && - !w.ending && - !w.ended + (w[kState] & (kEnding | kEnded | kDestroyed | kErrored)) === 0 ); }, set(val) { @@ -788,79 +1105,116 @@ Object.defineProperties(Writable.prototype, { } }, }, + writableFinished: { + __proto__: null, get() { - return !!this._writableState?.finished; + const state = this._writableState; + return state ? (state[kState] & kFinished) !== 0 : false; }, }, + writableObjectMode: { + __proto__: null, get() { - return !!this._writableState?.objectMode; + const state = this._writableState; + return state ? (state[kState] & kObjectMode) !== 0 : false; }, }, + writableBuffer: { + __proto__: null, get() { - return this._writableState?.getBuffer(); + const state = this._writableState; + return state && state.getBuffer(); }, }, + writableEnded: { + __proto__: null, get() { - return !!this._writableState?.ending; + const state = this._writableState; + return state ? (state[kState] & kEnding) !== 0 : false; }, }, + writableNeedDrain: { + __proto__: null, get() { - const wState = this._writableState; - if (!wState) return false; - return !wState.destroyed && !wState.ending && wState.needDrain; + const state = this._writableState; + return state + ? (state[kState] & (kDestroyed | kEnding | kNeedDrain)) === kNeedDrain + : false; }, }, + writableHighWaterMark: { + __proto__: null, get() { - return this._writableState?.highWaterMark; + const state = this._writableState; + return state?.highWaterMark; }, }, + writableCorked: { + __proto__: null, get() { - return this._writableState?.corked | 0; + const state = this._writableState; + return state ? state.corked : 0; }, }, + writableLength: { + __proto__: null, get() { - return this._writableState?.length; + const state = this._writableState; + return state?.length; }, }, - writableAborted: { + + errored: { + __proto__: null, enumerable: false, get() { - return !!( - this._writableState.writable !== false && - (this._writableState.destroyed || this._writableState.errored) && - !this._writableState.finished + const state = this._writableState; + return state ? state.errored : null; + }, + }, + + writableAborted: { + __proto__: null, + get: function () { + const state = this._writableState; + return ( + (state[kState] & (kHasWritable | kWritable)) !== kHasWritable && + (state[kState] & (kDestroyed | kErrored)) !== 0 && + (state[kState] & kFinished) === 0 ); }, }, }); +const destroy = destroyImpl.destroy; Writable.prototype.destroy = function (err, cb) { const state = this._writableState; // Invoke pending callbacks. if ( - !state.destroyed && - (state.bufferedIndex < state.buffered.length || state[kOnFinished].length) + (state[kState] & (kBuffered | kOnFinished)) !== 0 && + (state[kState] & kDestroyed) === 0 ) { nextTick(errorBuffer, state); } + destroy.call(this, err, cb); return this; }; -Writable.prototype._undestroy = undestroy; - +Writable.prototype._undestroy = destroyImpl.undestroy; Writable.prototype._destroy = function (err, cb) { - if (cb) cb(err); + cb(err); }; + Writable.prototype[EventEmitter.captureRejectionSymbol] = function (err) { this.destroy(err); }; @@ -876,6 +1230,19 @@ export function toWeb(streamWritable) { Writable.fromWeb = fromWeb; Writable.toWeb = toWeb; +Writable.prototype[Symbol.asyncDispose] = async function () { + let error; + if (!this.destroyed) { + error = this.writableFinished ? null : new AbortError(); + this.destroy(error); + } + await new Promise((resolve, reject) => + eos(this, (err) => + err && err.name !== 'AbortError' ? reject(err) : resolve(null) + ) + ); +}; + /** * @param {Writable} streamWritable * @returns {WritableStream} @@ -899,14 +1266,14 @@ export function newWritableStreamFromStreamWritable(streamWritable) { } if (isDestroyed(streamWritable) || !isWritable(streamWritable)) { - const writable = new WritableStream(); + const writable = new globalThis.WritableStream(); writable.close(); return writable; } const highWaterMark = streamWritable.writableHighWaterMark; const strategy = streamWritable.writableObjectMode - ? new CountQueuingStrategy({ highWaterMark }) + ? new globalThis.CountQueuingStrategy({ highWaterMark }) : { highWaterMark }; let controller; @@ -917,11 +1284,8 @@ export function newWritableStreamFromStreamWritable(streamWritable) { if (backpressurePromise !== undefined) backpressurePromise.resolve(); } - const cleanup = eos(streamWritable, (error) => { - if (error?.code === 'ERR_STREAM_PREMATURE_CLOSE') { - const err = new AbortError(undefined, { cause: error }); - error = err; - } + const cleanup = finished(streamWritable, (error) => { + error = handleKnownInternalErrors(error); cleanup(); // This is a protection against non-standard, legacy streams @@ -952,13 +1316,13 @@ export function newWritableStreamFromStreamWritable(streamWritable) { streamWritable.on('drain', onDrain); - return new WritableStream( + return new globalThis.WritableStream( { start(c) { controller = c; }, - async write(chunk) { + write(chunk) { if (streamWritable.writableNeedDrain || !streamWritable.write(chunk)) { backpressurePromise = Promise.withResolvers(); return backpressurePromise.promise.finally(() => { @@ -968,7 +1332,7 @@ export function newWritableStreamFromStreamWritable(streamWritable) { }, abort(reason) { - destroy.call(streamWritable, reason); + destroy(streamWritable, reason); }, close() { @@ -1000,7 +1364,7 @@ export function newStreamWritableFromWritableStream( writableStream, options = {} ) { - if (!(writableStream instanceof WritableStream)) { + if (!isWritableStream(writableStream)) { throw new ERR_INVALID_ARG_TYPE( 'writableStream', 'WritableStream', @@ -1039,7 +1403,7 @@ export function newStreamWritableFromWritableStream( // thrown we don't want those to cause an unhandled // rejection. Let's just escape the promise and // handle it separately. - nextTick(() => destroy.call(writable, error)); + nextTick(() => destroy(writable, error)); } } @@ -1071,7 +1435,7 @@ export function newStreamWritableFromWritableStream( try { callback(error); } catch (error) { - destroy.call(writable, error); + destroy(writable, error); } } @@ -1118,7 +1482,7 @@ export function newStreamWritableFromWritableStream( // thrown we don't want those to cause an unhandled // rejection. Let's just escape the promise and // handle it separately. - nextTick(() => destroy.call(writable, error)); + nextTick(() => destroy(writable, error)); } } @@ -1134,13 +1498,13 @@ export function newStreamWritableFromWritableStream( // ended, we signal an error on the stream.Writable. closed = true; if (!isWritableEnded(writable)) - destroy.call(writable, new ERR_STREAM_PREMATURE_CLOSE()); + destroy(writable, new ERR_STREAM_PREMATURE_CLOSE()); }, (error) => { // If the WritableStream errors before the stream.Writable has been // destroyed, signal an error on the stream.Writable. closed = true; - destroy.call(writable, error); + destroy(writable, error); } ); diff --git a/src/workerd/api/node/tests/streams-test.js b/src/workerd/api/node/tests/streams-test.js index 0b25d4ed64d..062540b6256 100644 --- a/src/workerd/api/node/tests/streams-test.js +++ b/src/workerd/api/node/tests/streams-test.js @@ -1,16 +1,7 @@ import { - deepEqual, deepStrictEqual, - doesNotMatch, - doesNotReject, - doesNotThrow, - equal, fail, ifError, - match, - notDeepEqual, - notDeepStrictEqual, - notEqual, notStrictEqual, ok, rejects, @@ -6075,7 +6066,7 @@ export const writable_end_cb_error = { const finishCalled = Promise.withResolvers(); writable.end('asd', (err) => { called = true; - strictEqual(err, undefined); + strictEqual(err, null); endCalled.resolve(); }); writable.on('error', (err) => { @@ -7203,7 +7194,7 @@ export const uint8array = { ok(!(chunk instanceof Buffer)); ok(chunk instanceof Uint8Array); strictEqual(chunk, ABC); - strictEqual(encoding, 'utf8'); + strictEqual(encoding, undefined); cb(); writeCalled.resolve(); }, diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp index 943fa0c3cc9..11d7028038a 100644 --- a/src/workerd/io/compatibility-date.capnp +++ b/src/workerd/io/compatibility-date.capnp @@ -1208,4 +1208,10 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { $compatDisableFlag("cache_reload_disabled") $experimental; # Enables the use of cache: reload in the fetch api. + + streamsNodejsV24Compat @143 :Bool + $compatEnableFlag("enable_streams_nodejs_v24_compat") + $compatDisableFlag("disable_streams_nodejs_v24_compat") + $impliedByAfterDate(name = "nodeJsCompat", date = "2025-11-15"); + # Enables breaking changes to Node.js streams done with the release of Node.js v24. }