diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b632ab9..fd85cc8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,10 +16,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [16.x, 18.x, 20.x] + node-version: [18.x, 20.x, 22.x] steps: - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - run: npm ci @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version: 20 - run: npm ci diff --git a/.gitignore b/.gitignore index c055e7f..f6b4d9a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ dist/ # ignore codec builds codecs/**/build/ +# pixi environments +.pixi +*.egg-info diff --git a/codecs/zstd/build.sh b/codecs/zstd/build.sh index cd8f557..f259cd1 100755 --- a/codecs/zstd/build.sh +++ b/codecs/zstd/build.sh @@ -49,7 +49,7 @@ cd ../../../ ${OPTIMIZE} \ -I "$CODEC_DIR/lib" \ --closure 1 \ - --bind \ + -fwasm-exceptions \ -s ALLOW_MEMORY_GROWTH=1 \ -s MODULARIZE=1 \ -s EXPORT_ES6=1 \ @@ -57,8 +57,10 @@ cd ../../../ -s ENVIRONMENT="webview" \ -s MALLOC=emmalloc \ -s EXPORT_NAME="zstd_codec" \ + -s EXPORT_EXCEPTION_HANDLING_HELPERS=1 \ -x c++ \ --std=c++17 \ + -lembind \ -lzstd \ -L "$BUILD_DIR/lib" \ -o "zstd_codec.js" diff --git a/codecs/zstd/zstd_codec.cpp b/codecs/zstd/zstd_codec.cpp index 777724c..fe9dc32 100644 --- a/codecs/zstd/zstd_codec.cpp +++ b/codecs/zstd/zstd_codec.cpp @@ -20,16 +20,114 @@ val compress(std::string source, int level) val decompress(std::string source) { + // number of bytes to grow the output buffer if more space is needed + const size_t DEST_GROWTH_SIZE = ZSTD_DStreamOutSize(); + // setup source buffer const char *source_ptr = source.c_str(); int source_size = source.size(); + // create and initialize decompression stream / context + // use the streaming API so that we can handle unkown frame content size + ZSTD_DStream *zds = ZSTD_createDStream(); + + size_t status = ZSTD_initDStream(zds); + if (ZSTD_isError(status)) { + ZSTD_freeDStream(zds); + throw std::runtime_error("zstd codec error: " + std::string(ZSTD_getErrorName(status))); + } + + ZSTD_inBuffer input = { + .src = (void*) source.c_str(), + .size = (size_t) source.size(), + .pos = 0 + }; + ZSTD_outBuffer output = { + .dst = NULL, + .size = 0, + .pos = 0, + }; + // setup destination buffer - int dest_size = ZSTD_getFrameContentSize(source_ptr, source_size); - dest_ptr = (char *)malloc((size_t)dest_size); + unsigned long long dest_size = ZSTD_getFrameContentSize(source_ptr, source_size); + + // If Zstd_compressStream was used, we may not know the frame content size. + // https://github.com/manzt/numcodecs.js/issues/46 + if (dest_size == ZSTD_CONTENTSIZE_UNKNOWN) { + // guess decompressed buffer size based on source size + dest_size = source_size*2; + + // Initialize the destination size to DEST_GROWTH_SIZE (default: 128 KiB) at minimum + if (dest_size < DEST_GROWTH_SIZE) + dest_size = DEST_GROWTH_SIZE; + + } else if (dest_size == ZSTD_CONTENTSIZE_ERROR) { + ZSTD_freeDStream(zds); + throw std::runtime_error("zstd codec error: content size error"); + } else if (dest_size < 0) { + // unknown error + ZSTD_freeDStream(zds); + throw std::runtime_error("zstd codec error: unknown ZSTD_getFrameContentSize error"); + } + + // the output buffer will either be assigned to dest_ptr to be freed by free_result, or freed on error + output.dst = malloc((size_t) dest_size); + + if (output.dst == NULL) { + // error, cannot allocate memory + ZSTD_freeDStream(zds); + throw std::runtime_error("zstd codec error: cannot allocate output buffer"); + } + + output.size = dest_size; + + // Call ZSTD_decompressStream repeatedly until status == 0 or error (status < 0) + do { + status = ZSTD_decompressStream(zds, &output, &input); + + if (ZSTD_isError(status)) { + if (dest_ptr == output.dst) + dest_ptr = (char *) NULL; + ZSTD_freeDStream(zds); + free(output.dst); + throw std::runtime_error("zstd codec error: " + std::string(ZSTD_getErrorName(status))); + } + + if (status > 0 && output.pos == output.size ) { + // attempt to expand output buffer in DEST_GROWTH_SIZE increments + size_t new_size = output.size + DEST_GROWTH_SIZE; + + if (new_size < output.size || new_size < DEST_GROWTH_SIZE) { + // overflow error + ZSTD_freeDStream(zds); + free(output.dst); + throw std::runtime_error("zstd codec error: output buffer overflow"); + } + + // Increase output buffer size + void *new_dst = realloc(output.dst, new_size); + + if (new_dst == NULL) { + // free the original pointer if realloc fails. + ZSTD_freeDStream(zds); + free(output.dst); + throw std::runtime_error("zstd codec error: could not expand output buffer"); + } + // the old output.dst is freed by realloc is it succeeds + output.dst = new_dst; + + output.size = new_size; + } + + // status > 0 indicates there are additional bytes to process in this frame + // status == 0 and input.pos < input.size suggests there may be an additional frame + } while (status > 0 || input.pos < input.size); + + ZSTD_freeDStream(zds); + + dest_ptr = (char *) output.dst; - int decompressed_size = ZSTD_decompress(dest_ptr, dest_size, source_ptr, source_size); - return val(typed_memory_view(decompressed_size, (uint8_t *)dest_ptr)); + return val(typed_memory_view(output.pos, (uint8_t *)dest_ptr)); } void free_result() diff --git a/codecs/zstd/zstd_codec.d.ts b/codecs/zstd/zstd_codec.d.ts index 9cf0076..8a6b9e1 100644 --- a/codecs/zstd/zstd_codec.d.ts +++ b/codecs/zstd/zstd_codec.d.ts @@ -2,6 +2,7 @@ export interface ZstdModule extends EmscriptenModule { compress(data: BufferSource, level: number): Uint8Array; decompress(data: BufferSource): Uint8Array; free_result(): void; + getExceptionMessage(err: WebAssembly.Exception): [string, string]; } declare const moduleFactory: EmscriptenModuleFactory; diff --git a/codecs/zstd/zstd_codec.js b/codecs/zstd/zstd_codec.js index b2d86c8..d0d549b 100644 --- a/codecs/zstd/zstd_codec.js +++ b/codecs/zstd/zstd_codec.js @@ -1,49 +1,46 @@ var zstd_codec = (() => { - var _scriptDir = typeof document !== 'undefined' && document.currentScript ? document.currentScript.src : undefined; + var _scriptName = typeof document != 'undefined' ? document.currentScript?.src : undefined; return ( function(moduleArg = {}) { + var moduleRtn; -var f=moduleArg,aa,t;f.ready=new Promise((a,b)=>{aa=a;t=b});var ba=Object.assign({},f),u=f.printErr||console.error.bind(console);Object.assign(f,ba);ba=null;var v;f.wasmBinary&&(v=f.wasmBinary);"object"!=typeof WebAssembly&&x("no native wasm support detected");var z,da=!1,C,D,E,F,G,H,ea,fa; -function ha(){var a=z.buffer;f.HEAP8=C=new Int8Array(a);f.HEAP16=E=new Int16Array(a);f.HEAPU8=D=new Uint8Array(a);f.HEAPU16=F=new Uint16Array(a);f.HEAP32=G=new Int32Array(a);f.HEAPU32=H=new Uint32Array(a);f.HEAPF32=ea=new Float32Array(a);f.HEAPF64=fa=new Float64Array(a)}var ia=[],ja=[],ka=[];function la(){var a=f.preRun.shift();ia.unshift(a)}var I=0,J=null,L=null; -function x(a){f.onAbort?.(a);a="Aborted("+a+")";u(a);da=!0;a=new WebAssembly.RuntimeError(a+". Build with -sASSERTIONS for more info.");t(a);throw a;}var ma=a=>a.startsWith("data:application/octet-stream;base64,"),na=a=>a.startsWith("file://"),M;M="zstd_codec.wasm";if(!ma(M)){var oa=M;M=f.locateFile?f.locateFile(oa,""):""+oa}function pa(a){return Promise.resolve().then(()=>{if(a==M&&v)var b=new Uint8Array(v);else throw"both async and sync fetching of the wasm failed";return b})} -function qa(a,b,c){return pa(a).then(d=>WebAssembly.instantiate(d,b)).then(d=>d).then(c,d=>{u(`failed to asynchronously prepare wasm: ${d}`);x(d)})}function ra(a,b){var c=M;return v||"function"!=typeof WebAssembly.instantiateStreaming||ma(c)||na(c)||"function"!=typeof fetch?qa(c,a,b):fetch(c,{credentials:"same-origin"}).then(d=>WebAssembly.instantiateStreaming(d,a).then(b,function(e){u(`wasm streaming compile failed: ${e}`);u("falling back to ArrayBuffer instantiation");return qa(c,a,b)}))} -var N=a=>{for(;0>2]=b};this.J=function(b){H[this.D+8>>2]=b};this.F=function(b,c){this.G();this.K(b);this.J(c)};this.G=function(){H[this.D+16>>2]=0}} -var ta=0,ua=0,va,O=a=>{for(var b="";D[a];)b+=va[D[a++]];return b},P={},Q={},R={},S,wa=a=>{throw new S(a);},T,xa=(a,b)=>{function c(l){l=b(l);if(l.length!==d.length)throw new T("Mismatched type converter count");for(var g=0;g{Q.hasOwnProperty(l)?e[g]=Q[l]:(h.push(l),P.hasOwnProperty(l)||(P[l]=[]),P[l].push(()=>{e[g]=Q[l];++k;k===h.length&&c(e)}))});0===h.length&&c(e)}; -function ya(a,b,c={}){var d=b.name;if(!a)throw new S(`type "${d}" must have a positive integer typeid pointer`);if(Q.hasOwnProperty(a)){if(c.M)return;throw new S(`Cannot register type '${d}' twice`);}Q[a]=b;delete R[a];P.hasOwnProperty(a)&&(b=P[a],delete P[a],b.forEach(e=>e()))}function U(a,b,c={}){if(!("argPackAdvance"in b))throw new TypeError("registerType registeredInstance requires argPackAdvance");ya(a,b,c)}function za(){this.B=[void 0];this.H=[]} -var V=new za,Aa=a=>{a>=V.D&&0===--V.get(a).I&&V.G(a)},Ba=a=>{switch(a){case void 0:return 1;case null:return 2;case !0:return 3;case !1:return 4;default:return V.F({I:1,value:a})}};function Ca(a){return this.fromWireType(G[a>>2])} -var Da=(a,b)=>{switch(b){case 4:return function(c){return this.fromWireType(ea[c>>2])};case 8:return function(c){return this.fromWireType(fa[c>>3])};default:throw new TypeError(`invalid float width (${b}): ${a}`);}},Ga=(a,b)=>Object.defineProperty(b,"name",{value:a}),Ha=a=>{for(;a.length;){var b=a.pop();a.pop()(b)}};function Ia(a){for(var b=1;b{if(void 0===f[a].A){var c=f[a];f[a]=function(){if(!f[a].A.hasOwnProperty(arguments.length))throw new S(`Function '${b}' called with an invalid number of arguments (${arguments.length}) - expects one of (${f[a].A})!`);return f[a].A[arguments.length].apply(this,arguments)};f[a].A=[];f[a].A[c.L]=c}},La=(a,b,c)=>{if(f.hasOwnProperty(a)){if(void 0===c||void 0!==f[a].A&&void 0!==f[a].A[c])throw new S(`Cannot register public name '${a}' twice`);Ka(a,a);if(f.hasOwnProperty(c))throw new S(`Cannot register multiple overloads of a function with the same number of arguments (${c})!`); -f[a].A[c]=b}else f[a]=b,void 0!==c&&(f[a].O=c)},Ma=(a,b)=>{for(var c=[],d=0;d>2]);return c},Na,Oa=(a,b)=>{var c=[];return function(){c.length=0;Object.assign(c,arguments);if(a.includes("j")){var d=f["dynCall_"+a];d=c&&c.length?d.apply(null,[b].concat(c)):d.call(null,b)}else d=Na.get(b).apply(null,c);return d}},Pa=(a,b)=>{a=O(a);var c=a.includes("j")?Oa(a,b):Na.get(b);if("function"!=typeof c)throw new S(`unknown function pointer with signature ${a}: ${b}`);return c},Qa,Sa=a=> -{a=Ra(a);var b=O(a);W(a);return b},Ta=(a,b)=>{function c(h){e[h]||Q[h]||(R[h]?R[h].forEach(c):(d.push(h),e[h]=!0))}var d=[],e={};b.forEach(c);throw new Qa(`${a}: `+d.map(Sa).join([", "]));},Ua=a=>{a=a.trim();const b=a.indexOf("(");return-1!==b?a.substr(0,b):a},Va=(a,b,c)=>{switch(b){case 1:return c?d=>C[d>>0]:d=>D[d>>0];case 2:return c?d=>E[d>>1]:d=>F[d>>1];case 4:return c?d=>G[d>>2]:d=>H[d>>2];default:throw new TypeError(`invalid integer width (${b}): ${a}`);}}; -function Wa(a){return this.fromWireType(H[a>>2])} -for(var Xa="undefined"!=typeof TextDecoder?new TextDecoder("utf8"):void 0,Ya="undefined"!=typeof TextDecoder?new TextDecoder("utf-16le"):void 0,Za=(a,b)=>{var c=a>>1;for(var d=c+b/2;!(c>=d)&&F[c];)++c;c<<=1;if(32=b/2);++d){var e=E[a+2*d>>1];if(0==e)break;c+=String.fromCharCode(e)}return c},$a=(a,b,c)=>{c??=2147483647;if(2>c)return 0;c-=2;var d=b;c=c<2*a.length?c/2:a.length;for(var e=0;e>1]=a.charCodeAt(e),b+=2;E[b>>1]=0;return b- -d},ab=a=>2*a.length,bb=(a,b)=>{for(var c=0,d="";!(c>=b/4);){var e=G[a+4*c>>2];if(0==e)break;++c;65536<=e?(e-=65536,d+=String.fromCharCode(55296|e>>10,56320|e&1023)):d+=String.fromCharCode(e)}return d},cb=(a,b,c)=>{c??=2147483647;if(4>c)return 0;var d=b;c=d+c-4;for(var e=0;e=h){var k=a.charCodeAt(++e);h=65536+((h&1023)<<10)|k&1023}G[b>>2]=h;b+=4;if(b+4>c)break}G[b>>2]=0;return b-d},db=a=>{for(var b=0,c=0;c=d&&++c;b+=4}return b},eb=Array(256),X=0;256>X;++X)eb[X]=String.fromCharCode(X);va=eb;S=f.BindingError=class extends Error{constructor(a){super(a);this.name="BindingError"}};T=f.InternalError=class extends Error{constructor(a){super(a);this.name="InternalError"}};Object.assign(za.prototype,{get(a){return this.B[a]},has(a){return void 0!==this.B[a]},F(a){var b=this.H.pop()||this.B.length;this.B[b]=a;return b},G(a){this.B[a]=void 0;this.H.push(a)}}); -V.B.push({value:void 0},{value:null},{value:!0},{value:!1});V.D=V.B.length;f.count_emval_handles=()=>{for(var a=0,b=V.D;b{var c=Ga(b,function(d){this.name=b;this.message=d;d=Error(d).stack;void 0!==d&&(this.stack=this.toString()+"\n"+d.replace(/^Error(:[^\n]*)?\n/,""))});c.prototype=Object.create(a.prototype);c.prototype.constructor=c;c.prototype.toString=function(){return void 0===this.message?this.name:`${this.name}: ${this.message}`};return c})(Error,"UnboundTypeError"); -var gb={n:(a,b,c)=>{(new sa(a)).F(b,c);ta=a;ua++;throw ta;},o:()=>{},l:(a,b,c,d)=>{b=O(b);U(a,{name:b,fromWireType:function(e){return!!e},toWireType:function(e,h){return h?c:d},argPackAdvance:8,readValueFromPointer:function(e){return this.fromWireType(D[e])},C:null})},k:(a,b)=>{b=O(b);U(a,{name:b,fromWireType:c=>{if(!c)throw new S("Cannot use deleted val. handle = "+c);var d=V.get(c).value;Aa(c);return d},toWireType:(c,d)=>Ba(d),argPackAdvance:8,readValueFromPointer:Ca,C:null})},i:(a,b,c)=>{b=O(b); -U(a,{name:b,fromWireType:d=>d,toWireType:(d,e)=>e,argPackAdvance:8,readValueFromPointer:Da(b,c),C:null})},d:(a,b,c,d,e,h,k)=>{var l=Ma(b,c);a=O(a);a=Ua(a);e=Pa(d,e);La(a,function(){Ta(`Cannot call ${a} due to unbound types`,l)},b-1);xa(l,function(g){var n=a;var p=a;g=[g[0],null].concat(g.slice(1));var q=e,m=g.length;if(2>m)throw new S("argTypes array size mismatch! Must at least get return value and 'this' types!");var r=null!==g[1]&&!1,y=Ia(g),A="void"!==g[0].name;q=[wa,q,h,Ha,g[0],g[1]];for(var w= -0;w{b=O(b);-1===e&&(e=4294967295);e=l=>l;if(0===d){var h=32-8*c;e=l=>l<>>h}var k=b.includes("unsigned")?function(l,g){return g>>>0}:function(l,g){return g};U(a,{name:b,fromWireType:e,toWireType:k,argPackAdvance:8, -readValueFromPointer:Va(b,c,0!==d),C:null})},a:(a,b,c)=>{function d(h){return new e(C.buffer,H[h+4>>2],H[h>>2])}var e=[Int8Array,Uint8Array,Int16Array,Uint16Array,Int32Array,Uint32Array,Float32Array,Float64Array][b];c=O(c);U(a,{name:c,fromWireType:d,argPackAdvance:8,readValueFromPointer:d},{M:!0})},e:(a,b)=>{b=O(b);var c="std::string"===b;U(a,{name:b,fromWireType:function(d){var e=H[d>>2],h=d+4;if(c)for(var k=h,l=0;l<=e;++l){var g=h+l;if(l==e||0==D[g]){if(k){var n=k;var p=D,q=n+(g-k);for(k=n;p[k]&& -!(k>=q);)++k;if(16m?q+=String.fromCharCode(m):(m-=65536,q+=String.fromCharCode(55296|m>>10,56320|m&1023))}}else q+=String.fromCharCode(m)}n=q}}else n="";if(void 0===A)var A=n;else A+=String.fromCharCode(0),A+=n;k=g+1}}else{A=Array(e);for(l=0;l=g?l++:2047>=g?l+=2:55296<=g&&57343>=g?(l+=4,++h):l+=3}else l=e.length;h=l;l=fb(4+h+1);g=l+4;H[l>>2]=h;if(c&&k){if(k=g,g=h+1,h=D,0=p){var q=e.charCodeAt(++n);p=65536+((p&1023)<<10)|q&1023}if(127>=p){if(k>=g)break;h[k++]=p}else{if(2047>=p){if(k+1>=g)break;h[k++]=192|p>>6}else{if(65535>=p){if(k+2>=g)break;h[k++]=224|p>>12}else{if(k+3>=g)break;h[k++]=240|p>>18;h[k++]=128|p>>12&63}h[k++]=128|p>>6&63}h[k++]=128|p&63}}h[k]=0}}else if(k)for(k=0;k{c=O(c);if(2===b){var d=Za;var e=$a;var h=ab;var k=()=>F;var l=1}else 4===b&&(d=bb,e=cb,h=db,k=()=>H,l=2);U(a,{name:c,fromWireType:g=>{for(var n=H[g>>2],p=k(),q,m=g+4,r=0;r<=n;++r){var y=g+4+r*b;if(r==n||0==p[y>>l])m=d(m,y-m),void 0===q?q=m:(q+=String.fromCharCode(0),q+=m),m=y+b}W(g);return q},toWireType:(g,n)=>{if("string"!=typeof n)throw new S(`Cannot pass non-string to C++ string type ${c}`); -var p=h(n),q=fb(4+p+b);H[q>>2]=p>>l;e(n,q+4,p+b);null!==g&&g.push(W,q);return q},argPackAdvance:8,readValueFromPointer:Ca,C(g){W(g)}})},m:(a,b)=>{b=O(b);U(a,{N:!0,name:b,argPackAdvance:0,fromWireType:()=>{},toWireType:()=>{}})},g:Aa,j:a=>{4{var c=Q[a];if(void 0===c)throw a="_emval_take_value has unknown type "+Sa(a),new S(a);a=c;a=a.readValueFromPointer(b);return Ba(a)},h:()=>{x("")},q:(a,b,c)=>D.copyWithin(a,b,b+c),p:a=>{var b=D.length;a>>>=0;if(2147483648=c;c*=2){var d=b*(1+.2/c);d=Math.min(d,a+100663296);var e=Math;d=Math.max(a,d);a:{e=(e.min.call(e,2147483648,d+(65536-d%65536)%65536)-z.buffer.byteLength+65535)/65536;try{z.grow(e);ha();var h=1;break a}catch(k){}h=void 0}if(h)return!0}return!1}},Y=function(){function a(c){Y=c.exports;z=Y.r;ha();Na=Y.w;ja.unshift(Y.s);I--;f.monitorRunDependencies?.(I);0==I&&(null!==J&&(clearInterval(J),J=null),L&&(c=L,L=null,c()));return Y}var b={a:gb};I++;f.monitorRunDependencies?.(I);if(f.instantiateWasm)try{return f.instantiateWasm(b, -a)}catch(c){u(`Module.instantiateWasm callback failed with error: ${c}`),t(c)}ra(b,function(c){a(c.instance)}).catch(t);return{}}(),fb=a=>(fb=Y.t)(a),W=a=>(W=Y.u)(a),Ra=a=>(Ra=Y.v)(a),Z;L=function ib(){Z||jb();Z||(L=ib)}; -function jb(){function a(){if(!Z&&(Z=!0,f.calledRun=!0,!da)){N(ja);aa(f);if(f.onRuntimeInitialized)f.onRuntimeInitialized();if(f.postRun)for("function"==typeof f.postRun&&(f.postRun=[f.postRun]);f.postRun.length;){var b=f.postRun.shift();ka.unshift(b)}N(ka)}}if(!(0{aa=a;q=b}),ca=Object.assign({},f),r=f.printErr||console.error.bind(console);Object.assign(f,ca);ca=null;var t;f.wasmBinary&&(t=f.wasmBinary);var u,da=!1,x,z,A,B,D,E,ea,ha;function ia(){var a=u.buffer;f.HEAP8=x=new Int8Array(a);f.HEAP16=A=new Int16Array(a);f.HEAPU8=z=new Uint8Array(a);f.HEAPU16=B=new Uint16Array(a);f.HEAP32=D=new Int32Array(a);f.HEAPU32=E=new Uint32Array(a);f.HEAPF32=ea=new Float32Array(a);f.HEAPF64=ha=new Float64Array(a)} +var ja=[],ka=[],la=[],ma=!1;function na(){var a=f.preRun.shift();ja.unshift(a)}var F=0,H=null,I=null;function oa(a){f.onAbort?.(a);a="Aborted("+a+")";r(a);da=!0;a+=". Build with -sASSERTIONS for more info.";ma&&pa();a=new WebAssembly.RuntimeError(a);q(a);throw a;}var qa=a=>a.startsWith("data:application/octet-stream;base64,"),ra=a=>a.startsWith("file://"),J;function sa(a){if(a==J&&t)return new Uint8Array(t);throw"both async and sync fetching of the wasm failed";} +function ta(a){return t?Promise.resolve().then(()=>sa(a)):(void 0)(a).then(b=>new Uint8Array(b),()=>sa(a))}function ua(a,b,c){return ta(a).then(d=>WebAssembly.instantiate(d,b)).then(c,d=>{r(`failed to asynchronously prepare wasm: ${d}`);oa(d)})} +function va(a,b){var c=J;return t||"function"!=typeof WebAssembly.instantiateStreaming||qa(c)||ra(c)||"function"!=typeof fetch?ua(c,a,b):fetch(c,{credentials:"same-origin"}).then(d=>WebAssembly.instantiateStreaming(d,a).then(b,function(e){r(`wasm streaming compile failed: ${e}`);r("falling back to ArrayBuffer instantiation");return ua(c,a,b)}))} +var K=a=>{for(;0{for(var b="";z[a];)b+=wa[z[a++]];return b},N={},O={},P={},Q,xa=a=>{throw new Q(a);},R,ya=(a,b)=>{function c(k){k=b(k);if(k.length!==d.length)throw new R("Mismatched type converter count");for(var g=0;g{O.hasOwnProperty(k)?e[g]=O[k]:(h.push(k),N.hasOwnProperty(k)||(N[k]=[]),N[k].push(()=>{e[g]=O[k];++l;l===h.length&&c(e)}))});0===h.length&& +c(e)};function za(a,b,c={}){var d=b.name;if(!a)throw new Q(`type "${d}" must have a positive integer typeid pointer`);if(O.hasOwnProperty(a)){if(c.H)return;throw new Q(`Cannot register type '${d}' twice`);}O[a]=b;delete P[a];N.hasOwnProperty(a)&&(b=N[a],delete N[a],b.forEach(e=>e()))}function S(a,b,c={}){if(!("argPackAdvance"in b))throw new TypeError("registerType registeredInstance requires argPackAdvance");return za(a,b,c)} +var T=[],U=[],Aa=a=>{switch(a){case void 0:return 2;case null:return 4;case !0:return 6;case !1:return 8;default:const b=T.pop()||U.length;U[b]=a;U[b+1]=1;return b}};function V(a){return this.fromWireType(E[a>>2])} +var Ba={name:"emscripten::val",fromWireType:a=>{if(!a)throw new Q("Cannot use deleted val. handle = "+a);var b=U[a];9Aa(b),argPackAdvance:8,readValueFromPointer:V,F:null},Ca=(a,b)=>{switch(b){case 4:return function(c){return this.fromWireType(ea[c>>2])};case 8:return function(c){return this.fromWireType(ha[c>>3])};default:throw new TypeError(`invalid float width (${b}): ${a}`);}},Da=(a,b)=>Object.defineProperty(b,"name",{value:a}), +Ea=a=>{for(;a.length;){var b=a.pop();a.pop()(b)}};function Fa(a){for(var b=1;b{if(void 0===f[a].D){var c=f[a];f[a]=function(...d){if(!f[a].D.hasOwnProperty(d.length))throw new Q(`Function '${b}' called with an invalid number of arguments (${d.length}) - expects one of (${f[a].D})!`);return f[a].D[d.length].apply(this,d)};f[a].D=[];f[a].D[c.G]=c}},Ia=(a,b,c)=>{if(f.hasOwnProperty(a)){if(void 0===c||void 0!==f[a].D&&void 0!==f[a].D[c])throw new Q(`Cannot register public name '${a}' twice`);Ha(a,a);if(f.hasOwnProperty(c))throw new Q(`Cannot register multiple overloads of a function with the same number of arguments (${c})!`); +f[a].D[c]=b}else f[a]=b,void 0!==c&&(f[a].J=c)},Ja=(a,b)=>{for(var c=[],d=0;d>2]);return c},Ka,La=(a,b,c=[])=>{a.includes("j")?(a=a.replace(/p/g,"i"),b=(0,f["dynCall_"+a])(b,...c)):b=Ka.get(b)(...c);return b},Pa=(a,b)=>(...c)=>La(a,b,c),Qa=(a,b)=>{a=L(a);var c=a.includes("j")?Pa(a,b):Ka.get(b);if("function"!=typeof c)throw new Q(`unknown function pointer with signature ${a}: ${b}`);return c},Ra,Ta=a=>{a=Sa(a);var b=L(a);W(a);return b},Ua=(a,b)=>{function c(h){e[h]||O[h]||(P[h]? +P[h].forEach(c):(d.push(h),e[h]=!0))}var d=[],e={};b.forEach(c);throw new Ra(`${a}: `+d.map(Ta).join([", "]));},Va=a=>{a=a.trim();const b=a.indexOf("(");return-1!==b?a.substr(0,b):a},Wa=(a,b,c)=>{switch(b){case 1:return c?d=>x[d]:d=>z[d];case 2:return c?d=>A[d>>1]:d=>B[d>>1];case 4:return c?d=>D[d>>2]:d=>E[d>>2];default:throw new TypeError(`invalid integer width (${b}): ${a}`);}},Xa="undefined"!=typeof TextDecoder?new TextDecoder:void 0,Ya=(a,b)=>{var c=z,d=a+b;for(b=a;c[b]&&!(b>=d);)++b;if(16e?d+=String.fromCharCode(e):(e-=65536,d+=String.fromCharCode(55296|e>>10,56320|e&1023))}}else d+=String.fromCharCode(e)}return d},Za="undefined"!=typeof TextDecoder?new TextDecoder("utf-16le"):void 0,$a=(a,b)=>{var c=a>>1;for(var d=c+b/2;!(c>=d)&&B[c];)++c; +c<<=1;if(32=b/2);++d){var e=A[a+2*d>>1];if(0==e)break;c+=String.fromCharCode(e)}return c},ab=(a,b,c)=>{c??=2147483647;if(2>c)return 0;c-=2;var d=b;c=c<2*a.length?c/2:a.length;for(var e=0;e>1]=a.charCodeAt(e),b+=2;A[b>>1]=0;return b-d},bb=a=>2*a.length,cb=(a,b)=>{for(var c=0,d="";!(c>=b/4);){var e=D[a+4*c>>2];if(0==e)break;++c;65536<=e?(e-=65536,d+=String.fromCharCode(55296|e>>10,56320|e&1023)):d+=String.fromCharCode(e)}return d}, +db=(a,b,c)=>{c??=2147483647;if(4>c)return 0;var d=b;c=d+c-4;for(var e=0;e=h){var l=a.charCodeAt(++e);h=65536+((h&1023)<<10)|l&1023}D[b>>2]=h;b+=4;if(b+4>c)break}D[b>>2]=0;return b-d},eb=a=>{for(var b=0,c=0;c=d&&++c;b+=4}return b},gb=a=>{a=a.getArg(X.C,0);return fb(a)};f.incrementExceptionRefcount=a=>{a=gb(a);hb(a)};f.decrementExceptionRefcount=a=>{a=gb(a);ib(a)}; +f.getExceptionMessage=a=>{var b=gb(a);a=jb();var c=kb(4),d=kb(4);lb(b,c,d);b=E[c>>2];d=E[d>>2];c=b?Ya(b):"";W(b);if(d){var e=d?Ya(d):"";W(d)}mb(a);return[c,e]};for(var nb=Array(256),Y=0;256>Y;++Y)nb[Y]=String.fromCharCode(Y);wa=nb;Q=f.BindingError=class extends Error{constructor(a){super(a);this.name="BindingError"}};R=f.InternalError=class extends Error{constructor(a){super(a);this.name="InternalError"}};U.push(0,1,void 0,1,null,1,!0,1,!1,1);f.count_emval_handles=()=>U.length/2-5-T.length; +Ra=f.UnboundTypeError=((a,b)=>{var c=Da(b,function(d){this.name=b;this.message=d;d=Error(d).stack;void 0!==d&&(this.stack=this.toString()+"\n"+d.replace(/^Error(:[^\n]*)?\n/,""))});c.prototype=Object.create(a.prototype);c.prototype.constructor=c;c.prototype.toString=function(){return void 0===this.message?this.name:`${this.name}: ${this.message}`};return c})(Error,"UnboundTypeError"); +var pb={l:()=>{oa("")},k:()=>{},i:(a,b,c,d)=>{b=L(b);S(a,{name:b,fromWireType:function(e){return!!e},toWireType:function(e,h){return h?c:d},argPackAdvance:8,readValueFromPointer:function(e){return this.fromWireType(z[e])},F:null})},h:a=>S(a,Ba),e:(a,b,c)=>{b=L(b);S(a,{name:b,fromWireType:d=>d,toWireType:(d,e)=>e,argPackAdvance:8,readValueFromPointer:Ca(b,c),F:null})},d:(a,b,c,d,e,h,l)=>{var k=Ja(b,c);a=L(a);a=Va(a);e=Qa(d,e);Ia(a,function(){Ua(`Cannot call ${a} due to unbound types`,k)},b-1);ya(k, +g=>{var m=a;var n=a;g=[g[0],null].concat(g.slice(1));var w=e,p=g.length;if(2>p)throw new Q("argTypes array size mismatch! Must at least get return value and 'this' types!");var C=null!==g[1]&&!1,M=Fa(g),Ma="void"!==g[0].name;w=[n,xa,w,h,Ea,g[0],g[1]];for(var v=0;v{b=L(b);-1===e&&(e=4294967295);e=k=>k;if(0===d){var h=32-8*c;e=k=>k<>>h}var l=b.includes("unsigned")?function(k,g){return g>>>0}:function(k,g){return g};S(a,{name:b,fromWireType:e,toWireType:l,argPackAdvance:8,readValueFromPointer:Wa(b,c,0!==d),F:null})},a:(a,b,c)=>{function d(h){return new e(x.buffer,E[h+4>>2],E[h>>2])}var e=[Int8Array,Uint8Array,Int16Array,Uint16Array,Int32Array,Uint32Array,Float32Array,Float64Array][b];c=L(c);S(a,{name:c,fromWireType:d, +argPackAdvance:8,readValueFromPointer:d},{H:!0})},f:(a,b)=>{b=L(b);var c="std::string"===b;S(a,{name:b,fromWireType:function(d){var e=E[d>>2],h=d+4;if(c)for(var l=h,k=0;k<=e;++k){var g=h+k;if(k==e||0==z[g]){l=l?Ya(l,g-l):"";if(void 0===m)var m=l;else m+=String.fromCharCode(0),m+=l;l=g+1}}else{m=Array(e);for(k=0;k=g?k++:2047>=g?k+=2:55296<=g&&57343>=g?(k+=4,++h):k+=3}else k=e.length;h=k;k=ob(4+h+1);g=k+4;E[k>>2]=h;if(c&&l){if(l=g,g=h+1,h=z,0=n){var w=e.charCodeAt(++m);n=65536+((n&1023)<<10)|w&1023}if(127>=n){if(l>=g)break;h[l++]=n}else{if(2047>= +n){if(l+1>=g)break;h[l++]=192|n>>6}else{if(65535>=n){if(l+2>=g)break;h[l++]=224|n>>12}else{if(l+3>=g)break;h[l++]=240|n>>18;h[l++]=128|n>>12&63}h[l++]=128|n>>6&63}h[l++]=128|n&63}}h[l]=0}}else if(l)for(l=0;l{c=L(c);if(2===b){var d=$a;var e=ab;var h=bb; +var l=k=>B[k>>1]}else 4===b&&(d=cb,e=db,h=eb,l=k=>E[k>>2]);S(a,{name:c,fromWireType:k=>{for(var g=E[k>>2],m,n=k+4,w=0;w<=g;++w){var p=k+4+w*b;if(w==g||0==l(p))n=d(n,p-n),void 0===m?m=n:(m+=String.fromCharCode(0),m+=n),n=p+b}W(k);return m},toWireType:(k,g)=>{if("string"!=typeof g)throw new Q(`Cannot pass non-string to C++ string type ${c}`);var m=h(g),n=ob(4+m+b);E[n>>2]=m/b;e(g,n+4,m+b);null!==k&&k.push(W,n);return n},argPackAdvance:8,readValueFromPointer:V,F(k){W(k)}})},j:(a,b)=>{b=L(b);S(a,{I:!0, +name:b,argPackAdvance:0,fromWireType:()=>{},toWireType:()=>{}})},n:(a,b,c)=>z.copyWithin(a,b,b+c),g:(a,b)=>{var c=O[a];if(void 0===c)throw a=`${"_emval_take_value"} has unknown type ${Ta(a)}`,new Q(a);a=c;a=a.readValueFromPointer(b);return Aa(a)},m:a=>{var b=z.length;a>>>=0;if(2147483648=c;c*=2){var d=b*(1+.2/c);d=Math.min(d,a+100663296);var e=Math;d=Math.max(a,d);a:{e=(e.min.call(e,2147483648,d+(65536-d%65536)%65536)-u.buffer.byteLength+65535)/65536;try{u.grow(e);ia();var h= +1;break a}catch(l){}h=void 0}if(h)return!0}return!1}},X=function(){function a(c){X=c.exports;u=X.o;ia();Ka=X.t;ka.unshift(X.p);F--;f.monitorRunDependencies?.(F);0==F&&(null!==H&&(clearInterval(H),H=null),I&&(c=I,I=null,c()));return X}var b={a:pb};F++;f.monitorRunDependencies?.(F);if(f.instantiateWasm)try{return f.instantiateWasm(b,a)}catch(c){r(`Module.instantiateWasm callback failed with error: ${c}`),q(c)}J||=qa("zstd_codec.wasm")?"zstd_codec.wasm":f.locateFile?f.locateFile("zstd_codec.wasm",""): +"zstd_codec.wasm";va(b,function(c){a(c.instance)}).catch(q);return{}}(),ob=a=>(ob=X.q)(a),W=a=>(W=X.r)(a),Sa=a=>(Sa=X.s)(a),pa=()=>(pa=X.u)(),mb=a=>(mb=X.v)(a),kb=a=>(kb=X.w)(a),jb=()=>(jb=X.x)(),ib=a=>(ib=X.y)(a),hb=a=>(hb=X.z)(a),fb=a=>(fb=X.A)(a),lb=(a,b,c)=>(lb=X.B)(a,b,c),Z;I=function rb(){Z||sb();Z||(I=rb)}; +function sb(){function a(){if(!Z&&(Z=!0,f.calledRun=!0,!da)){ma=!0;K(ka);aa(f);f.onRuntimeInitialized?.();if(f.postRun)for("function"==typeof f.postRun&&(f.postRun=[f.postRun]);f.postRun.length;){var b=f.postRun.shift();la.unshift(b)}K(la)}}if(!(0 = class Zstd implements Codec { emscriptenModule = init(); } const module = await emscriptenModule; - const view = module.decompress(data); - const result = new Uint8Array(view); // Copy view and free wasm memory - module.free_result(); - if (out !== undefined) { - out.set(result); - return out; + try { + const view = module.decompress(data); + const result = new Uint8Array(view); // Copy view and free wasm memory + module.free_result(); + if (out !== undefined) { + out.set(result); + return out; + } + return result; + } catch (err) { + throw new Error(module.getExceptionMessage(err).toString()); } - return result; } }; diff --git a/test/zstd.test.js b/test/zstd.test.js index 3a4087e..c5b3ed5 100644 --- a/test/zstd.test.js +++ b/test/zstd.test.js @@ -39,3 +39,52 @@ test('Static constructor', async t => { const encAndDec = await checkAsyncEncodeDecode(codec, arrays[0]); t.equal(arrays[0], encAndDec); }); + +test('Streaming decompression', async t => { + // Test input frames with unknown frame content size + const config = { id: 'zstd' }; + const codec = Zstd.fromConfig(config); + + // Encode bytes directly that were the result of streaming compression + const bytes = new Uint8Array([ + 40, 181, 47, 253, 0, 88, 97, 0, 0, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, + ]); + const dec = await codec.decode(bytes); + const str = Buffer.from(dec).toString(); + t.equal(str, 'Hello World!'); + + // Two consecutive frames given as input + const bytes2 = new Uint8Array(bytes.length * 2); + bytes2.set(bytes, 0); + bytes2.set(bytes, bytes.length); + const dec2 = await codec.decode(bytes2); + const str2 = Buffer.from(dec2).toString(); + t.equal(str2, 'Hello World!Hello World!'); + + // Single long frame that decompresses to a large output + const bytes3 = new Uint8Array([ + 40, 181, 47, 253, 0, 88, 36, 2, 0, 164, 3, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 1, 0, 58, 252, 223, 115, 5, 5, 76, 0, 0, 8, + 115, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 107, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 99, 1, 0, 252, 255, 57, + 16, 2, 76, 0, 0, 8, 91, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 83, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 75, 1, + 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 67, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 117, 1, 0, 252, 255, 57, 16, 2, 76, + 0, 0, 8, 109, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 101, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 93, 1, 0, 252, + 255, 57, 16, 2, 76, 0, 0, 8, 85, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 77, 1, 0, 252, 255, 57, 16, 2, 77, 0, 0, 8, + 69, 1, 0, 252, 127, 29, 8, 1, + ]); + const dec3 = await codec.decode(bytes3); + const str3 = Buffer.from(dec3).toString(); + t.equal(str3, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz'.repeat(1024 * 32)); + + // Garbage input results in an error + const bytes4 = new Uint8Array([0, 0, 0, 0, 0, 0, 0, 0]); + codec.decode(bytes4).then( + result => { + t.fail(); + }, + error => { + t.ok(error.message.match('zstd codec error: content size error')); + } + ); +});