From e47535a25bc81325868a0b074a7338c97b4b5a6c Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Sat, 11 Aug 2018 01:09:09 +0200 Subject: [PATCH 01/23] testing subtests --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c884213..d63bd62 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "main": "index.js", "scripts": { "test": "npm run test:lint && npm run test:unit", - "test:unit": "tap -- ts_test/*.ts", + "test:unit": "tap -- ts_test/*.ts ts_test/**/*.ts", "test:lint": "tslint -c .tslintrc.json ts/**/*.ts && standard", "build:types-js": "pbjs -t static-module -w commonjs -l \"eslint-disable one-var, no-mixed-operators\" --es6 spec/*.proto -o ts/Types.js && standard --fix ts/Types.js", "build:types-ts": "npm run build:types-js && pbts ts/Types.js -o ts/Types.d.ts && tslint -c .tslintrc.json --fix ts/Types.d.ts", From fc08b517a2ee0dfa551484094801342a1dc62994 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Fri, 10 Aug 2018 23:09:32 +0200 Subject: [PATCH 02/23] The typing of Feature prevented feature tests. --- ts/api/Feature.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ts/api/Feature.ts b/ts/api/Feature.ts index 0b49e47..c8ae527 100644 --- a/ts/api/Feature.ts +++ b/ts/api/Feature.ts @@ -3,7 +3,7 @@ import FeatureType from './FeatureType' const { double, uint8 } = FeatureType -const Feature: { [k: string]: IFeature } = { +export default { x: { name: 'x', type: double }, y: { name: 'y', type: double }, z: { name: 'z', type: double }, @@ -12,5 +12,3 @@ const Feature: { [k: string]: IFeature } = { b: { name: 'b', type: uint8 }, a: { name: 'a', type: uint8 } } - -export default Feature From 4326aec79d73138f31ce15d0aebf40de92534740 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Fri, 10 Aug 2018 23:14:20 +0200 Subject: [PATCH 03/23] After rethinking the documentation this new type structure makes a lot more sense. --- spec/FeatureType.proto | 48 +++++++++++------------ ts/api/FeatureType.ts | 86 +++++++++++++++++++++++++++++------------- 2 files changed, 82 insertions(+), 52 deletions(-) diff --git a/spec/FeatureType.proto b/spec/FeatureType.proto index fdeced9..c5cbbcd 100644 --- a/spec/FeatureType.proto +++ b/spec/FeatureType.proto @@ -1,27 +1,25 @@ enum FeatureType { - uint8 = 1; - uint16 = 2; - uint32 = 3; - uint64 = 4; - int8 = 5; - int16 = 6; - int32 = 7; - int64 = 8; - sint8 = 9; - sint16 = 10; - sint32 = 11; - sint64 = 12; - fixedint8 = 13; - fixedint16 = 14; - fixedint32 = 15; - fixedint64 = 16; - sfixedint8 = 17; - sfixedint16 = 18; - sfixedint32 = 19; - sfixedint64 = 20; - string = 21 [(length) = -1]; - bytes = 22 [(length) = -1]; - bool = 23; - float = 24; - double = 25; + int8 = 1; // fixed size 1 byte number: -127~128 + uint8 = 2; // fixed size 1 byte number: 0~255 + int16 = 3; // fixed size 2 byte number: -32768~32768 + uint16 = 4; // fixed size 2 byte number: 0~255 + (0~255)<<16 => 0~65536 + int32 = 4; // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 -2147483648 => 2147483648~-2147483648 + uint32 = 5; // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 => 0~4294967296 + sint32 = 6; // fixed size 4 byte, reverse stored number: -2147483648~2147483648 + varint32 = 7; // variable size 2~5 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15) => 2147483648~-2147483648 + varuint32 = 8; // variable size 2~5 byte, reverse stored number: 0~4294967296 + varsint32 = 9; // variable size 2~5 byte, reverse stored number: -2147483648~2147483648 + int64 = 10; // fixed size 8 byte number: -9223372036854776000~9223372036854776000 + uint64 = 11; // fixed size 8 byte number: 0~18446744073709552000 + sint64 = 12; // fixed size 8 byte, reverse stored number: 9223372036854776000~-9223372036854776000 + varuint64 = 14; // variable size 2~9 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15) => 9223372036854776000~-9223372036854776000 + varint64 = 13; // variable size 2~9 byte number: 0~18446744073709552000 + varsint64 = 15; // variable size 2~9 byte, reverse stored number: -9223372036854776000~9223372036854776000 + bytes = 16; // variable size bytes: uint32-size-number + amount of bytes + string = 17; // variable size string: utf-8 encoded bytes + bool = 18; // 0 = false, 1~7 = true + float = 19; // fixed size 4 byte floating point + double = 20; // fixed size 8 byte floating point + fixedstring = 21[(length) = -1]; // fixed amount of string + fixedbytes = 22[(length) = -1]; // fixed amount of bytes } diff --git a/ts/api/FeatureType.ts b/ts/api/FeatureType.ts index 5b3e8b6..f374128 100644 --- a/ts/api/FeatureType.ts +++ b/ts/api/FeatureType.ts @@ -1,31 +1,63 @@ enum FeatureType { - uint8 = 1, - uint16 = 2, - uint32 = 3, - uint64 = 4, - int8 = 5, - int16 = 6, - int32 = 7, - int64 = 8, - sint8 = 9, - sint16 = 10, - sint32 = 11, - sint64 = 12, - fixedint8 = 13, - fixedint16 = 14, - fixedint32 = 15, - fixedint64 = 16, - sfixedint8 = 17, - sfixedint16 = 18, - sfixedint32 = 19, - sfixedint64 = 20, - string = 21, - bytes = 22, - bool = 23, - float = 24, - double = 25, - varstring = 26, - varbytes = 27 + int8 = 1, // fixed size 1 byte number: -127~128 + uint8 = 2, // fixed size 1 byte number: 0~255 + + int16 = 3, // fixed size 2 byte number: -32768~32768 + uint16 = 4, // fixed size 2 byte number: 0~255 + (0~255)<<16 => 0~65536 + + int32 = 4, // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 -2147483648 => 2147483648~-2147483648 + uint32 = 5, // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 => 0~4294967296 + sint32 = 6, // fixed size 4 byte, reverse stored number: -2147483648~2147483648 + varint32 = 7, // variable size 1~5 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15) => 2147483648~-2147483648 + varuint32 = 8, // variable size 1~5 byte number: (same like varint) 0~4294967296 + varsint32 = 9, // variable size 1~5 byte, reverse stored number: -2147483648~2147483648 + + int64 = 10, // fixed size 8 byte number: -9223372036854776000~9223372036854776000 + uint64 = 11, // fixed size 8 byte number: 0~18446744073709552000 + sint64 = 12, // fixed size 8 byte, reverse stored number: 9223372036854776000~-9223372036854776000 + varint64 = 13, // variable size 1~9 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15)... => 9223372036854776000~-9223372036854776000 + varuint64 = 14, // variable size 1~9 byte number: (same like varint) 0~18446744073709552000 + varsint64 = 15, // variable size 1~9 byte, reverse stored number: -9223372036854776000~9223372036854776000 + + bytes = 16, // variable size bytes: uint32-size-number + amount of bytes + string = 17, // variable size string: utf-8 encoded bytes + + bool = 18, // 0 = false, 1~7 = true + + float = 19, // fixed size 4 byte floating point + double = 21, // fixed size 8 byte floating point + + fixedstring = 22, // fixed amount of string + fixedbytes = 23 // fixed amount of bytes } export default FeatureType + +export function parseFeatureString (type: string): FeatureType { + switch (type) { + case 'int8': return FeatureType.int8 + case 'uint8': return FeatureType.uint8 + case 'int16': return FeatureType.int16 + case 'uint16': return FeatureType.uint16 + case 'int32': return FeatureType.int32 + case 'uint32': return FeatureType.uint32 + case 'sint32': return FeatureType.sint32 + case 'varint32': return FeatureType.varint32 + case 'varuint32': return FeatureType.varuint32 + case 'varsint32': return FeatureType.varsint32 + case 'int64': return FeatureType.int64 + case 'uint64': return FeatureType.uint64 + case 'sint64': return FeatureType.sint64 + case 'varint64': return FeatureType.varint64 + case 'varuint64': return FeatureType.varuint64 + case 'varsint64': return FeatureType.varsint64 + case 'bytes': return FeatureType.bytes + case 'string': return FeatureType.string + case 'bool': return FeatureType.bool + case 'float': return FeatureType.float + case 'double': return FeatureType.double + case 'fixedstring': return FeatureType.fixedstring + case 'fixedbytes': return FeatureType.fixedbytes + } + throw new Error(`Unknown feature type ${type}`) +} From ff0595f2db0713f69d2266aab9486de199c096b8 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Fri, 10 Aug 2018 23:22:15 +0200 Subject: [PATCH 04/23] First stab at implementing readers for types. Uses DataView to make it a bit easier. --- package-lock.json | 5 + package.json | 2 + ts/reader/IDynamicResult.ts | 5 + ts/reader/IReader.ts | 32 ++++++ ts/reader/readerForFeatureType.ts | 174 ++++++++++++++++++++++++++++++ tsconfig.json | 2 +- 6 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 ts/reader/IDynamicResult.ts create mode 100644 ts/reader/IReader.ts create mode 100644 ts/reader/readerForFeatureType.ts diff --git a/package-lock.json b/package-lock.json index 9ffcb01..690e18d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -630,6 +630,11 @@ "dev": true, "optional": true }, + "decode-utf8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/decode-utf8/-/decode-utf8-1.0.1.tgz", + "integrity": "sha512-iTpp497Pgdg5JdQ3HkHbG0vwSSYRDrtVXKpz/ieFhY+LjLOZ2HjVtdmNKyO4f25lr4Q6O0cYSzlo6h27LgsLDA==" + }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", diff --git a/package.json b/package.json index d63bd62..be66a81 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,8 @@ "typescript-eslint-parser": "^16.0.1" }, "dependencies": { + "bluebird": "^3.5.1", + "decode-utf8": "^1.0.1", "long": "^4.0.0", "protobufjs": "^6.8.6", "three": "^0.94.0", diff --git a/ts/reader/IDynamicResult.ts b/ts/reader/IDynamicResult.ts new file mode 100644 index 0000000..beb68bb --- /dev/null +++ b/ts/reader/IDynamicResult.ts @@ -0,0 +1,5 @@ +export default interface IDynamicResult { + size: number + data: any + byteOffset: number +} diff --git a/ts/reader/IReader.ts b/ts/reader/IReader.ts new file mode 100644 index 0000000..24e932b --- /dev/null +++ b/ts/reader/IReader.ts @@ -0,0 +1,32 @@ +import IDynamicResult from './IDynamicResult' + +export default interface IReader { + fixedSize: boolean + size?: number + minSize: number + readDynamic (view: DataView, byteOffset): IDynamicResult + read (view: DataView, byteOffset: number): any +} + +export function createFixedReader (size: number, read: (view: DataView, byteOffset: number) => any): IReader { + return { + fixedSize: true, + minSize: size, + size, + readDynamic: (view: DataView, byteOffset: number): IDynamicResult => ({ + size, + byteOffset: byteOffset + size, + data: read(view, byteOffset) + }), + read + } +} + +export function createDynamicReader (minSize: number, readDynamic: (view: DataView, byteOffset: number) => IDynamicResult): IReader { + return { + fixedSize: false, + readDynamic, + minSize, + read: (view: DataView, byteOffset: number) => readDynamic(view, byteOffset).data + } +} diff --git a/ts/reader/readerForFeatureType.ts b/ts/reader/readerForFeatureType.ts new file mode 100644 index 0000000..6fdb84e --- /dev/null +++ b/ts/reader/readerForFeatureType.ts @@ -0,0 +1,174 @@ +import IReader, { createFixedReader, createDynamicReader } from './IReader' +import FeatureType from '../api/FeatureType' +import Long from 'long' +import IDynamicResult from './IDynamicResult' +import decodeUtf8 from 'decode-utf8' + +function zzDecodeLong (long: Long) { + // Adapted from https://github.com/dcodeIO/protobuf.js/blob/69623a91c1e4a99d5210b5295a9e5b39d9517554/src/util/longbits.js#L176-L181 + const mask = -(long.low & 1) + long.low = ((long.low >>> 1 | long.high << 31) ^ mask) >>> 0 + long.high = (long.high >>> 1 ^ mask) >>> 0 + return long +} + +function zzDecode (result: IDynamicResult): IDynamicResult { + // Adapted from https://github.com/dcodeIO/protobuf.js/blob/69623a91c1e4a99d5210b5295a9e5b39d9517554/src/reader.js#L113-L114 + result.data = result.data >>> 1 ^ -(result.data & 1) | 0 + return result +} + +const boolReader: IReader = createFixedReader(1, (view: DataView, byteOffset: number) => view.getInt8(byteOffset) === 0) +const doubleReader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => view.getFloat64(byteOffset)) +const floatReader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getFloat32(byteOffset)) +const int8Reader: IReader = createFixedReader(1, (view: DataView, byteOffset: number) => view.getInt8(byteOffset)) +const uint8Reader: IReader = createFixedReader(1, (view: DataView, byteOffset: number) => view.getUint8(byteOffset)) +const int16Reader: IReader = createFixedReader(2, (view: DataView, byteOffset: number) => view.getInt16(byteOffset)) +const uint16Reader: IReader = createFixedReader(2, (view: DataView, byteOffset: number) => view.getUint16(byteOffset)) +const int32Reader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getInt32(byteOffset)) +const uint32Reader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getUint32(byteOffset)) +const sint32Reader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getInt32(byteOffset) | 0) +const int64Reader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => new Long(view.getInt32(byteOffset), view.getInt32(byteOffset), false)) +const uint64Reader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => new Long(view.getInt32(byteOffset), view.getInt32(byteOffset), true)) +const sint64Reader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => zzDecodeLong(new Long(view.getInt32(byteOffset), view.getInt32(byteOffset), false))) + +function readVarInt (view: DataView, byteOffset: number) { + // Adapted from https://github.com/dcodeIO/protobuf.js/blob/69623a91c1e4a99d5210b5295a9e5b39d9517554/src/reader.js#L85-L89 + const a = view.getUint8(byteOffset++) + let data = (a & 127) >>> 0 + if (a < 128) return { size: 1, data, byteOffset, next: 0 } + const b = view.getUint8(byteOffset++) + data = data | ((b && 127) << 7) >>> 0 + if (b < 128) return { size: 2, data, byteOffset, next: 0 } + const c = view.getUint8(byteOffset++) + data = data | ((c && 127) << 14) >>> 0 + if (c < 128) return { size: 3, data, byteOffset, next: 0 } + const d = view.getUint8(byteOffset++) + data = data | ((d && 127) << 21) >>> 0 + if (d < 128) return { size: 4, data, byteOffset, next: 0 } + const e = view.getUint8(byteOffset++) + data = data | ((e && 127) << 28) >>> 0 + return { size: 5, data, byteOffset, next: e & 128 } +} + +function readVarUint32 (view: DataView, byteOffset: number): IDynamicResult { + const result = readVarInt(view, byteOffset) + if (result.next !== 0) { + throw new Error('Invalid var-int 32 encoding!') + } + return result +} + +const varInt32Reader: IReader = createDynamicReader(1, (view: DataView, byteOffset: number): IDynamicResult => { + const result = readVarUint32(view, byteOffset) + result.data = result.data | 0 + return result +}) + +const varUint32Reader: IReader = createDynamicReader(1, readVarUint32) +const varSint32Reader: IReader = createDynamicReader(1, (view: DataView, byteOffset: number): IDynamicResult => zzDecode(readVarUint32(view, byteOffset))) + +function readVarInt64 (unsigned: boolean, view: DataView, byteOffset: number): IDynamicResult { + const low = readVarInt(view, byteOffset) + if (low.next === 0) { + if (!unsigned) { + low.data = low.data | 0 + } + return low + } + const high = readVarInt(view, low.byteOffset) + if (high.next !== 0) { + throw new Error('Invalid var-int 64 encoding!') + } + return { + size: low.size + high.size, + byteOffset: high.byteOffset, + data: new Long(low.data, high.data, unsigned) + } +} + +const varInt64Reader: IReader = createDynamicReader(1, readVarInt64.bind(null, false)) +const varUint64Reader: IReader = createDynamicReader(1, readVarInt64.bind(null, true)) +const varSint64Reader: IReader = createDynamicReader(1, (view: DataView, byteOffset: number): IDynamicResult => { + const result = readVarInt64(false, view, byteOffset) + if (typeof result.data === 'number') { + return zzDecode(result) + } + result.data = zzDecodeLong(result.data) + return result +}) + +function fixedStringReader (length: number): IReader { + if (isNaN(length)) { + throw new Error(`A fixed string needs a length: ${length}`) + } + const bytesReader = fixedBytesReader(length) + return createFixedReader(length, (view: DataView, byteOffset: number) => decodeUtf8(bytesReader.read(view, byteOffset))) +} + +function fixedBytesReader (length: number): IReader { + if (isNaN(length)) { + throw new Error(`A fixed string needs a length: ${length}`) + } + return createFixedReader(length, (view: DataView, byteOffset: number) => view.buffer.slice(byteOffset, byteOffset + length)) +} + +function readBytes (view: DataView, byteOffset: number) { + const size = int32Reader.read(view, byteOffset) + const start = byteOffset + byteOffset += size + if (view.buffer.byteLength < byteOffset) { + return null + } + return { + size, + byteOffset, + data: view.buffer.slice(start, byteOffset) + } +} + +const bytesReader = createDynamicReader(2, readBytes) + +const stringReader = createDynamicReader(2, (view: DataView, byteOffset: number) => { + const result = bytesReader.readDynamic(view, byteOffset) + if (result === null) { + return null + } + const buffer: ArrayBuffer = result.data + result.data = decodeUtf8(buffer) + return result +}) + +export default function readerForFeatureType (type: FeatureType, length?: number): IReader { + switch (type) { + case FeatureType.fixedstring: return fixedStringReader(length) + case FeatureType.fixedbytes: return fixedBytesReader(length) + } + if (length !== undefined) { + throw new Error(`FeatureType#${type} is not a dynamic-size type, don't specify a length.`) + } + switch (type) { + case FeatureType.int8: return int8Reader + case FeatureType.uint8: return uint8Reader + case FeatureType.int16: return int16Reader + case FeatureType.uint16: return uint16Reader + case FeatureType.int32: return int32Reader + case FeatureType.uint32: return uint32Reader + case FeatureType.sint32: return sint32Reader + case FeatureType.varint32: return varInt32Reader + case FeatureType.varuint32: return varUint32Reader + case FeatureType.varsint32: return varSint32Reader + case FeatureType.int64: return int64Reader + case FeatureType.uint64: return uint64Reader + case FeatureType.sint64: return sint64Reader + case FeatureType.varint64: return varInt64Reader + case FeatureType.varuint64: return varUint64Reader + case FeatureType.varsint64: return varSint64Reader + case FeatureType.bytes: return bytesReader + case FeatureType.string: return stringReader + case FeatureType.bool: return boolReader + case FeatureType.float: return floatReader + case FeatureType.double: return doubleReader + } + throw new Error(`No reader available for FeatureType#${type}.`) +} diff --git a/tsconfig.json b/tsconfig.json index 745ae88..4fd95cb 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,7 +8,7 @@ "allowJs": true, "esModuleInterop": true, "outDir": "build", - "lib": [ "es6", "es2015.promise", "dom" ], + "lib": [ "dom", "es2017", "webworker" ], "rootDir": "ts" }, "exclude": [ From e3be5f6db27c7375176493770754b41baeb3c68e Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Fri, 10 Aug 2018 23:28:55 +0200 Subject: [PATCH 05/23] Combines different readers into one reader. --- ts/reader/readerForFeatures.ts | 60 ++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 ts/reader/readerForFeatures.ts diff --git a/ts/reader/readerForFeatures.ts b/ts/reader/readerForFeatures.ts new file mode 100644 index 0000000..6952731 --- /dev/null +++ b/ts/reader/readerForFeatures.ts @@ -0,0 +1,60 @@ +import IFeature from '../api/IFeature' +import IReader, { createDynamicReader, createFixedReader } from './IReader' +import readerForFeatureType from './readerForFeatureType' +import IDynamicResult from './IDynamicResult' + +function readerForDynamicFeatures (features: IFeature[], readers: IReader[]): IReader { + const minSize = readers.reduce((minSize: number, reader: IReader) => minSize + reader.minSize, 0) + return createDynamicReader(minSize, (view: DataView, byteOffset: number): IDynamicResult => { + const result: IDynamicResult = { + size: 0, + byteOffset, + data: {} + } + readers.forEach((reader, index): void => { + const feature = features[index] + // TODO: Implement a size check + if (reader.fixedSize) { + const featureResult = reader.read(view, byteOffset) + result.size += reader.size + result.data[feature.name] = featureResult.data + byteOffset += reader.size + } else { + const featureResult = reader.readDynamic(view, byteOffset) + result.size += featureResult.size + result.data[feature.name] = featureResult.data + byteOffset = featureResult.byteOffset + } + }) + return result + }) +} + +function readerForFixedFeatures (features: IFeature[], readers: IReader[]): IReader { + let size = 0 + const readFns = features.map((feature, index) => { + const reader = readers[index] + const offset = size + const target = feature.name + size += reader.size + return (view: DataView, byteOffset: number, result: { [k: string]: any }) => { + result[target] = reader.read(view, byteOffset + offset) + } + }) + return createFixedReader(size, (view: DataView, byteOffset: number) => { + const result = {} + for (const readFn of readFns) { + readFn(view, byteOffset, result) + } + return result + }) +} + +export default function readerForFeatures (features: IFeature[]): IReader { + const readers: IReader[] = features.map(feature => readerForFeatureType(feature.type, feature.length)) + const fixedSize: boolean = readers.find(reader => !reader.fixedSize) === undefined + if (fixedSize) { + return readerForFixedFeatures(features, readers) + } + return readerForDynamicFeatures(features, readers) +} From dc96f23025af7bd5539acb9395b3934932894b0a Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Sat, 11 Aug 2018 00:17:03 +0200 Subject: [PATCH 06/23] Started on implementation of reading a stream of data --- ts/reader/readFromStream.ts | 55 +++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 ts/reader/readFromStream.ts diff --git a/ts/reader/readFromStream.ts b/ts/reader/readFromStream.ts new file mode 100644 index 0000000..da4e7f4 --- /dev/null +++ b/ts/reader/readFromStream.ts @@ -0,0 +1,55 @@ +import Stream, { ReadableStream } from 'ts-stream' +import IReader from './IReader' +import { mapSeries } from 'bluebird' + +function combine (a: Uint8Array, b: Uint8Array) { + const combined = new Uint8Array(a.length + b.length) + combined.set(a, 0) + combined.set(b, a.length) + return combined +} + +function readFixedSize (out: Stream, inStream: ReadableStream, reader: IReader) { + let leftOver = null + return inStream.forEach((data: Uint8Array) => { + let end = reader.size + let start = 0 + if (leftOver !== null) { + data = combine(leftOver, data) + } + let entries: any[] + while (end <= data.length) { + if (entries === undefined) { + entries = [] + } + entries.push(reader.read(new DataView(data.buffer), start)) + start = end + end += reader.size + } + if (end === data.length) { + leftOver = null + } else { + leftOver = data.subarray(start) + } + if (entries === undefined) { + return + } + return mapSeries(entries, entry => out.write(entry)) + }).then(() => leftOver) +} + +function readDynamicSize (out: Stream, inStream: ReadableStream, reader: IReader) { + return Promise.reject(new Error('TODO: Implement')) +} + +export default function readFromStream (inStream: ReadableStream, reader: IReader): ReadableStream { + const out = new Stream() + let process = reader.fixedSize + ? readFixedSize(out, inStream, reader) + : readDynamicSize(out, inStream, reader) + + process + .then((leftOver) => out.end(null, leftOver)) + .catch((reason: Error) => out.end(reason)) + return out +} From b441c06a2a95918f57d671127c5f93b0546638f0 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Sat, 11 Aug 2018 00:20:04 +0200 Subject: [PATCH 07/23] Added implementation to transform a ReadableStreamReader (fetch) into a TsStream --- ts/reader/readerConv.ts | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 ts/reader/readerConv.ts diff --git a/ts/reader/readerConv.ts b/ts/reader/readerConv.ts new file mode 100644 index 0000000..09765fa --- /dev/null +++ b/ts/reader/readerConv.ts @@ -0,0 +1,25 @@ +import TsStream from 'ts-stream' + +function consume (from: ReadableStreamReader, to: TsStream): void { + from + .read() + .then(({ done, value }) => { + if (done) { + to.end() + return + } + to.write(value) + .then(() => consume(from, to)) + .catch(() => { + from.cancel() + from.releaseLock() + }) + }) + .catch(error => to.abort(error)) +} + +export default function readerConv (from: ReadableStreamReader): TsStream { + const to = new TsStream() + consume(from, to) + return to +} From 27bbaf1a8f4621e888118f5bd5aa7025d0a8b74d Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Sat, 11 Aug 2018 00:21:40 +0200 Subject: [PATCH 08/23] Added implementation to transform a Stream (node) into a TsStream --- ts/reader/streamConv.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 ts/reader/streamConv.ts diff --git a/ts/reader/streamConv.ts b/ts/reader/streamConv.ts new file mode 100644 index 0000000..7d73f51 --- /dev/null +++ b/ts/reader/streamConv.ts @@ -0,0 +1,28 @@ +import { Stream as TsStream } from 'ts-stream' +import { Readable, Writable, WritableOptions } from 'stream' +import { mapSeries } from 'bluebird' + +export function streamConv (from: Readable): TsStream { + const to = new TsStream() + from.pipe(new Writable({ + write: (chunk, encoding, callback: (error?: Error) => void) => + to + .write(chunk) + .then(() => callback()) + .catch(callback), + + writev: (chunks: any[], callback: (error?: Error) => void) => + mapSeries(chunks, chunk => to.write(chunk)) + .then(() => callback()) + .catch(callback), + + destroy: (error: Error) => to.abort(error), + + final: (callback: (error?: Error) => void) => + to + .end() + .then(() => callback()) + .catch(callback) + })) + return to +} From 11ac35a41d9aef56beb34a0ab83d6d9dc3a0a6c0 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Sat, 11 Aug 2018 00:30:01 +0200 Subject: [PATCH 09/23] First shot at tests --- ts_test/reader/readFromStream.ts | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 ts_test/reader/readFromStream.ts diff --git a/ts_test/reader/readFromStream.ts b/ts_test/reader/readFromStream.ts new file mode 100755 index 0000000..abef2d3 --- /dev/null +++ b/ts_test/reader/readFromStream.ts @@ -0,0 +1,27 @@ +#!/usr/bin/env node --require ts-node/register +import { test } from 'tap' +import readFromStream from '../../ts/reader/readFromStream' +import Stream from 'ts-stream' +import readerForFeatures from '../../ts/reader/readerForFeatures' +import FeatureType from '../../ts/api/FeatureType' + +function c (char: string): number { + return char.charCodeAt(0) +} + +test('reading simple stream of features', async t => { + const points = await readFromStream(Stream.from([ + new Uint8Array([1, 2, c('a'), c('b')]), + new Uint8Array([c('c'), 3]), + new Uint8Array([4, c('d'), c('e'), c('f')]) + ]), readerForFeatures([ + { name: 'x', type: FeatureType.uint8 }, + { name: 'y', type: FeatureType.uint8 }, + { name: 'desc', type: FeatureType.fixedstring, length: 3 } + ])).toArray() + + t.deepEquals(points, [ + { x: 1, y: 2, desc: 'abc' }, + { x: 3, y: 4, desc: 'def' } + ]) +}) From b54c9808a4057b515e03d6d317f4f61dab9d875a Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Mon, 13 Aug 2018 23:23:21 +0200 Subject: [PATCH 10/23] Since minSize already contains the data, this commit drops the size property (redundant) --- ts/reader/IReader.ts | 2 -- ts/reader/readFromStream.ts | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ts/reader/IReader.ts b/ts/reader/IReader.ts index 24e932b..8f9c016 100644 --- a/ts/reader/IReader.ts +++ b/ts/reader/IReader.ts @@ -2,7 +2,6 @@ import IDynamicResult from './IDynamicResult' export default interface IReader { fixedSize: boolean - size?: number minSize: number readDynamic (view: DataView, byteOffset): IDynamicResult read (view: DataView, byteOffset: number): any @@ -12,7 +11,6 @@ export function createFixedReader (size: number, read: (view: DataView, byteOffs return { fixedSize: true, minSize: size, - size, readDynamic: (view: DataView, byteOffset: number): IDynamicResult => ({ size, byteOffset: byteOffset + size, diff --git a/ts/reader/readFromStream.ts b/ts/reader/readFromStream.ts index da4e7f4..4304076 100644 --- a/ts/reader/readFromStream.ts +++ b/ts/reader/readFromStream.ts @@ -12,7 +12,7 @@ function combine (a: Uint8Array, b: Uint8Array) { function readFixedSize (out: Stream, inStream: ReadableStream, reader: IReader) { let leftOver = null return inStream.forEach((data: Uint8Array) => { - let end = reader.size + let end = reader.minSize let start = 0 if (leftOver !== null) { data = combine(leftOver, data) @@ -24,7 +24,7 @@ function readFixedSize (out: Stream, inStream: ReadableStream, } entries.push(reader.read(new DataView(data.buffer), start)) start = end - end += reader.size + end += reader.minSize } if (end === data.length) { leftOver = null From c933c0f12744f19429be623e4ae427a7ba946b14 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Mon, 13 Aug 2018 23:24:37 +0200 Subject: [PATCH 11/23] Missing minSize --- ts/reader/readerForFeatures.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ts/reader/readerForFeatures.ts b/ts/reader/readerForFeatures.ts index 6952731..f8acabf 100644 --- a/ts/reader/readerForFeatures.ts +++ b/ts/reader/readerForFeatures.ts @@ -36,7 +36,7 @@ function readerForFixedFeatures (features: IFeature[], readers: IReader[]): IRea const reader = readers[index] const offset = size const target = feature.name - size += reader.size + size += reader.minSize return (view: DataView, byteOffset: number, result: { [k: string]: any }) => { result[target] = reader.read(view, byteOffset + offset) } From 69c02cf77a3d95bb51352ed997a991df761b4e71 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Mon, 13 Aug 2018 23:25:49 +0200 Subject: [PATCH 12/23] Implemented and tested the dynamic reading of data. --- ts/reader/IDynamicContext.ts | 5 + ts/reader/IDynamicResult.ts | 5 - ts/reader/IReader.ts | 31 ++++-- ts/reader/readFromStream.ts | 39 ++++++- ts/reader/readerForFeatureType.ts | 168 ++++++++++++++++++------------ ts/reader/readerForFeatures.ts | 52 +++++---- ts_test/reader/readFromStream.ts | 30 ++++++ 7 files changed, 226 insertions(+), 104 deletions(-) create mode 100644 ts/reader/IDynamicContext.ts delete mode 100644 ts/reader/IDynamicResult.ts diff --git a/ts/reader/IDynamicContext.ts b/ts/reader/IDynamicContext.ts new file mode 100644 index 0000000..34ad39e --- /dev/null +++ b/ts/reader/IDynamicContext.ts @@ -0,0 +1,5 @@ +export default interface IDynamicContext { + data: any // previous data before read, current data after read + size: number // size of the previous data before read, size of the current data after read + byteOffset: number // start byte offset before read, end byte offset after read +} diff --git a/ts/reader/IDynamicResult.ts b/ts/reader/IDynamicResult.ts deleted file mode 100644 index beb68bb..0000000 --- a/ts/reader/IDynamicResult.ts +++ /dev/null @@ -1,5 +0,0 @@ -export default interface IDynamicResult { - size: number - data: any - byteOffset: number -} diff --git a/ts/reader/IReader.ts b/ts/reader/IReader.ts index 8f9c016..c92a319 100644 --- a/ts/reader/IReader.ts +++ b/ts/reader/IReader.ts @@ -1,9 +1,9 @@ -import IDynamicResult from './IDynamicResult' +import IDynamicContext from './IDynamicContext' export default interface IReader { fixedSize: boolean minSize: number - readDynamic (view: DataView, byteOffset): IDynamicResult + readDynamic (view: DataView, context: IDynamicContext): boolean read (view: DataView, byteOffset: number): any } @@ -11,20 +11,33 @@ export function createFixedReader (size: number, read: (view: DataView, byteOffs return { fixedSize: true, minSize: size, - readDynamic: (view: DataView, byteOffset: number): IDynamicResult => ({ - size, - byteOffset: byteOffset + size, - data: read(view, byteOffset) - }), + readDynamic: (view: DataView, context: IDynamicContext) => { + const offset = context.byteOffset + context.data = read(view, offset) + context.byteOffset = offset + size + context.size = size + return true + }, read } } -export function createDynamicReader (minSize: number, readDynamic: (view: DataView, byteOffset: number) => IDynamicResult): IReader { +const helperContext: IDynamicContext = { + data: null, + byteOffset: 0, + size: 0 +} + +export function createDynamicReader (minSize: number, readDynamic: (view: DataView, context: IDynamicContext) => boolean): IReader { return { fixedSize: false, readDynamic, minSize, - read: (view: DataView, byteOffset: number) => readDynamic(view, byteOffset).data + read: (view: DataView, byteOffset: number) => { + helperContext.byteOffset = byteOffset + helperContext.data = undefined + readDynamic(view, helperContext) + return helperContext.data + } } } diff --git a/ts/reader/readFromStream.ts b/ts/reader/readFromStream.ts index 4304076..a9156c6 100644 --- a/ts/reader/readFromStream.ts +++ b/ts/reader/readFromStream.ts @@ -1,6 +1,7 @@ import Stream, { ReadableStream } from 'ts-stream' import IReader from './IReader' import { mapSeries } from 'bluebird' +import IDynamicContext from './IDynamicContext' function combine (a: Uint8Array, b: Uint8Array) { const combined = new Uint8Array(a.length + b.length) @@ -38,8 +39,44 @@ function readFixedSize (out: Stream, inStream: ReadableStream, }).then(() => leftOver) } +const workContext: IDynamicContext = { + byteOffset: 0, + size: 0, + data: null +} + function readDynamicSize (out: Stream, inStream: ReadableStream, reader: IReader) { - return Promise.reject(new Error('TODO: Implement')) + let leftOver = null + const context = { + byteOffset: 0 + } + return inStream.forEach((data: Uint8Array) => { + let end = reader.minSize + workContext.byteOffset = 0 + if (leftOver !== null) { + data = combine(leftOver, data) + } + let entries: any[] + while (end <= data.length) { + if (entries === undefined) { + entries = [] + } + if (!reader.readDynamic(new DataView(data.buffer), workContext)) { + break + } + entries.push(workContext.data) + end = workContext.byteOffset + workContext.size + reader.minSize + } + if (end === data.length) { + leftOver = null + } else { + leftOver = data.subarray(workContext.byteOffset) + } + if (entries === undefined) { + return + } + return mapSeries(entries, entry => out.write(entry)) + }).then(() => leftOver) } export default function readFromStream (inStream: ReadableStream, reader: IReader): ReadableStream { diff --git a/ts/reader/readerForFeatureType.ts b/ts/reader/readerForFeatureType.ts index 6fdb84e..593bbbf 100644 --- a/ts/reader/readerForFeatureType.ts +++ b/ts/reader/readerForFeatureType.ts @@ -1,7 +1,7 @@ import IReader, { createFixedReader, createDynamicReader } from './IReader' import FeatureType from '../api/FeatureType' import Long from 'long' -import IDynamicResult from './IDynamicResult' +import IDynamicContext from './IDynamicContext' import decodeUtf8 from 'decode-utf8' function zzDecodeLong (long: Long) { @@ -12,10 +12,9 @@ function zzDecodeLong (long: Long) { return long } -function zzDecode (result: IDynamicResult): IDynamicResult { +function zzDecode (input: number) { // Adapted from https://github.com/dcodeIO/protobuf.js/blob/69623a91c1e4a99d5210b5295a9e5b39d9517554/src/reader.js#L113-L114 - result.data = result.data >>> 1 ^ -(result.data & 1) | 0 - return result + return input >>> 1 ^ -(input & 1) | 0 } const boolReader: IReader = createFixedReader(1, (view: DataView, byteOffset: number) => view.getInt8(byteOffset) === 0) @@ -32,70 +31,106 @@ const int64Reader: IReader = createFixedReader(8, (view: DataView, byteOffset: n const uint64Reader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => new Long(view.getInt32(byteOffset), view.getInt32(byteOffset), true)) const sint64Reader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => zzDecodeLong(new Long(view.getInt32(byteOffset), view.getInt32(byteOffset), false))) -function readVarInt (view: DataView, byteOffset: number) { +function readVarInt (view: DataView, context: IDynamicContext) { // Adapted from https://github.com/dcodeIO/protobuf.js/blob/69623a91c1e4a99d5210b5295a9e5b39d9517554/src/reader.js#L85-L89 + let byteOffset = context.byteOffset const a = view.getUint8(byteOffset++) let data = (a & 127) >>> 0 - if (a < 128) return { size: 1, data, byteOffset, next: 0 } - const b = view.getUint8(byteOffset++) - data = data | ((b && 127) << 7) >>> 0 - if (b < 128) return { size: 2, data, byteOffset, next: 0 } - const c = view.getUint8(byteOffset++) - data = data | ((c && 127) << 14) >>> 0 - if (c < 128) return { size: 3, data, byteOffset, next: 0 } - const d = view.getUint8(byteOffset++) - data = data | ((d && 127) << 21) >>> 0 - if (d < 128) return { size: 4, data, byteOffset, next: 0 } - const e = view.getUint8(byteOffset++) - data = data | ((e && 127) << 28) >>> 0 - return { size: 5, data, byteOffset, next: e & 128 } + let size = a < 128 ? 1 : 2 + const len = view.byteLength + if (size === 2) { + if (byteOffset + 1 === len) return false + const b = view.getUint8(byteOffset++) + data = data | ((b && 127) << 7) >>> 0 + if (b >= 128) size = 3 + } + if (size === 3) { + if (byteOffset + 2 === len) return false + const c = view.getUint8(byteOffset++) + data = data | ((c && 127) << 14) >>> 0 + if (c >= 128) size = 4 + } + if (size === 4) { + if (byteOffset + 3 === len) return false + const d = view.getUint8(byteOffset++) + data = data | ((d && 127) << 21) >>> 0 + if (d >= 128) size = 5 + } + if (size === 5) { + if (byteOffset + 4 === len) return false + const e = view.getUint8(byteOffset++) + data = data | ((e && 127) << 28) >>> 0 + if (e >= 128) size = 6 + } + // Only set this after it is certain that there is a return value + context.byteOffset = byteOffset + context.data = data + context.size = size + return true } -function readVarUint32 (view: DataView, byteOffset: number): IDynamicResult { - const result = readVarInt(view, byteOffset) - if (result.next !== 0) { - throw new Error('Invalid var-int 32 encoding!') +function readVarUint32 (view: DataView, context: IDynamicContext) { + if (!readVarInt(view, context)) { + return false } - return result + if (context.size === 6) { + throw new Error('Invalid int-32 encoding!') + } + return true } -const varInt32Reader: IReader = createDynamicReader(1, (view: DataView, byteOffset: number): IDynamicResult => { - const result = readVarUint32(view, byteOffset) - result.data = result.data | 0 - return result +const varInt32Reader: IReader = createDynamicReader(1, (view: DataView, context: IDynamicContext) => { + if (!readVarUint32(view, context)) { + return false + } + context.data = context.data | 0 + return true }) const varUint32Reader: IReader = createDynamicReader(1, readVarUint32) -const varSint32Reader: IReader = createDynamicReader(1, (view: DataView, byteOffset: number): IDynamicResult => zzDecode(readVarUint32(view, byteOffset))) +const varSint32Reader: IReader = createDynamicReader(1, (view: DataView, context: IDynamicContext) => { + if (!readVarUint32(view, context)) { + return false + } + context.data = zzDecode(context.data) + return true +}) -function readVarInt64 (unsigned: boolean, view: DataView, byteOffset: number): IDynamicResult { - const low = readVarInt(view, byteOffset) - if (low.next === 0) { - if (!unsigned) { - low.data = low.data | 0 - } - return low - } - const high = readVarInt(view, low.byteOffset) - if (high.next !== 0) { - throw new Error('Invalid var-int 64 encoding!') +function readVarInt64 (unsigned: boolean, view: DataView, context: IDynamicContext) { + const prevByteOffset = context.byteOffset + if (!readVarInt(view, context)) { + return false } - return { - size: low.size + high.size, - byteOffset: high.byteOffset, - data: new Long(low.data, high.data, unsigned) + const low: number = context.data + if (context.size !== 6) { + if (!unsigned) context.data = low | 0 + return true } + if (!readVarInt(view, context)) { + // the context was written, but still, lets reduce the byteOffset again + context.byteOffset = prevByteOffset + return false + } + if (context.size === 6) { + throw new Error('Invalid var-int 64 encoding!') + } + context.data = new Long(low, context.data, unsigned) + context.size = context.size + 5 + return true } const varInt64Reader: IReader = createDynamicReader(1, readVarInt64.bind(null, false)) const varUint64Reader: IReader = createDynamicReader(1, readVarInt64.bind(null, true)) -const varSint64Reader: IReader = createDynamicReader(1, (view: DataView, byteOffset: number): IDynamicResult => { - const result = readVarInt64(false, view, byteOffset) - if (typeof result.data === 'number') { - return zzDecode(result) +const varSint64Reader: IReader = createDynamicReader(1, (view: DataView, context: IDynamicContext) => { + if (!readVarInt64(false, view, context)) { + return false + } + if (typeof context.data === 'number') { + context.data = zzDecode(context.data) + } else { + context.data = zzDecodeLong(context.data) } - result.data = zzDecodeLong(result.data) - return result + return true }) function fixedStringReader (length: number): IReader { @@ -113,30 +148,27 @@ function fixedBytesReader (length: number): IReader { return createFixedReader(length, (view: DataView, byteOffset: number) => view.buffer.slice(byteOffset, byteOffset + length)) } -function readBytes (view: DataView, byteOffset: number) { - const size = int32Reader.read(view, byteOffset) - const start = byteOffset - byteOffset += size - if (view.buffer.byteLength < byteOffset) { - return null - } - return { - size, - byteOffset, - data: view.buffer.slice(start, byteOffset) +function readBytes (view: DataView, context: IDynamicContext) { + const size = view.getUint32(context.byteOffset) + const start = view.byteOffset + context.byteOffset + uint32Reader.minSize + const end = start + size + if (end >= view.buffer.byteLength) { + return false } + context.size = size + uint32Reader.minSize + context.byteOffset = end + context.data = view.buffer.slice(start, end) + return true } const bytesReader = createDynamicReader(2, readBytes) -const stringReader = createDynamicReader(2, (view: DataView, byteOffset: number) => { - const result = bytesReader.readDynamic(view, byteOffset) - if (result === null) { - return null - } - const buffer: ArrayBuffer = result.data - result.data = decodeUtf8(buffer) - return result +const stringReader = createDynamicReader(2, (view: DataView, context: IDynamicContext) => { + if (!readBytes(view, context)) { + return false + } + context.data = decodeUtf8(context.data) + return true }) export default function readerForFeatureType (type: FeatureType, length?: number): IReader { diff --git a/ts/reader/readerForFeatures.ts b/ts/reader/readerForFeatures.ts index f8acabf..8d56abc 100644 --- a/ts/reader/readerForFeatures.ts +++ b/ts/reader/readerForFeatures.ts @@ -1,32 +1,42 @@ import IFeature from '../api/IFeature' import IReader, { createDynamicReader, createFixedReader } from './IReader' import readerForFeatureType from './readerForFeatureType' -import IDynamicResult from './IDynamicResult' - +import IDynamicContext from './IDynamicContext' +const workContext: IDynamicContext = { + byteOffset: 0, + data: null, + size: 0 +} function readerForDynamicFeatures (features: IFeature[], readers: IReader[]): IReader { const minSize = readers.reduce((minSize: number, reader: IReader) => minSize + reader.minSize, 0) - return createDynamicReader(minSize, (view: DataView, byteOffset: number): IDynamicResult => { - const result: IDynamicResult = { - size: 0, - byteOffset, - data: {} - } - readers.forEach((reader, index): void => { - const feature = features[index] - // TODO: Implement a size check + return createDynamicReader(minSize, (view: DataView, context: IDynamicContext): boolean => { + const data = {} + let size = 0 + let index = 0 + // Some of the sub-readers could finish successfully and thus change the byteOffset for + // Operation this would break the IDynamicContext contract because it wouldn't have actually + // finished its work. To prevent this the workContext is used to check and later passed + // on the to the actual context + workContext.byteOffset = context.byteOffset + for (const reader of readers) { + const feature = features[index++] if (reader.fixedSize) { - const featureResult = reader.read(view, byteOffset) - result.size += reader.size - result.data[feature.name] = featureResult.data - byteOffset += reader.size + data[feature.name] = reader.read(view, workContext.byteOffset) + workContext.byteOffset += reader.minSize + size += reader.minSize } else { - const featureResult = reader.readDynamic(view, byteOffset) - result.size += featureResult.size - result.data[feature.name] = featureResult.data - byteOffset = featureResult.byteOffset + if (!reader.readDynamic(view, workContext)) { + return false + } + data[feature.name] = workContext.data + size += workContext.size } - }) - return result + } + // Reapply the new sub + context.byteOffset = workContext.byteOffset + context.size = size + context.data = data + return true }) } diff --git a/ts_test/reader/readFromStream.ts b/ts_test/reader/readFromStream.ts index abef2d3..2399429 100755 --- a/ts_test/reader/readFromStream.ts +++ b/ts_test/reader/readFromStream.ts @@ -25,3 +25,33 @@ test('reading simple stream of features', async t => { { x: 3, y: 4, desc: 'def' } ]) }) + +test('reading dynamic simple stream of features', async t => { + const data = [ + 1, + 2, + 0, 0, 0, 3, + c('a'), c('b'), c('c'), + 3, + 5, + 6, + 0, 0, 0, 2, + c('d'), c('e'), + 7 + ] + const points = await readFromStream(Stream.from([ + new Uint8Array(data.slice(0, 7)), + new Uint8Array(data.slice(7, 14)), + new Uint8Array(data.slice(14)) + ]), readerForFeatures([ + { name: 'x', type: FeatureType.uint8 }, + { name: 'y', type: FeatureType.uint8 }, + { name: 'desc', type: FeatureType.string }, + { name: 'z', type: FeatureType.uint8 } + ])).toArray() + + t.deepEquals(points, [ + { x: 1, y: 2, z: 3, desc: 'abc' }, + { x: 5, y: 6, z: 7, desc: 'de' } + ]) +}) From 39a0399e3ca65bf635726843bbaf160d20e6f1fa Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Mon, 13 Aug 2018 23:28:21 +0200 Subject: [PATCH 13/23] Adds a varbytes and varstring to string/byte that allows for var-int32-size headers. The protocol buffer string would be in a varbytes/varstring while a regular string is a "string". --- spec/FeatureType.proto | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/spec/FeatureType.proto b/spec/FeatureType.proto index c5cbbcd..9601b8d 100644 --- a/spec/FeatureType.proto +++ b/spec/FeatureType.proto @@ -17,9 +17,11 @@ enum FeatureType { varsint64 = 15; // variable size 2~9 byte, reverse stored number: -9223372036854776000~9223372036854776000 bytes = 16; // variable size bytes: uint32-size-number + amount of bytes string = 17; // variable size string: utf-8 encoded bytes - bool = 18; // 0 = false, 1~7 = true - float = 19; // fixed size 4 byte floating point - double = 20; // fixed size 8 byte floating point - fixedstring = 21[(length) = -1]; // fixed amount of string - fixedbytes = 22[(length) = -1]; // fixed amount of bytes + varbytes = 19; // variable size bytes with varsize length: varuint32-number + amount of bytes + varstring = 20; // variable size string with varsize length: utf-8 encoded varbytes + bool = 21; // 0 = false, 1~7 = true + float = 22; // fixed size 4 byte floating point + double = 23; // fixed size 8 byte floating point + fixedstring = 24[(length) = -1]; // fixed amount of string + fixedbytes = 25[(length) = -1]; // fixed amount of bytes } From c483979adefa33e28dcbfa9c24591485b5f3c345 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Fri, 17 Aug 2018 21:47:39 +0200 Subject: [PATCH 14/23] Added LE encoding in addition to the default BE encoding. --- spec/FeatureType.proto | 55 ++++++++++++++++++------------- ts/api/FeatureType.ts | 32 +++++++++++------- ts/reader/readerForFeatureType.ts | 28 ++++++++++++++++ 3 files changed, 81 insertions(+), 34 deletions(-) diff --git a/spec/FeatureType.proto b/spec/FeatureType.proto index 9601b8d..24ee321 100644 --- a/spec/FeatureType.proto +++ b/spec/FeatureType.proto @@ -2,26 +2,37 @@ enum FeatureType { int8 = 1; // fixed size 1 byte number: -127~128 uint8 = 2; // fixed size 1 byte number: 0~255 int16 = 3; // fixed size 2 byte number: -32768~32768 - uint16 = 4; // fixed size 2 byte number: 0~255 + (0~255)<<16 => 0~65536 - int32 = 4; // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 -2147483648 => 2147483648~-2147483648 - uint32 = 5; // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 => 0~4294967296 - sint32 = 6; // fixed size 4 byte, reverse stored number: -2147483648~2147483648 - varint32 = 7; // variable size 2~5 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15) => 2147483648~-2147483648 - varuint32 = 8; // variable size 2~5 byte, reverse stored number: 0~4294967296 - varsint32 = 9; // variable size 2~5 byte, reverse stored number: -2147483648~2147483648 - int64 = 10; // fixed size 8 byte number: -9223372036854776000~9223372036854776000 - uint64 = 11; // fixed size 8 byte number: 0~18446744073709552000 - sint64 = 12; // fixed size 8 byte, reverse stored number: 9223372036854776000~-9223372036854776000 - varuint64 = 14; // variable size 2~9 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15) => 9223372036854776000~-9223372036854776000 - varint64 = 13; // variable size 2~9 byte number: 0~18446744073709552000 - varsint64 = 15; // variable size 2~9 byte, reverse stored number: -9223372036854776000~9223372036854776000 - bytes = 16; // variable size bytes: uint32-size-number + amount of bytes - string = 17; // variable size string: utf-8 encoded bytes - varbytes = 19; // variable size bytes with varsize length: varuint32-number + amount of bytes - varstring = 20; // variable size string with varsize length: utf-8 encoded varbytes - bool = 21; // 0 = false, 1~7 = true - float = 22; // fixed size 4 byte floating point - double = 23; // fixed size 8 byte floating point - fixedstring = 24[(length) = -1]; // fixed amount of string - fixedbytes = 25[(length) = -1]; // fixed amount of bytes + int16LE = 4; // fixed size 2 byte number: -32768~32768 + uint16 = 5; // fixed size 2 byte number: 0~65536 + uint16LE = 6; // fixed size 2 byte number: 0~255 + (0~255)<<16 => 0~665536 + int32 = 7; // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 -2147483648 => 2147483648~-2147483648 + int32LE = 8; // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 -2147483648 => 2147483648~-2147483648 + uint32 = 9; // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 => 0~4294967296 + uint32LE = 10; // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 => 0~4294967296 + sint32 = 11; // fixed size 4 byte, reverse stored number: -2147483648~2147483648 + sint32LE = 12; // fixed size 4 byte, reverse stored number: -2147483648~2147483648 + varint32 = 13; // variable size 2~5 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15) => 2147483648~-2147483648 + varuint32 = 14; // variable size 2~5 byte, reverse stored number: 0~4294967296 + varsint32 = 15; // variable size 2~5 byte, reverse stored number: -2147483648~2147483648 + int64 = 16; // fixed size 8 byte number: -9223372036854776000~9223372036854776000 + int64LE = 17; // fixed size 8 byte number: -9223372036854776000~9223372036854776000 + uint64 = 18; // fixed size 8 byte number: 0~18446744073709552000 + uint64LE = 19; // fixed size 8 byte number: 0~18446744073709552000 + sint64 = 20; // fixed size 8 byte, reverse stored number: 9223372036854776000~-9223372036854776000 + sint64LE = 21; // fixed size 8 byte, reverse stored number: 9223372036854776000~-9223372036854776000 + varuint64 = 22; // variable size 2~9 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15) => 9223372036854776000~-9223372036854776000 + varint64 = 23; // variable size 2~9 byte number: 0~18446744073709552000 + varsint64 = 24; // variable size 2~9 byte, reverse stored number: -9223372036854776000~9223372036854776000 + bytes = 25; // variable size bytes: uint32-size-number + amount of bytes + bytesLE = 26; // variable size bytes: uint32-size-number + amount of bytes + varbytes = 27; // variable size bytes with varsize length: varuint32-number + amount of bytes + varbytes = 28; // variable size bytes with varsize length: varuint32-number + amount of bytes + string = 29; // variable size string: utf-8 encoded bytes + stringLE = 30; // variable size string: utf-8 encoded bytes + varstring = 31; // variable size string with varsize length: utf-8 encoded varbytes + bool = 32; // 0 = false, 1~7 = true + float = 33; // fixed size 4 byte floating point + double = 34; // fixed size 8 byte floating point + fixedstring = 35[(length) = -1]; // fixed amount of string + fixedbytes = 36[(length) = -1]; // fixed amount of bytes } diff --git a/ts/api/FeatureType.ts b/ts/api/FeatureType.ts index f374128..e79cacf 100644 --- a/ts/api/FeatureType.ts +++ b/ts/api/FeatureType.ts @@ -3,18 +3,26 @@ enum FeatureType { uint8 = 2, // fixed size 1 byte number: 0~255 int16 = 3, // fixed size 2 byte number: -32768~32768 - uint16 = 4, // fixed size 2 byte number: 0~255 + (0~255)<<16 => 0~65536 - - int32 = 4, // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 -2147483648 => 2147483648~-2147483648 - uint32 = 5, // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 => 0~4294967296 - sint32 = 6, // fixed size 4 byte, reverse stored number: -2147483648~2147483648 - varint32 = 7, // variable size 1~5 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15) => 2147483648~-2147483648 - varuint32 = 8, // variable size 1~5 byte number: (same like varint) 0~4294967296 - varsint32 = 9, // variable size 1~5 byte, reverse stored number: -2147483648~2147483648 - - int64 = 10, // fixed size 8 byte number: -9223372036854776000~9223372036854776000 - uint64 = 11, // fixed size 8 byte number: 0~18446744073709552000 - sint64 = 12, // fixed size 8 byte, reverse stored number: 9223372036854776000~-9223372036854776000 + int16LE = 5, // fixed size 2 byte number: -32768~32768 (little endian) + uint16 = 5, // fixed size 2 byte number: 0~255 + (0~255)<<16 => 0~65536 + uint16LE = 6, // fixed size 2 byte number: 0~65536 (little endian) + + int32 = 7, // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 -2147483648 => 2147483648~-2147483648 + int32LE = 8, // fixed size 4 byte number: 2147483648~-2147483648 (little endian) + uint32 = 9, // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 => 0~4294967296 + uint32LE = 10, // fixed size 4 byte number: 0~4294967296 (little endian) + sint32 = 11, // fixed size 4 byte, reverse stored number: -2147483648~2147483648 + sint32LE = 12, // fixed size 4 byte, reverse stored number: -2147483648~2147483648 (little endian) + varint32 = 13, // variable size 1~5 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15) => 2147483648~-2147483648 + varuint32 = 14, // variable size 1~5 byte number: (same like varint) 0~4294967296 + varsint32 = 15, // variable size 1~5 byte, reverse stored number: -2147483648~2147483648 + + int64 = 16, // fixed size 8 byte number: -9223372036854776000~9223372036854776000 + int64LE = 17, // fixed size 8 byte number: -9223372036854776000~9223372036854776000 (little endian) + uint64 = 18, // fixed size 8 byte number: 0~18446744073709552000 + uint64LE = 19, // fixed size 8 byte number: 0~18446744073709552000 (little endian) + sint64 = 20, // fixed size 8 byte, reverse stored number: 9223372036854776000~-9223372036854776000 + sint64LE = 21, // fixed size 8 byte, reverse stored number: 9223372036854776000~-9223372036854776000 (little endian) varint64 = 13, // variable size 1~9 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15)... => 9223372036854776000~-9223372036854776000 varuint64 = 14, // variable size 1~9 byte number: (same like varint) 0~18446744073709552000 varsint64 = 15, // variable size 1~9 byte, reverse stored number: -9223372036854776000~9223372036854776000 diff --git a/ts/reader/readerForFeatureType.ts b/ts/reader/readerForFeatureType.ts index 593bbbf..fde250a 100644 --- a/ts/reader/readerForFeatureType.ts +++ b/ts/reader/readerForFeatureType.ts @@ -23,13 +23,33 @@ const floatReader: IReader = createFixedReader(4, (view: DataView, byteOffset: n const int8Reader: IReader = createFixedReader(1, (view: DataView, byteOffset: number) => view.getInt8(byteOffset)) const uint8Reader: IReader = createFixedReader(1, (view: DataView, byteOffset: number) => view.getUint8(byteOffset)) const int16Reader: IReader = createFixedReader(2, (view: DataView, byteOffset: number) => view.getInt16(byteOffset)) +const int16LEReader: IReader = createFixedReader(2, (view: DataView, byteOffset: number) => view.getInt16(byteOffset, true)) const uint16Reader: IReader = createFixedReader(2, (view: DataView, byteOffset: number) => view.getUint16(byteOffset)) +const uint16LEReader: IReader = createFixedReader(2, (view: DataView, byteOffset: number) => view.getUint16(byteOffset, true)) const int32Reader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getInt32(byteOffset)) +const int32LEReader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getInt32(byteOffset, true)) const uint32Reader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getUint32(byteOffset)) +const uint32LEReader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getUint32(byteOffset, true)) const sint32Reader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getInt32(byteOffset) | 0) +const sint32LEReader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getInt32(byteOffset, true) | 0) const int64Reader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => new Long(view.getInt32(byteOffset), view.getInt32(byteOffset), false)) +const int64LEReader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => { + const high = view.getInt32(byteOffset, true) + const low = view.getInt32(byteOffset, true) + return new Long(low, high, false) +}) const uint64Reader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => new Long(view.getInt32(byteOffset), view.getInt32(byteOffset), true)) +const uint64LEReader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => { + const high = view.getInt32(byteOffset, true) + const low = view.getInt32(byteOffset, true) + return new Long(low, high, true) +}) const sint64Reader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => zzDecodeLong(new Long(view.getInt32(byteOffset), view.getInt32(byteOffset), false))) +const sint64LEReader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => { + const high = view.getInt32(byteOffset, true) + const low = view.getInt32(byteOffset, true) + return zzDecodeLong(new Long(low, high, false)) +}) function readVarInt (view: DataView, context: IDynamicContext) { // Adapted from https://github.com/dcodeIO/protobuf.js/blob/69623a91c1e4a99d5210b5295a9e5b39d9517554/src/reader.js#L85-L89 @@ -183,16 +203,24 @@ export default function readerForFeatureType (type: FeatureType, length?: number case FeatureType.int8: return int8Reader case FeatureType.uint8: return uint8Reader case FeatureType.int16: return int16Reader + case FeatureType.int16LE: return int16LEReader case FeatureType.uint16: return uint16Reader + case FeatureType.uint16LE: return uint16LEReader case FeatureType.int32: return int32Reader + case FeatureType.int32LE: return int32LEReader case FeatureType.uint32: return uint32Reader + case FeatureType.uint32LE: return uint32LEReader case FeatureType.sint32: return sint32Reader + case FeatureType.sint32LE: return sint32LEReader case FeatureType.varint32: return varInt32Reader case FeatureType.varuint32: return varUint32Reader case FeatureType.varsint32: return varSint32Reader case FeatureType.int64: return int64Reader + case FeatureType.int64LE: return int64LEReader case FeatureType.uint64: return uint64Reader + case FeatureType.uint64LE: return uint64LEReader case FeatureType.sint64: return sint64Reader + case FeatureType.sint64LE: return sint64LEReader case FeatureType.varint64: return varInt64Reader case FeatureType.varuint64: return varUint64Reader case FeatureType.varsint64: return varSint64Reader From ca8b7eac9a62501204ff9e56200cc8ad92411405 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Fri, 17 Aug 2018 22:13:47 +0200 Subject: [PATCH 15/23] Added var string/bytes --- spec/FeatureType.proto | 19 +++++++++--------- ts/api/FeatureType.ts | 26 +++++++++++++----------- ts/reader/readerForFeatureType.ts | 33 +++++++++++++++++++++++++------ 3 files changed, 51 insertions(+), 27 deletions(-) diff --git a/spec/FeatureType.proto b/spec/FeatureType.proto index 24ee321..fb15662 100644 --- a/spec/FeatureType.proto +++ b/spec/FeatureType.proto @@ -24,15 +24,14 @@ enum FeatureType { varint64 = 23; // variable size 2~9 byte number: 0~18446744073709552000 varsint64 = 24; // variable size 2~9 byte, reverse stored number: -9223372036854776000~9223372036854776000 bytes = 25; // variable size bytes: uint32-size-number + amount of bytes - bytesLE = 26; // variable size bytes: uint32-size-number + amount of bytes + bytesLE = 26; // variable size bytes: uint32LE-size-number + amount of bytes varbytes = 27; // variable size bytes with varsize length: varuint32-number + amount of bytes - varbytes = 28; // variable size bytes with varsize length: varuint32-number + amount of bytes - string = 29; // variable size string: utf-8 encoded bytes - stringLE = 30; // variable size string: utf-8 encoded bytes - varstring = 31; // variable size string with varsize length: utf-8 encoded varbytes - bool = 32; // 0 = false, 1~7 = true - float = 33; // fixed size 4 byte floating point - double = 34; // fixed size 8 byte floating point - fixedstring = 35[(length) = -1]; // fixed amount of string - fixedbytes = 36[(length) = -1]; // fixed amount of bytes + string = 28; // variable size string: utf-8 encoded bytes + stringLE = 29; // variable size string: utf-8 encoded bytes + varstring = 30; // variable size string with varsize length: utf-8 encoded varbytes + bool = 31; // 0 = false, 1~7 = true + float = 32; // fixed size 4 byte floating point + double = 33; // fixed size 8 byte floating point + fixedstring = 34[(length) = -1]; // fixed amount of string + fixedbytes = 35[(length) = -1]; // fixed amount of bytes } diff --git a/ts/api/FeatureType.ts b/ts/api/FeatureType.ts index e79cacf..876ba2e 100644 --- a/ts/api/FeatureType.ts +++ b/ts/api/FeatureType.ts @@ -3,7 +3,7 @@ enum FeatureType { uint8 = 2, // fixed size 1 byte number: 0~255 int16 = 3, // fixed size 2 byte number: -32768~32768 - int16LE = 5, // fixed size 2 byte number: -32768~32768 (little endian) + int16LE = 4, // fixed size 2 byte number: -32768~32768 (little endian) uint16 = 5, // fixed size 2 byte number: 0~255 + (0~255)<<16 => 0~65536 uint16LE = 6, // fixed size 2 byte number: 0~65536 (little endian) @@ -23,20 +23,24 @@ enum FeatureType { uint64LE = 19, // fixed size 8 byte number: 0~18446744073709552000 (little endian) sint64 = 20, // fixed size 8 byte, reverse stored number: 9223372036854776000~-9223372036854776000 sint64LE = 21, // fixed size 8 byte, reverse stored number: 9223372036854776000~-9223372036854776000 (little endian) - varint64 = 13, // variable size 1~9 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15)... => 9223372036854776000~-9223372036854776000 - varuint64 = 14, // variable size 1~9 byte number: (same like varint) 0~18446744073709552000 - varsint64 = 15, // variable size 1~9 byte, reverse stored number: -9223372036854776000~9223372036854776000 + varint64 = 22, // variable size 1~9 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15)... => 9223372036854776000~-9223372036854776000 + varuint64 = 23, // variable size 1~9 byte number: (same like varint) 0~18446744073709552000 + varsint64 = 24, // variable size 1~9 byte, reverse stored number: -9223372036854776000~9223372036854776000 - bytes = 16, // variable size bytes: uint32-size-number + amount of bytes - string = 17, // variable size string: utf-8 encoded bytes + bytes = 25, // variable size bytes: uint32-size-number + amount of bytes + bytesLE = 26, // variable size bytes: uint32LE-size-number + amount of bytes + varbytes = 27, // variable size bytes: varint32-size-number + amount of bytes + string = 28, // variable size string: utf-8 encoded bytes + stringLE = 29, // variable size string: utf-8 encoded bytesLE + varstring = 30, // variable size string: utf-8 encoded varbytes - bool = 18, // 0 = false, 1~7 = true + bool = 31, // 0 = false, 1~7 = true - float = 19, // fixed size 4 byte floating point - double = 21, // fixed size 8 byte floating point + float = 32, // fixed size 4 byte floating point + double = 33, // fixed size 8 byte floating point - fixedstring = 22, // fixed amount of string - fixedbytes = 23 // fixed amount of bytes + fixedstring = 34, // fixed amount of string + fixedbytes = 35 // fixed amount of bytes } export default FeatureType diff --git a/ts/reader/readerForFeatureType.ts b/ts/reader/readerForFeatureType.ts index fde250a..49e7641 100644 --- a/ts/reader/readerForFeatureType.ts +++ b/ts/reader/readerForFeatureType.ts @@ -168,8 +168,7 @@ function fixedBytesReader (length: number): IReader { return createFixedReader(length, (view: DataView, byteOffset: number) => view.buffer.slice(byteOffset, byteOffset + length)) } -function readBytes (view: DataView, context: IDynamicContext) { - const size = view.getUint32(context.byteOffset) +function readBytesBase (size: number, view: DataView, context: IDynamicContext) { const start = view.byteOffset + context.byteOffset + uint32Reader.minSize const end = start + size if (end >= view.buffer.byteLength) { @@ -181,15 +180,33 @@ function readBytes (view: DataView, context: IDynamicContext) { return true } -const bytesReader = createDynamicReader(2, readBytes) +function readBytes (view: DataView, context: IDynamicContext) { + return readBytesBase(view.getUint32(context.byteOffset), view, context) +} -const stringReader = createDynamicReader(2, (view: DataView, context: IDynamicContext) => { - if (!readBytes(view, context)) { +function readBytesLE (view: DataView, context: IDynamicContext) { + return readBytesBase(view.getUint32(context.byteOffset, true), view, context) +} + +function readVarbytes (view: DataView, context: IDynamicContext) { + if (!readVarUint32(view, context)) { return false } + return readBytesBase(context.data, view, context) +} + +function decodeUtf8Context (context) { context.data = decodeUtf8(context.data) return true -}) +} + +const bytesReader = createDynamicReader(2, readBytes) +const bytesLEReader = createDynamicReader(2, readBytesLE) +const varbytesReader = createDynamicReader(2, readVarbytes) + +const stringReader = createDynamicReader(2, (view: DataView, context: IDynamicContext) => readBytes(view, context) && decodeUtf8Context(context)) +const stringLEReader = createDynamicReader(2, (view: DataView, context: IDynamicContext) => readBytesLE(view, context) && decodeUtf8Context(context)) +const varstringReader = createDynamicReader(2, (view: DataView, context: IDynamicContext) => readVarbytes(view, context) && decodeUtf8Context(context)) export default function readerForFeatureType (type: FeatureType, length?: number): IReader { switch (type) { @@ -225,7 +242,11 @@ export default function readerForFeatureType (type: FeatureType, length?: number case FeatureType.varuint64: return varUint64Reader case FeatureType.varsint64: return varSint64Reader case FeatureType.bytes: return bytesReader + case FeatureType.bytesLE: return bytesLEReader + case FeatureType.varbytes: return varbytesReader case FeatureType.string: return stringReader + case FeatureType.stringLE: return stringLEReader + case FeatureType.varstring: return varstringReader case FeatureType.bool: return boolReader case FeatureType.float: return floatReader case FeatureType.double: return doubleReader From b82cda4eea6004889366528e0d0d111ff6d87b9e Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Sat, 18 Aug 2018 18:35:06 +0200 Subject: [PATCH 16/23] Extracted types into own files. --- ts/reader/IReader.ts | 37 +-- ts/reader/readFromStream.ts | 2 +- ts/reader/readerForFeatureType.ts | 314 ++++++------------------ ts/reader/readerForFeatures.ts | 7 +- ts/reader/type/bool.ts | 3 + ts/reader/type/bytes.ts | 4 + ts/reader/type/bytesLE.ts | 4 + ts/reader/type/double.ts | 3 + ts/reader/type/fixedBytes.ts | 9 + ts/reader/type/fixedString.ts | 12 + ts/reader/type/float.ts | 3 + ts/reader/type/int16.ts | 3 + ts/reader/type/int16LE.ts | 3 + ts/reader/type/int32.ts | 3 + ts/reader/type/int32LE.ts | 3 + ts/reader/type/int64.ts | 4 + ts/reader/type/int64LE.ts | 8 + ts/reader/type/int8.ts | 3 + ts/reader/type/sint32.ts | 3 + ts/reader/type/sint32LE.ts | 3 + ts/reader/type/sint64.ts | 5 + ts/reader/type/sint64LE.ts | 9 + ts/reader/type/string.ts | 6 + ts/reader/type/stringLE.ts | 6 + ts/reader/type/uint16.ts | 3 + ts/reader/type/uint16LE.ts | 3 + ts/reader/type/uint32.ts | 3 + ts/reader/type/uint32LE.ts | 3 + ts/reader/type/uint64.ts | 4 + ts/reader/type/uint64LE.ts | 8 + ts/reader/type/uint8.ts | 3 + ts/reader/type/varbytes.ts | 4 + ts/reader/type/varint32.ts | 11 + ts/reader/type/varint64.ts | 4 + ts/reader/type/varsint32.ts | 12 + ts/reader/type/varsint64.ts | 17 ++ ts/reader/type/varstring.ts | 6 + ts/reader/type/varuint32.ts | 4 + ts/reader/type/varuint64.ts | 4 + ts/reader/{ => util}/IDynamicContext.ts | 0 ts/reader/util/createDynamicReader.ts | 22 ++ ts/reader/util/createFixedReader.ts | 17 ++ ts/reader/util/decodeUtf8Context.ts | 6 + ts/reader/util/readBytes.ts | 6 + ts/reader/util/readBytesBase.ts | 13 + ts/reader/util/readBytesLE.ts | 6 + ts/reader/util/readVarInt.ts | 39 +++ ts/reader/util/readVarInt64.ts | 26 ++ ts/reader/util/readVarUint32.ts | 12 + ts/reader/util/readVarbytes.ts | 10 + ts/reader/util/zzDecode.ts | 4 + ts/reader/util/zzDecodeLong.ts | 7 + 52 files changed, 432 insertions(+), 282 deletions(-) create mode 100644 ts/reader/type/bool.ts create mode 100644 ts/reader/type/bytes.ts create mode 100644 ts/reader/type/bytesLE.ts create mode 100644 ts/reader/type/double.ts create mode 100644 ts/reader/type/fixedBytes.ts create mode 100644 ts/reader/type/fixedString.ts create mode 100644 ts/reader/type/float.ts create mode 100644 ts/reader/type/int16.ts create mode 100644 ts/reader/type/int16LE.ts create mode 100644 ts/reader/type/int32.ts create mode 100644 ts/reader/type/int32LE.ts create mode 100644 ts/reader/type/int64.ts create mode 100644 ts/reader/type/int64LE.ts create mode 100644 ts/reader/type/int8.ts create mode 100644 ts/reader/type/sint32.ts create mode 100644 ts/reader/type/sint32LE.ts create mode 100644 ts/reader/type/sint64.ts create mode 100644 ts/reader/type/sint64LE.ts create mode 100644 ts/reader/type/string.ts create mode 100644 ts/reader/type/stringLE.ts create mode 100644 ts/reader/type/uint16.ts create mode 100644 ts/reader/type/uint16LE.ts create mode 100644 ts/reader/type/uint32.ts create mode 100644 ts/reader/type/uint32LE.ts create mode 100644 ts/reader/type/uint64.ts create mode 100644 ts/reader/type/uint64LE.ts create mode 100644 ts/reader/type/uint8.ts create mode 100644 ts/reader/type/varbytes.ts create mode 100644 ts/reader/type/varint32.ts create mode 100644 ts/reader/type/varint64.ts create mode 100644 ts/reader/type/varsint32.ts create mode 100644 ts/reader/type/varsint64.ts create mode 100644 ts/reader/type/varstring.ts create mode 100644 ts/reader/type/varuint32.ts create mode 100644 ts/reader/type/varuint64.ts rename ts/reader/{ => util}/IDynamicContext.ts (100%) create mode 100644 ts/reader/util/createDynamicReader.ts create mode 100644 ts/reader/util/createFixedReader.ts create mode 100644 ts/reader/util/decodeUtf8Context.ts create mode 100644 ts/reader/util/readBytes.ts create mode 100644 ts/reader/util/readBytesBase.ts create mode 100644 ts/reader/util/readBytesLE.ts create mode 100644 ts/reader/util/readVarInt.ts create mode 100644 ts/reader/util/readVarInt64.ts create mode 100644 ts/reader/util/readVarUint32.ts create mode 100644 ts/reader/util/readVarbytes.ts create mode 100644 ts/reader/util/zzDecode.ts create mode 100644 ts/reader/util/zzDecodeLong.ts diff --git a/ts/reader/IReader.ts b/ts/reader/IReader.ts index c92a319..83ad2b0 100644 --- a/ts/reader/IReader.ts +++ b/ts/reader/IReader.ts @@ -1,4 +1,4 @@ -import IDynamicContext from './IDynamicContext' +import IDynamicContext from './util/IDynamicContext' export default interface IReader { fixedSize: boolean @@ -6,38 +6,3 @@ export default interface IReader { readDynamic (view: DataView, context: IDynamicContext): boolean read (view: DataView, byteOffset: number): any } - -export function createFixedReader (size: number, read: (view: DataView, byteOffset: number) => any): IReader { - return { - fixedSize: true, - minSize: size, - readDynamic: (view: DataView, context: IDynamicContext) => { - const offset = context.byteOffset - context.data = read(view, offset) - context.byteOffset = offset + size - context.size = size - return true - }, - read - } -} - -const helperContext: IDynamicContext = { - data: null, - byteOffset: 0, - size: 0 -} - -export function createDynamicReader (minSize: number, readDynamic: (view: DataView, context: IDynamicContext) => boolean): IReader { - return { - fixedSize: false, - readDynamic, - minSize, - read: (view: DataView, byteOffset: number) => { - helperContext.byteOffset = byteOffset - helperContext.data = undefined - readDynamic(view, helperContext) - return helperContext.data - } - } -} diff --git a/ts/reader/readFromStream.ts b/ts/reader/readFromStream.ts index a9156c6..0e46066 100644 --- a/ts/reader/readFromStream.ts +++ b/ts/reader/readFromStream.ts @@ -1,7 +1,7 @@ import Stream, { ReadableStream } from 'ts-stream' import IReader from './IReader' import { mapSeries } from 'bluebird' -import IDynamicContext from './IDynamicContext' +import IDynamicContext from './util/IDynamicContext' function combine (a: Uint8Array, b: Uint8Array) { const combined = new Uint8Array(a.length + b.length) diff --git a/ts/reader/readerForFeatureType.ts b/ts/reader/readerForFeatureType.ts index 49e7641..af3ab73 100644 --- a/ts/reader/readerForFeatureType.ts +++ b/ts/reader/readerForFeatureType.ts @@ -1,255 +1,83 @@ -import IReader, { createFixedReader, createDynamicReader } from './IReader' +import IReader from './IReader' import FeatureType from '../api/FeatureType' -import Long from 'long' -import IDynamicContext from './IDynamicContext' -import decodeUtf8 from 'decode-utf8' - -function zzDecodeLong (long: Long) { - // Adapted from https://github.com/dcodeIO/protobuf.js/blob/69623a91c1e4a99d5210b5295a9e5b39d9517554/src/util/longbits.js#L176-L181 - const mask = -(long.low & 1) - long.low = ((long.low >>> 1 | long.high << 31) ^ mask) >>> 0 - long.high = (long.high >>> 1 ^ mask) >>> 0 - return long -} - -function zzDecode (input: number) { - // Adapted from https://github.com/dcodeIO/protobuf.js/blob/69623a91c1e4a99d5210b5295a9e5b39d9517554/src/reader.js#L113-L114 - return input >>> 1 ^ -(input & 1) | 0 -} - -const boolReader: IReader = createFixedReader(1, (view: DataView, byteOffset: number) => view.getInt8(byteOffset) === 0) -const doubleReader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => view.getFloat64(byteOffset)) -const floatReader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getFloat32(byteOffset)) -const int8Reader: IReader = createFixedReader(1, (view: DataView, byteOffset: number) => view.getInt8(byteOffset)) -const uint8Reader: IReader = createFixedReader(1, (view: DataView, byteOffset: number) => view.getUint8(byteOffset)) -const int16Reader: IReader = createFixedReader(2, (view: DataView, byteOffset: number) => view.getInt16(byteOffset)) -const int16LEReader: IReader = createFixedReader(2, (view: DataView, byteOffset: number) => view.getInt16(byteOffset, true)) -const uint16Reader: IReader = createFixedReader(2, (view: DataView, byteOffset: number) => view.getUint16(byteOffset)) -const uint16LEReader: IReader = createFixedReader(2, (view: DataView, byteOffset: number) => view.getUint16(byteOffset, true)) -const int32Reader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getInt32(byteOffset)) -const int32LEReader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getInt32(byteOffset, true)) -const uint32Reader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getUint32(byteOffset)) -const uint32LEReader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getUint32(byteOffset, true)) -const sint32Reader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getInt32(byteOffset) | 0) -const sint32LEReader: IReader = createFixedReader(4, (view: DataView, byteOffset: number) => view.getInt32(byteOffset, true) | 0) -const int64Reader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => new Long(view.getInt32(byteOffset), view.getInt32(byteOffset), false)) -const int64LEReader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => { - const high = view.getInt32(byteOffset, true) - const low = view.getInt32(byteOffset, true) - return new Long(low, high, false) -}) -const uint64Reader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => new Long(view.getInt32(byteOffset), view.getInt32(byteOffset), true)) -const uint64LEReader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => { - const high = view.getInt32(byteOffset, true) - const low = view.getInt32(byteOffset, true) - return new Long(low, high, true) -}) -const sint64Reader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => zzDecodeLong(new Long(view.getInt32(byteOffset), view.getInt32(byteOffset), false))) -const sint64LEReader: IReader = createFixedReader(8, (view: DataView, byteOffset: number) => { - const high = view.getInt32(byteOffset, true) - const low = view.getInt32(byteOffset, true) - return zzDecodeLong(new Long(low, high, false)) -}) - -function readVarInt (view: DataView, context: IDynamicContext) { - // Adapted from https://github.com/dcodeIO/protobuf.js/blob/69623a91c1e4a99d5210b5295a9e5b39d9517554/src/reader.js#L85-L89 - let byteOffset = context.byteOffset - const a = view.getUint8(byteOffset++) - let data = (a & 127) >>> 0 - let size = a < 128 ? 1 : 2 - const len = view.byteLength - if (size === 2) { - if (byteOffset + 1 === len) return false - const b = view.getUint8(byteOffset++) - data = data | ((b && 127) << 7) >>> 0 - if (b >= 128) size = 3 - } - if (size === 3) { - if (byteOffset + 2 === len) return false - const c = view.getUint8(byteOffset++) - data = data | ((c && 127) << 14) >>> 0 - if (c >= 128) size = 4 - } - if (size === 4) { - if (byteOffset + 3 === len) return false - const d = view.getUint8(byteOffset++) - data = data | ((d && 127) << 21) >>> 0 - if (d >= 128) size = 5 - } - if (size === 5) { - if (byteOffset + 4 === len) return false - const e = view.getUint8(byteOffset++) - data = data | ((e && 127) << 28) >>> 0 - if (e >= 128) size = 6 - } - // Only set this after it is certain that there is a return value - context.byteOffset = byteOffset - context.data = data - context.size = size - return true -} - -function readVarUint32 (view: DataView, context: IDynamicContext) { - if (!readVarInt(view, context)) { - return false - } - if (context.size === 6) { - throw new Error('Invalid int-32 encoding!') - } - return true -} - -const varInt32Reader: IReader = createDynamicReader(1, (view: DataView, context: IDynamicContext) => { - if (!readVarUint32(view, context)) { - return false - } - context.data = context.data | 0 - return true -}) - -const varUint32Reader: IReader = createDynamicReader(1, readVarUint32) -const varSint32Reader: IReader = createDynamicReader(1, (view: DataView, context: IDynamicContext) => { - if (!readVarUint32(view, context)) { - return false - } - context.data = zzDecode(context.data) - return true -}) - -function readVarInt64 (unsigned: boolean, view: DataView, context: IDynamicContext) { - const prevByteOffset = context.byteOffset - if (!readVarInt(view, context)) { - return false - } - const low: number = context.data - if (context.size !== 6) { - if (!unsigned) context.data = low | 0 - return true - } - if (!readVarInt(view, context)) { - // the context was written, but still, lets reduce the byteOffset again - context.byteOffset = prevByteOffset - return false - } - if (context.size === 6) { - throw new Error('Invalid var-int 64 encoding!') - } - context.data = new Long(low, context.data, unsigned) - context.size = context.size + 5 - return true -} - -const varInt64Reader: IReader = createDynamicReader(1, readVarInt64.bind(null, false)) -const varUint64Reader: IReader = createDynamicReader(1, readVarInt64.bind(null, true)) -const varSint64Reader: IReader = createDynamicReader(1, (view: DataView, context: IDynamicContext) => { - if (!readVarInt64(false, view, context)) { - return false - } - if (typeof context.data === 'number') { - context.data = zzDecode(context.data) - } else { - context.data = zzDecodeLong(context.data) - } - return true -}) - -function fixedStringReader (length: number): IReader { - if (isNaN(length)) { - throw new Error(`A fixed string needs a length: ${length}`) - } - const bytesReader = fixedBytesReader(length) - return createFixedReader(length, (view: DataView, byteOffset: number) => decodeUtf8(bytesReader.read(view, byteOffset))) -} - -function fixedBytesReader (length: number): IReader { - if (isNaN(length)) { - throw new Error(`A fixed string needs a length: ${length}`) - } - return createFixedReader(length, (view: DataView, byteOffset: number) => view.buffer.slice(byteOffset, byteOffset + length)) -} - -function readBytesBase (size: number, view: DataView, context: IDynamicContext) { - const start = view.byteOffset + context.byteOffset + uint32Reader.minSize - const end = start + size - if (end >= view.buffer.byteLength) { - return false - } - context.size = size + uint32Reader.minSize - context.byteOffset = end - context.data = view.buffer.slice(start, end) - return true -} - -function readBytes (view: DataView, context: IDynamicContext) { - return readBytesBase(view.getUint32(context.byteOffset), view, context) -} - -function readBytesLE (view: DataView, context: IDynamicContext) { - return readBytesBase(view.getUint32(context.byteOffset, true), view, context) -} - -function readVarbytes (view: DataView, context: IDynamicContext) { - if (!readVarUint32(view, context)) { - return false - } - return readBytesBase(context.data, view, context) -} - -function decodeUtf8Context (context) { - context.data = decodeUtf8(context.data) - return true -} - -const bytesReader = createDynamicReader(2, readBytes) -const bytesLEReader = createDynamicReader(2, readBytesLE) -const varbytesReader = createDynamicReader(2, readVarbytes) - -const stringReader = createDynamicReader(2, (view: DataView, context: IDynamicContext) => readBytes(view, context) && decodeUtf8Context(context)) -const stringLEReader = createDynamicReader(2, (view: DataView, context: IDynamicContext) => readBytesLE(view, context) && decodeUtf8Context(context)) -const varstringReader = createDynamicReader(2, (view: DataView, context: IDynamicContext) => readVarbytes(view, context) && decodeUtf8Context(context)) +import fixedString from './type/fixedString' +import fixedBytes from './type/fixedBytes' +import int8 from './type/int8' +import uint8 from './type/uint8' +import int16 from './type/int16' +import int16LE from './type/int16LE' +import uint16 from './type/uint16' +import uint16LE from './type/uint16LE' +import int32 from './type/int32' +import int32LE from './type/int32LE' +import uint32 from './type/uint32' +import uint32LE from './type/uint32LE' +import sint32 from './type/sint32' +import sint32LE from './type/sint32LE' +import varint32 from './type/varint32' +import varuint32 from './type/varuint32' +import varsint32 from './type/varsint32' +import int64 from './type/int64' +import int64LE from './type/int64LE' +import uint64 from './type/uint64' +import uint64LE from './type/uint64LE' +import sint64 from './type/sint64' +import sint64LE from './type/sint64LE' +import varint64 from './type/varint64' +import varuint64 from './type/varuint64' +import varsint64 from './type/varsint64' +import bytes from './type/bytes' +import bytesLE from './type/bytesLE' +import varbytes from './type/varbytes' +import string from './type/string' +import stringLE from './type/stringLE' +import varstring from './type/varstring' +import bool from './type/bool' +import float from './type/float' +import double from './type/double' export default function readerForFeatureType (type: FeatureType, length?: number): IReader { switch (type) { - case FeatureType.fixedstring: return fixedStringReader(length) - case FeatureType.fixedbytes: return fixedBytesReader(length) + case FeatureType.fixedstring: return fixedString(length) + case FeatureType.fixedbytes: return fixedBytes(length) } if (length !== undefined) { throw new Error(`FeatureType#${type} is not a dynamic-size type, don't specify a length.`) } switch (type) { - case FeatureType.int8: return int8Reader - case FeatureType.uint8: return uint8Reader - case FeatureType.int16: return int16Reader - case FeatureType.int16LE: return int16LEReader - case FeatureType.uint16: return uint16Reader - case FeatureType.uint16LE: return uint16LEReader - case FeatureType.int32: return int32Reader - case FeatureType.int32LE: return int32LEReader - case FeatureType.uint32: return uint32Reader - case FeatureType.uint32LE: return uint32LEReader - case FeatureType.sint32: return sint32Reader - case FeatureType.sint32LE: return sint32LEReader - case FeatureType.varint32: return varInt32Reader - case FeatureType.varuint32: return varUint32Reader - case FeatureType.varsint32: return varSint32Reader - case FeatureType.int64: return int64Reader - case FeatureType.int64LE: return int64LEReader - case FeatureType.uint64: return uint64Reader - case FeatureType.uint64LE: return uint64LEReader - case FeatureType.sint64: return sint64Reader - case FeatureType.sint64LE: return sint64LEReader - case FeatureType.varint64: return varInt64Reader - case FeatureType.varuint64: return varUint64Reader - case FeatureType.varsint64: return varSint64Reader - case FeatureType.bytes: return bytesReader - case FeatureType.bytesLE: return bytesLEReader - case FeatureType.varbytes: return varbytesReader - case FeatureType.string: return stringReader - case FeatureType.stringLE: return stringLEReader - case FeatureType.varstring: return varstringReader - case FeatureType.bool: return boolReader - case FeatureType.float: return floatReader - case FeatureType.double: return doubleReader + case FeatureType.int8: return int8 + case FeatureType.uint8: return uint8 + case FeatureType.int16: return int16 + case FeatureType.int16LE: return int16LE + case FeatureType.uint16: return uint16 + case FeatureType.uint16LE: return uint16LE + case FeatureType.int32: return int32 + case FeatureType.int32LE: return int32LE + case FeatureType.uint32: return uint32 + case FeatureType.uint32LE: return uint32LE + case FeatureType.sint32: return sint32 + case FeatureType.sint32LE: return sint32LE + case FeatureType.varint32: return varint32 + case FeatureType.varuint32: return varuint32 + case FeatureType.varsint32: return varsint32 + case FeatureType.int64: return int64 + case FeatureType.int64LE: return int64LE + case FeatureType.uint64: return uint64 + case FeatureType.uint64LE: return uint64LE + case FeatureType.sint64: return sint64 + case FeatureType.sint64LE: return sint64LE + case FeatureType.varint64: return varint64 + case FeatureType.varuint64: return varuint64 + case FeatureType.varsint64: return varsint64 + case FeatureType.bytes: return bytes + case FeatureType.bytesLE: return bytesLE + case FeatureType.varbytes: return varbytes + case FeatureType.string: return string + case FeatureType.stringLE: return stringLE + case FeatureType.varstring: return varstring + case FeatureType.bool: return bool + case FeatureType.float: return float + case FeatureType.double: return double } throw new Error(`No reader available for FeatureType#${type}.`) } diff --git a/ts/reader/readerForFeatures.ts b/ts/reader/readerForFeatures.ts index 8d56abc..5e5f1eb 100644 --- a/ts/reader/readerForFeatures.ts +++ b/ts/reader/readerForFeatures.ts @@ -1,7 +1,10 @@ import IFeature from '../api/IFeature' -import IReader, { createDynamicReader, createFixedReader } from './IReader' +import IReader from './IReader' import readerForFeatureType from './readerForFeatureType' -import IDynamicContext from './IDynamicContext' +import IDynamicContext from './util/IDynamicContext' +import createDynamicReader from './util/createDynamicReader' +import createFixedReader from './util/createFixedReader' + const workContext: IDynamicContext = { byteOffset: 0, data: null, diff --git a/ts/reader/type/bool.ts b/ts/reader/type/bool.ts new file mode 100644 index 0000000..ea06d4a --- /dev/null +++ b/ts/reader/type/bool.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(1, (view: DataView, byteOffset: number) => view.getInt8(byteOffset) === 0) diff --git a/ts/reader/type/bytes.ts b/ts/reader/type/bytes.ts new file mode 100644 index 0000000..ebeb7f4 --- /dev/null +++ b/ts/reader/type/bytes.ts @@ -0,0 +1,4 @@ +import createDynamicReader from '../util/createDynamicReader' +import readBytes from '../util/readBytes' + +export default createDynamicReader(5, readBytes) diff --git a/ts/reader/type/bytesLE.ts b/ts/reader/type/bytesLE.ts new file mode 100644 index 0000000..0ace7b8 --- /dev/null +++ b/ts/reader/type/bytesLE.ts @@ -0,0 +1,4 @@ +import createDynamicReader from '../util/createDynamicReader' +import readBytesLE from '../util/readBytesLE' + +export default createDynamicReader(5, readBytesLE) diff --git a/ts/reader/type/double.ts b/ts/reader/type/double.ts new file mode 100644 index 0000000..1391482 --- /dev/null +++ b/ts/reader/type/double.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(8, (view: DataView, byteOffset: number) => view.getFloat64(byteOffset)) diff --git a/ts/reader/type/fixedBytes.ts b/ts/reader/type/fixedBytes.ts new file mode 100644 index 0000000..b65d0f2 --- /dev/null +++ b/ts/reader/type/fixedBytes.ts @@ -0,0 +1,9 @@ +import IReader from '../IReader' +import createFixedReader from '../util/createFixedReader' + +export default function fixedBytesReader (length: number): IReader { + if (isNaN(length)) { + throw new Error(`A fixed string needs a length: ${length}`) + } + return createFixedReader(length, (view: DataView, byteOffset: number) => view.buffer.slice(byteOffset, byteOffset + length)) +} diff --git a/ts/reader/type/fixedString.ts b/ts/reader/type/fixedString.ts new file mode 100644 index 0000000..d2df7de --- /dev/null +++ b/ts/reader/type/fixedString.ts @@ -0,0 +1,12 @@ +import createFixedReader from '../util/createFixedReader' +import IReader from '../IReader' +import decodeUtf8 from 'decode-utf8' +import fixedBytes from './fixedBytes' + +export default function fixedStringReader (length: number): IReader { + if (isNaN(length)) { + throw new Error(`A fixed string needs a length: ${length}`) + } + const bytesReader = fixedBytes(length) + return createFixedReader(length, (view: DataView, byteOffset: number) => decodeUtf8(bytesReader.read(view, byteOffset))) +} diff --git a/ts/reader/type/float.ts b/ts/reader/type/float.ts new file mode 100644 index 0000000..3aa5cb6 --- /dev/null +++ b/ts/reader/type/float.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(4, (view: DataView, byteOffset: number) => view.getFloat32(byteOffset)) diff --git a/ts/reader/type/int16.ts b/ts/reader/type/int16.ts new file mode 100644 index 0000000..a7517de --- /dev/null +++ b/ts/reader/type/int16.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(2, (view: DataView, byteOffset: number) => view.getInt16(byteOffset)) diff --git a/ts/reader/type/int16LE.ts b/ts/reader/type/int16LE.ts new file mode 100644 index 0000000..017169f --- /dev/null +++ b/ts/reader/type/int16LE.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(2, (view: DataView, byteOffset: number) => view.getInt16(byteOffset, true)) diff --git a/ts/reader/type/int32.ts b/ts/reader/type/int32.ts new file mode 100644 index 0000000..2bf9e4b --- /dev/null +++ b/ts/reader/type/int32.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(4, (view: DataView, byteOffset: number) => view.getInt32(byteOffset)) diff --git a/ts/reader/type/int32LE.ts b/ts/reader/type/int32LE.ts new file mode 100644 index 0000000..413c6e0 --- /dev/null +++ b/ts/reader/type/int32LE.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(4, (view: DataView, byteOffset: number) => view.getInt32(byteOffset, true)) diff --git a/ts/reader/type/int64.ts b/ts/reader/type/int64.ts new file mode 100644 index 0000000..0dcb22f --- /dev/null +++ b/ts/reader/type/int64.ts @@ -0,0 +1,4 @@ +import createFixedReader from '../util/createFixedReader' +import Long from 'long' + +export default createFixedReader(8, (view: DataView, byteOffset: number) => new Long(view.getInt32(byteOffset), view.getInt32(byteOffset), false)) diff --git a/ts/reader/type/int64LE.ts b/ts/reader/type/int64LE.ts new file mode 100644 index 0000000..5ff51dd --- /dev/null +++ b/ts/reader/type/int64LE.ts @@ -0,0 +1,8 @@ +import createFixedReader from '../util/createFixedReader' +import Long from 'long' + +export default createFixedReader(8, (view: DataView, byteOffset: number) => { + const high = view.getInt32(byteOffset, true) + const low = view.getInt32(byteOffset, true) + return new Long(low, high, false) +}) diff --git a/ts/reader/type/int8.ts b/ts/reader/type/int8.ts new file mode 100644 index 0000000..c809f0d --- /dev/null +++ b/ts/reader/type/int8.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(1, (view: DataView, byteOffset: number) => view.getInt8(byteOffset)) diff --git a/ts/reader/type/sint32.ts b/ts/reader/type/sint32.ts new file mode 100644 index 0000000..59ca915 --- /dev/null +++ b/ts/reader/type/sint32.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(4, (view: DataView, byteOffset: number) => view.getInt32(byteOffset) | 0) diff --git a/ts/reader/type/sint32LE.ts b/ts/reader/type/sint32LE.ts new file mode 100644 index 0000000..42bf4f1 --- /dev/null +++ b/ts/reader/type/sint32LE.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(4, (view: DataView, byteOffset: number) => view.getInt32(byteOffset, true) | 0) diff --git a/ts/reader/type/sint64.ts b/ts/reader/type/sint64.ts new file mode 100644 index 0000000..6cab0b1 --- /dev/null +++ b/ts/reader/type/sint64.ts @@ -0,0 +1,5 @@ +import createFixedReader from '../util/createFixedReader' +import Long from 'long' +import zzDecodeLong from '../util/zzDecodeLong' + +export default createFixedReader(8, (view: DataView, byteOffset: number) => zzDecodeLong(new Long(view.getInt32(byteOffset), view.getInt32(byteOffset), false))) diff --git a/ts/reader/type/sint64LE.ts b/ts/reader/type/sint64LE.ts new file mode 100644 index 0000000..c7d5116 --- /dev/null +++ b/ts/reader/type/sint64LE.ts @@ -0,0 +1,9 @@ +import createFixedReader from '../util/createFixedReader' +import Long from 'long' +import zzDecodeLong from '../util/zzDecodeLong' + +export default createFixedReader(8, (view: DataView, byteOffset: number) => { + const high = view.getInt32(byteOffset, true) + const low = view.getInt32(byteOffset, true) + return zzDecodeLong(new Long(low, high, false)) +}) diff --git a/ts/reader/type/string.ts b/ts/reader/type/string.ts new file mode 100644 index 0000000..33b9c7a --- /dev/null +++ b/ts/reader/type/string.ts @@ -0,0 +1,6 @@ +import createDynamicReader from '../util/createDynamicReader' +import IDynamicContext from '../util/IDynamicContext' +import readBytes from '../util/readBytes' +import decodeUtf8Context from '../util/decodeUtf8Context' + +export default createDynamicReader(5, (view: DataView, context: IDynamicContext) => readBytes(view, context) && decodeUtf8Context(context)) diff --git a/ts/reader/type/stringLE.ts b/ts/reader/type/stringLE.ts new file mode 100644 index 0000000..d1559d3 --- /dev/null +++ b/ts/reader/type/stringLE.ts @@ -0,0 +1,6 @@ +import createDynamicReader from '../util/createDynamicReader' +import IDynamicContext from '../util/IDynamicContext' +import readBytesLE from '../util/readBytesLE' +import decodeUtf8Context from '../util/decodeUtf8Context' + +export default createDynamicReader(5, (view: DataView, context: IDynamicContext) => readBytesLE(view, context) && decodeUtf8Context(context)) diff --git a/ts/reader/type/uint16.ts b/ts/reader/type/uint16.ts new file mode 100644 index 0000000..e7534ff --- /dev/null +++ b/ts/reader/type/uint16.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(2, (view: DataView, byteOffset: number) => view.getUint16(byteOffset)) diff --git a/ts/reader/type/uint16LE.ts b/ts/reader/type/uint16LE.ts new file mode 100644 index 0000000..6092ef8 --- /dev/null +++ b/ts/reader/type/uint16LE.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(2, (view: DataView, byteOffset: number) => view.getUint16(byteOffset, true)) diff --git a/ts/reader/type/uint32.ts b/ts/reader/type/uint32.ts new file mode 100644 index 0000000..4ea22cd --- /dev/null +++ b/ts/reader/type/uint32.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(4, (view: DataView, byteOffset: number) => view.getUint32(byteOffset)) diff --git a/ts/reader/type/uint32LE.ts b/ts/reader/type/uint32LE.ts new file mode 100644 index 0000000..cf77334 --- /dev/null +++ b/ts/reader/type/uint32LE.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(4, (view: DataView, byteOffset: number) => view.getUint32(byteOffset, true)) diff --git a/ts/reader/type/uint64.ts b/ts/reader/type/uint64.ts new file mode 100644 index 0000000..ab5ea14 --- /dev/null +++ b/ts/reader/type/uint64.ts @@ -0,0 +1,4 @@ +import createFixedReader from '../util/createFixedReader' +import Long from 'long' + +export default createFixedReader(8, (view: DataView, byteOffset: number) => new Long(view.getInt32(byteOffset), view.getInt32(byteOffset), true)) diff --git a/ts/reader/type/uint64LE.ts b/ts/reader/type/uint64LE.ts new file mode 100644 index 0000000..8174761 --- /dev/null +++ b/ts/reader/type/uint64LE.ts @@ -0,0 +1,8 @@ +import createFixedReader from '../util/createFixedReader' +import Long from 'long' + +export default createFixedReader(8, (view: DataView, byteOffset: number) => { + const high = view.getInt32(byteOffset, true) + const low = view.getInt32(byteOffset, true) + return new Long(low, high, true) +}) diff --git a/ts/reader/type/uint8.ts b/ts/reader/type/uint8.ts new file mode 100644 index 0000000..7c3a351 --- /dev/null +++ b/ts/reader/type/uint8.ts @@ -0,0 +1,3 @@ +import createFixedReader from '../util/createFixedReader' + +export default createFixedReader(1, (view: DataView, byteOffset: number) => view.getUint8(byteOffset)) diff --git a/ts/reader/type/varbytes.ts b/ts/reader/type/varbytes.ts new file mode 100644 index 0000000..1f6dce5 --- /dev/null +++ b/ts/reader/type/varbytes.ts @@ -0,0 +1,4 @@ +import createDynamicReader from '../util/createDynamicReader' +import readVarbytes from '../util/readVarbytes' + +export default createDynamicReader(2, readVarbytes) diff --git a/ts/reader/type/varint32.ts b/ts/reader/type/varint32.ts new file mode 100644 index 0000000..6d80ede --- /dev/null +++ b/ts/reader/type/varint32.ts @@ -0,0 +1,11 @@ +import createDynamicReader from '../util/createDynamicReader' +import IDynamicContext from '../util/IDynamicContext' +import readVarUint32 from '../util/readVarUint32' + +export default createDynamicReader(1, (view: DataView, context: IDynamicContext) => { + if (!readVarUint32(view, context)) { + return false + } + context.data = context.data | 0 + return true +}) diff --git a/ts/reader/type/varint64.ts b/ts/reader/type/varint64.ts new file mode 100644 index 0000000..779d642 --- /dev/null +++ b/ts/reader/type/varint64.ts @@ -0,0 +1,4 @@ +import readVarInt64 from '../util/readVarInt64' +import createDynamicReader from '../util/createDynamicReader' + +export default createDynamicReader(1, readVarInt64.bind(null, false)) diff --git a/ts/reader/type/varsint32.ts b/ts/reader/type/varsint32.ts new file mode 100644 index 0000000..e3a536b --- /dev/null +++ b/ts/reader/type/varsint32.ts @@ -0,0 +1,12 @@ +import createDynamicReader from '../util/createDynamicReader' +import IDynamicContext from '../util/IDynamicContext' +import readVarUint32 from '../util/readVarUint32' +import zzDecode from '../util/zzDecode' + +export default createDynamicReader(1, (view: DataView, context: IDynamicContext) => { + if (!readVarUint32(view, context)) { + return false + } + context.data = zzDecode(context.data) + return true +}) diff --git a/ts/reader/type/varsint64.ts b/ts/reader/type/varsint64.ts new file mode 100644 index 0000000..be80087 --- /dev/null +++ b/ts/reader/type/varsint64.ts @@ -0,0 +1,17 @@ +import createDynamicReader from '../util/createDynamicReader' +import IDynamicContext from '../util/IDynamicContext' +import readVarInt64 from '../util/readVarInt64' +import zzDecode from '../util/zzDecode' +import zzDecodeLong from '../util/zzDecodeLong' + +export default createDynamicReader(1, (view: DataView, context: IDynamicContext) => { + if (!readVarInt64(false, view, context)) { + return false + } + if (typeof context.data === 'number') { + context.data = zzDecode(context.data) + } else { + context.data = zzDecodeLong(context.data) + } + return true +}) diff --git a/ts/reader/type/varstring.ts b/ts/reader/type/varstring.ts new file mode 100644 index 0000000..8aeeb08 --- /dev/null +++ b/ts/reader/type/varstring.ts @@ -0,0 +1,6 @@ +import createDynamicReader from '../util/createDynamicReader' +import readVarbytes from '../util/readVarbytes' +import IDynamicContext from '../util/IDynamicContext' +import decodeUtf8Context from '../util/decodeUtf8Context' + +export default createDynamicReader(2, (view: DataView, context: IDynamicContext) => readVarbytes(view, context) && decodeUtf8Context(context)) diff --git a/ts/reader/type/varuint32.ts b/ts/reader/type/varuint32.ts new file mode 100644 index 0000000..e4cac66 --- /dev/null +++ b/ts/reader/type/varuint32.ts @@ -0,0 +1,4 @@ +import createDynamicReader from '../util/createDynamicReader' +import readVarUint32 from '../util/readVarUint32' + +export default createDynamicReader(1, readVarUint32) diff --git a/ts/reader/type/varuint64.ts b/ts/reader/type/varuint64.ts new file mode 100644 index 0000000..6af2998 --- /dev/null +++ b/ts/reader/type/varuint64.ts @@ -0,0 +1,4 @@ +import createDynamicReader from '../util/createDynamicReader' +import readVarInt64 from '../util/readVarInt64' + +export default createDynamicReader(1, readVarInt64.bind(null, true)) diff --git a/ts/reader/IDynamicContext.ts b/ts/reader/util/IDynamicContext.ts similarity index 100% rename from ts/reader/IDynamicContext.ts rename to ts/reader/util/IDynamicContext.ts diff --git a/ts/reader/util/createDynamicReader.ts b/ts/reader/util/createDynamicReader.ts new file mode 100644 index 0000000..bf7bc56 --- /dev/null +++ b/ts/reader/util/createDynamicReader.ts @@ -0,0 +1,22 @@ +import IDynamicContext from './IDynamicContext' +import IReader from '../IReader' + +const helperContext: IDynamicContext = { + data: null, + byteOffset: 0, + size: 0 +} + +export default function createDynamicReader (minSize: number, readDynamic: (view: DataView, context: IDynamicContext) => boolean): IReader { + return { + fixedSize: false, + readDynamic, + minSize, + read: (view: DataView, byteOffset: number) => { + helperContext.byteOffset = byteOffset + helperContext.data = undefined + readDynamic(view, helperContext) + return helperContext.data + } + } +} diff --git a/ts/reader/util/createFixedReader.ts b/ts/reader/util/createFixedReader.ts new file mode 100644 index 0000000..f02a45f --- /dev/null +++ b/ts/reader/util/createFixedReader.ts @@ -0,0 +1,17 @@ +import IDynamicContext from './IDynamicContext' +import IReader from '../IReader' + +export default function createFixedReader (size: number, read: (view: DataView, byteOffset: number) => any): IReader { + return { + fixedSize: true, + minSize: size, + readDynamic: (view: DataView, context: IDynamicContext) => { + const offset = context.byteOffset + context.data = read(view, offset) + context.byteOffset = offset + size + context.size = size + return true + }, + read + } +} diff --git a/ts/reader/util/decodeUtf8Context.ts b/ts/reader/util/decodeUtf8Context.ts new file mode 100644 index 0000000..9aef1f2 --- /dev/null +++ b/ts/reader/util/decodeUtf8Context.ts @@ -0,0 +1,6 @@ +import decodeUtf8 from 'decode-utf8' + +export default function decodeUtf8Context (context) { + context.data = decodeUtf8(context.data) + return true +} diff --git a/ts/reader/util/readBytes.ts b/ts/reader/util/readBytes.ts new file mode 100644 index 0000000..1e84ae8 --- /dev/null +++ b/ts/reader/util/readBytes.ts @@ -0,0 +1,6 @@ +import IDynamicContext from './IDynamicContext' +import readBytesBase from './readBytesBase' + +export default function readBytes (view: DataView, context: IDynamicContext) { + return readBytesBase(view.getUint32(context.byteOffset), 4, view, context) +} diff --git a/ts/reader/util/readBytesBase.ts b/ts/reader/util/readBytesBase.ts new file mode 100644 index 0000000..5feefa4 --- /dev/null +++ b/ts/reader/util/readBytesBase.ts @@ -0,0 +1,13 @@ +import IDynamicContext from './IDynamicContext' + +export default function readBytesBase (size: number, sizeBytes: number, view: DataView, context: IDynamicContext) { + const start = view.byteOffset + context.byteOffset + sizeBytes + const end = start + size + if (end >= view.buffer.byteLength) { + return false + } + context.size = size + sizeBytes + context.byteOffset = end + context.data = view.buffer.slice(start, end) + return true +} diff --git a/ts/reader/util/readBytesLE.ts b/ts/reader/util/readBytesLE.ts new file mode 100644 index 0000000..0149f7f --- /dev/null +++ b/ts/reader/util/readBytesLE.ts @@ -0,0 +1,6 @@ +import IDynamicContext from './IDynamicContext' +import readBytesBase from './readBytesBase' + +export default function readBytesLE (view: DataView, context: IDynamicContext) { + return readBytesBase(view.getUint32(context.byteOffset, true), 4, view, context) +} diff --git a/ts/reader/util/readVarInt.ts b/ts/reader/util/readVarInt.ts new file mode 100644 index 0000000..277ddf2 --- /dev/null +++ b/ts/reader/util/readVarInt.ts @@ -0,0 +1,39 @@ +import IDynamicContext from './IDynamicContext' + +export default function readVarInt (view: DataView, context: IDynamicContext) { + // Adapted from https://github.com/dcodeIO/protobuf.js/blob/69623a91c1e4a99d5210b5295a9e5b39d9517554/src/reader.js#L85-L89 + let byteOffset = context.byteOffset + const a = view.getUint8(byteOffset++) + let data = (a & 127) >>> 0 + let size = a < 128 ? 1 : 2 + const len = view.byteLength + if (size === 2) { + if (byteOffset + 1 === len) return false + const b = view.getUint8(byteOffset++) + data = data | ((b && 127) << 7) >>> 0 + if (b >= 128) size = 3 + } + if (size === 3) { + if (byteOffset + 2 === len) return false + const c = view.getUint8(byteOffset++) + data = data | ((c && 127) << 14) >>> 0 + if (c >= 128) size = 4 + } + if (size === 4) { + if (byteOffset + 3 === len) return false + const d = view.getUint8(byteOffset++) + data = data | ((d && 127) << 21) >>> 0 + if (d >= 128) size = 5 + } + if (size === 5) { + if (byteOffset + 4 === len) return false + const e = view.getUint8(byteOffset++) + data = data | ((e && 127) << 28) >>> 0 + if (e >= 128) size = 6 + } + // Only set this after it is certain that there is a return value + context.byteOffset = byteOffset + context.data = data + context.size = size + return true +} diff --git a/ts/reader/util/readVarInt64.ts b/ts/reader/util/readVarInt64.ts new file mode 100644 index 0000000..c32aa26 --- /dev/null +++ b/ts/reader/util/readVarInt64.ts @@ -0,0 +1,26 @@ +import IDynamicContext from './IDynamicContext' +import readVarInt from './readVarInt' +import Long from 'long' + +export default function readVarInt64 (unsigned: boolean, view: DataView, context: IDynamicContext) { + const prevByteOffset = context.byteOffset + if (!readVarInt(view, context)) { + return false + } + const low: number = context.data + if (context.size !== 6) { + if (!unsigned) context.data = low | 0 + return true + } + if (!readVarInt(view, context)) { + // the context was written, but still, lets reduce the byteOffset again + context.byteOffset = prevByteOffset + return false + } + if (context.size === 6) { + throw new Error('Invalid var-int 64 encoding!') + } + context.data = new Long(low, context.data, unsigned) + context.size = context.size + 5 + return true +} diff --git a/ts/reader/util/readVarUint32.ts b/ts/reader/util/readVarUint32.ts new file mode 100644 index 0000000..0dad120 --- /dev/null +++ b/ts/reader/util/readVarUint32.ts @@ -0,0 +1,12 @@ +import readVarInt from './readVarInt' +import IDynamicContext from './IDynamicContext' + +export default function readVarUint32 (view: DataView, context: IDynamicContext) { + if (!readVarInt(view, context)) { + return false + } + if (context.size === 6) { + throw new Error('Invalid int-32 encoding!') + } + return true +} diff --git a/ts/reader/util/readVarbytes.ts b/ts/reader/util/readVarbytes.ts new file mode 100644 index 0000000..f27b730 --- /dev/null +++ b/ts/reader/util/readVarbytes.ts @@ -0,0 +1,10 @@ +import readVarUint32 from './readVarUint32' +import readBytesBase from './readBytesBase' +import IDynamicContext from './IDynamicContext' + +export default function readVarbytes (view: DataView, context: IDynamicContext) { + if (!readVarUint32(view, context)) { + return false + } + return readBytesBase(context.data, context.size, view, context) +} diff --git a/ts/reader/util/zzDecode.ts b/ts/reader/util/zzDecode.ts new file mode 100644 index 0000000..4958272 --- /dev/null +++ b/ts/reader/util/zzDecode.ts @@ -0,0 +1,4 @@ +export default function zzDecode (input: number) { + // Adapted from https://github.com/dcodeIO/protobuf.js/blob/69623a91c1e4a99d5210b5295a9e5b39d9517554/src/reader.js#L113-L114 + return input >>> 1 ^ -(input & 1) | 0 +} diff --git a/ts/reader/util/zzDecodeLong.ts b/ts/reader/util/zzDecodeLong.ts new file mode 100644 index 0000000..7a749d2 --- /dev/null +++ b/ts/reader/util/zzDecodeLong.ts @@ -0,0 +1,7 @@ +export default function zzDecodeLong (long: Long) { + // Adapted from https://github.com/dcodeIO/protobuf.js/blob/69623a91c1e4a99d5210b5295a9e5b39d9517554/src/util/longbits.js#L176-L181 + const mask = -(long.low & 1) + long.low = ((long.low >>> 1 | long.high << 31) ^ mask) >>> 0 + long.high = (long.high >>> 1 ^ mask) >>> 0 + return long +} From fccdd1bd656beccfee2ffc11c099115516a33458 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Tue, 21 Aug 2018 12:36:02 +0200 Subject: [PATCH 17/23] Tested and fixed the varint reader. --- ts/reader/util/readVarInt.ts | 21 +++++++++--------- ts_test/reader/util/readVarInt.ts | 36 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 10 deletions(-) create mode 100755 ts_test/reader/util/readVarInt.ts diff --git a/ts/reader/util/readVarInt.ts b/ts/reader/util/readVarInt.ts index 277ddf2..ea5a2a7 100644 --- a/ts/reader/util/readVarInt.ts +++ b/ts/reader/util/readVarInt.ts @@ -1,37 +1,38 @@ import IDynamicContext from './IDynamicContext' +let data = 4294967295 // optimizer type-hint, tends to deopt otherwise (?!) export default function readVarInt (view: DataView, context: IDynamicContext) { // Adapted from https://github.com/dcodeIO/protobuf.js/blob/69623a91c1e4a99d5210b5295a9e5b39d9517554/src/reader.js#L85-L89 let byteOffset = context.byteOffset const a = view.getUint8(byteOffset++) - let data = (a & 127) >>> 0 + data = (a & 127) >>> 0 let size = a < 128 ? 1 : 2 const len = view.byteLength if (size === 2) { - if (byteOffset + 1 === len) return false + if (byteOffset === len) return false const b = view.getUint8(byteOffset++) - data = data | ((b && 127) << 7) >>> 0 + const bPart = ((b & 127) << 7) >>> 0 + data = data | bPart if (b >= 128) size = 3 } if (size === 3) { - if (byteOffset + 2 === len) return false + if (byteOffset === len) return false const c = view.getUint8(byteOffset++) - data = data | ((c && 127) << 14) >>> 0 + data = (data | (c & 127) << 14) >>> 0 if (c >= 128) size = 4 } if (size === 4) { - if (byteOffset + 3 === len) return false + if (byteOffset === len) return false const d = view.getUint8(byteOffset++) - data = data | ((d && 127) << 21) >>> 0 + data = (data | (d & 127) << 21) >>> 0 if (d >= 128) size = 5 } if (size === 5) { - if (byteOffset + 4 === len) return false + if (byteOffset === len) return false const e = view.getUint8(byteOffset++) - data = data | ((e && 127) << 28) >>> 0 + data = (data | (e & 15) << 28) >>> 0 if (e >= 128) size = 6 } - // Only set this after it is certain that there is a return value context.byteOffset = byteOffset context.data = data context.size = size diff --git a/ts_test/reader/util/readVarInt.ts b/ts_test/reader/util/readVarInt.ts new file mode 100755 index 0000000..4477724 --- /dev/null +++ b/ts_test/reader/util/readVarInt.ts @@ -0,0 +1,36 @@ +#!/usr/bin/env node --require ts-node/register +import { test } from 'tap' +import readVarInt from '../../../ts/reader/util/readVarInt' +import IDynamicContext from '../../../ts/reader/util/IDynamicContext' +import { Texture } from 'three' + +function dArr (arr: number[]): DataView { + const buff = new Uint8Array(arr) + return new DataView(buff.buffer) +} + +function read (...arr: number[]) { + const c = { byteOffset: 0, size: 0, data: null } + const works = readVarInt(dArr(arr), c) + return { + works, + data: c.data, + size: c.size + } +} + +test('varuint32-read', async t => { + t.deepEquals(read(0), { works: true, data: 0, size: 1 }, 'single byte zero') + t.deepEquals(read(1), { works: true, data: 1, size: 1 }, 'single byte 1') + t.deepEquals(read(127), { works: true, data: 127, size: 1 }, 'single byte 127') + t.deepEquals(read(128), { works: false, data: null, size: 0 }, 'double byte, second byte missing') + t.deepEquals(read(255, 1), { works: true, data: 255, size: 2 }, 'double byte 255') + t.deepEquals(read(255, 128), { works: false, data: null, size: 0 }, 'triple byte, third byte missing') + t.deepEquals(read(255, 128, 3), { works: true, data: 49279, size: 3 }, 'triple byte 49279') + t.deepEquals(read(255, 128, 130), { works: false, data: null, size: 0 }, 'quadruple byte, forth byte missing') + t.deepEquals(read(255, 128, 130, 1), { works: true, data: 2130047, size: 4 }, 'quadruple byte 2130047') + t.deepEquals(read(255, 128, 130, 131), { works: false, data: null, size: 0 }, 'quituple byte, fifth byte missing') + t.deepEquals(read(255, 128, 130, 131, 1), { works: true, data: 274759807, size: 5 }, 'quituple byte, 274759807') + t.deepEquals(read(255, 255, 255, 255, 255), { works: true, data: 4294967295, size: 6 }, 'quituple byte, max value = 4294967295') + t.deepEquals(read(255, 128, 130, 131, 255, 1), { works: true, data: 4032856191, size: 6 }, 'additional bytes ignored') +}) From d25703add54c4d8b41a56be032ca7c445ed2b582 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Tue, 21 Aug 2018 14:39:57 +0200 Subject: [PATCH 18/23] Extracted two part reading logic into own util that is tested --- ts/reader/type/bytes.ts | 7 +-- ts/reader/type/bytesLE.ts | 7 +-- ts/reader/type/string.ts | 9 ++-- ts/reader/type/stringLE.ts | 9 ++-- ts/reader/type/varbytes.ts | 7 +-- ts/reader/type/varstring.ts | 9 ++-- ts/reader/util/readBytes.ts | 6 --- ts/reader/util/readBytesBase.ts | 13 ----- ts/reader/util/readBytesLE.ts | 6 --- ts/reader/util/readVarbytes.ts | 10 ---- ts/reader/util/twoPartReader.ts | 65 ++++++++++++++++++++++ ts_test/reader/util/twoPartReader.ts | 80 ++++++++++++++++++++++++++++ 12 files changed, 169 insertions(+), 59 deletions(-) delete mode 100644 ts/reader/util/readBytes.ts delete mode 100644 ts/reader/util/readBytesBase.ts delete mode 100644 ts/reader/util/readBytesLE.ts delete mode 100644 ts/reader/util/readVarbytes.ts create mode 100644 ts/reader/util/twoPartReader.ts create mode 100755 ts_test/reader/util/twoPartReader.ts diff --git a/ts/reader/type/bytes.ts b/ts/reader/type/bytes.ts index ebeb7f4..3d1e66b 100644 --- a/ts/reader/type/bytes.ts +++ b/ts/reader/type/bytes.ts @@ -1,4 +1,5 @@ -import createDynamicReader from '../util/createDynamicReader' -import readBytes from '../util/readBytes' +import twoPartReader from '../util/twoPartReader' +import uint32 from './uint32' +import fixedBytes from './fixedBytes' -export default createDynamicReader(5, readBytes) +export default twoPartReader(uint32, fixedBytes) diff --git a/ts/reader/type/bytesLE.ts b/ts/reader/type/bytesLE.ts index 0ace7b8..0eab273 100644 --- a/ts/reader/type/bytesLE.ts +++ b/ts/reader/type/bytesLE.ts @@ -1,4 +1,5 @@ -import createDynamicReader from '../util/createDynamicReader' -import readBytesLE from '../util/readBytesLE' +import twoPartReader from '../util/twoPartReader' +import uint32LE from './uint32LE' +import fixedBytesReader from './fixedBytes' -export default createDynamicReader(5, readBytesLE) +export default twoPartReader(uint32LE, fixedBytesReader) diff --git a/ts/reader/type/string.ts b/ts/reader/type/string.ts index 33b9c7a..1e172ca 100644 --- a/ts/reader/type/string.ts +++ b/ts/reader/type/string.ts @@ -1,6 +1,5 @@ -import createDynamicReader from '../util/createDynamicReader' -import IDynamicContext from '../util/IDynamicContext' -import readBytes from '../util/readBytes' -import decodeUtf8Context from '../util/decodeUtf8Context' +import twoPartReader from '../util/twoPartReader' +import uint32 from './uint32' +import fixedString from './fixedString' -export default createDynamicReader(5, (view: DataView, context: IDynamicContext) => readBytes(view, context) && decodeUtf8Context(context)) +export default twoPartReader(uint32, fixedString) diff --git a/ts/reader/type/stringLE.ts b/ts/reader/type/stringLE.ts index d1559d3..0599a88 100644 --- a/ts/reader/type/stringLE.ts +++ b/ts/reader/type/stringLE.ts @@ -1,6 +1,5 @@ -import createDynamicReader from '../util/createDynamicReader' -import IDynamicContext from '../util/IDynamicContext' -import readBytesLE from '../util/readBytesLE' -import decodeUtf8Context from '../util/decodeUtf8Context' +import twoPartReader from '../util/twoPartReader' +import uint32LE from './uint32LE' +import fixedString from './fixedString' -export default createDynamicReader(5, (view: DataView, context: IDynamicContext) => readBytesLE(view, context) && decodeUtf8Context(context)) +export default twoPartReader(uint32LE, fixedString) diff --git a/ts/reader/type/varbytes.ts b/ts/reader/type/varbytes.ts index 1f6dce5..482ee7f 100644 --- a/ts/reader/type/varbytes.ts +++ b/ts/reader/type/varbytes.ts @@ -1,4 +1,5 @@ -import createDynamicReader from '../util/createDynamicReader' -import readVarbytes from '../util/readVarbytes' +import twoPartReader from '../util/twoPartReader' +import varuint32 from './varuint32' +import fixedBytesReader from './fixedBytes' -export default createDynamicReader(2, readVarbytes) +export default twoPartReader(varuint32, fixedBytesReader) diff --git a/ts/reader/type/varstring.ts b/ts/reader/type/varstring.ts index 8aeeb08..2c969be 100644 --- a/ts/reader/type/varstring.ts +++ b/ts/reader/type/varstring.ts @@ -1,6 +1,5 @@ -import createDynamicReader from '../util/createDynamicReader' -import readVarbytes from '../util/readVarbytes' -import IDynamicContext from '../util/IDynamicContext' -import decodeUtf8Context from '../util/decodeUtf8Context' +import twoPartReader from '../util/twoPartReader' +import varuint32 from './varuint32' +import fixedString from './fixedString' -export default createDynamicReader(2, (view: DataView, context: IDynamicContext) => readVarbytes(view, context) && decodeUtf8Context(context)) +export default twoPartReader(varuint32, fixedString) diff --git a/ts/reader/util/readBytes.ts b/ts/reader/util/readBytes.ts deleted file mode 100644 index 1e84ae8..0000000 --- a/ts/reader/util/readBytes.ts +++ /dev/null @@ -1,6 +0,0 @@ -import IDynamicContext from './IDynamicContext' -import readBytesBase from './readBytesBase' - -export default function readBytes (view: DataView, context: IDynamicContext) { - return readBytesBase(view.getUint32(context.byteOffset), 4, view, context) -} diff --git a/ts/reader/util/readBytesBase.ts b/ts/reader/util/readBytesBase.ts deleted file mode 100644 index 5feefa4..0000000 --- a/ts/reader/util/readBytesBase.ts +++ /dev/null @@ -1,13 +0,0 @@ -import IDynamicContext from './IDynamicContext' - -export default function readBytesBase (size: number, sizeBytes: number, view: DataView, context: IDynamicContext) { - const start = view.byteOffset + context.byteOffset + sizeBytes - const end = start + size - if (end >= view.buffer.byteLength) { - return false - } - context.size = size + sizeBytes - context.byteOffset = end - context.data = view.buffer.slice(start, end) - return true -} diff --git a/ts/reader/util/readBytesLE.ts b/ts/reader/util/readBytesLE.ts deleted file mode 100644 index 0149f7f..0000000 --- a/ts/reader/util/readBytesLE.ts +++ /dev/null @@ -1,6 +0,0 @@ -import IDynamicContext from './IDynamicContext' -import readBytesBase from './readBytesBase' - -export default function readBytesLE (view: DataView, context: IDynamicContext) { - return readBytesBase(view.getUint32(context.byteOffset, true), 4, view, context) -} diff --git a/ts/reader/util/readVarbytes.ts b/ts/reader/util/readVarbytes.ts deleted file mode 100644 index f27b730..0000000 --- a/ts/reader/util/readVarbytes.ts +++ /dev/null @@ -1,10 +0,0 @@ -import readVarUint32 from './readVarUint32' -import readBytesBase from './readBytesBase' -import IDynamicContext from './IDynamicContext' - -export default function readVarbytes (view: DataView, context: IDynamicContext) { - if (!readVarUint32(view, context)) { - return false - } - return readBytesBase(context.data, context.size, view, context) -} diff --git a/ts/reader/util/twoPartReader.ts b/ts/reader/util/twoPartReader.ts new file mode 100644 index 0000000..b5734e7 --- /dev/null +++ b/ts/reader/util/twoPartReader.ts @@ -0,0 +1,65 @@ +import IReader from '../IReader' +import IDynamicContext from './IDynamicContext' +import createDynamicReader from './createDynamicReader' + +const contextA: IDynamicContext = { + data: null, + size: 0, + byteOffset: 0 +} + +const contextB: IDynamicContext = { + data: null, + size: 0, + byteOffset: 0 +} + +function readPartB (partB: IReader, view: DataView, context: IDynamicContext) { + const minSize = contextA.byteOffset + partB.minSize + if (minSize > view.byteLength) { + return false + } + if (partB.fixedSize) { + context.data = partB.read(view, contextA.byteOffset) + context.size = contextA.size + partB.minSize + context.byteOffset = contextA.byteOffset + partB.minSize + } else { + contextB.byteOffset = contextA.byteOffset + if (!partB.readDynamic(view, contextB)) { + return false + } + context.size = contextA.size + contextB.size + context.byteOffset = contextB.byteOffset + context.data = contextB.data + } + return true +} + +function twoPartFixedSizeReader (partA: IReader, template: (readerResult) => IReader) { + return (view: DataView, context: IDynamicContext) => { + const afterPartA = view.byteOffset + partA.minSize + contextA.data = partA.read(view, context.byteOffset) + contextA.byteOffset = context.byteOffset + partA.minSize + contextA.size = partA.minSize + return readPartB(template(contextA.data), view, context) + } +} + +function twoPartDynamicReader (partA: IReader, template: (readerResult) => IReader) { + return (view: DataView, context: IDynamicContext) => { + contextA.byteOffset = context.byteOffset + if (!partA.readDynamic(view, contextA)) { + return false + } + return readPartB(template(contextA.data), view, context) + } +} + +export default function twoPartReader (partA: IReader, template: (readerResult) => IReader) { + return createDynamicReader( + partA.minSize, + partA.fixedSize + ? twoPartFixedSizeReader(partA, template) + : twoPartDynamicReader(partA, template) + ) +} diff --git a/ts_test/reader/util/twoPartReader.ts b/ts_test/reader/util/twoPartReader.ts new file mode 100755 index 0000000..f29f66f --- /dev/null +++ b/ts_test/reader/util/twoPartReader.ts @@ -0,0 +1,80 @@ +#!/usr/bin/env node --require ts-node/register +import { test } from 'tap' +import twoPartReader from '../../../ts/reader/util/twoPartReader' +import uint32 from '../../../ts/reader/type/uint32' +import varuint32 from '../../../ts/reader/type/varuint32' +import fixedString from '../../../ts/reader/type/fixedString' +import nullReader from '../../../ts/reader/type/null' +import IDynamicContext from '../../../ts/reader/util/IDynamicContext' +import IReader from '../../../ts/reader/IReader' + +const BYTE_128_STRING = '01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567' + +function c (char: string) { + return char.charCodeAt(0) +} + +function chars (str: string) { + const arr: number[] = [] + for (let i = 0; i < str.length; i++) { + arr.push(str.charCodeAt(i)) + } + return arr +} + +function createRead (reader: IReader) { + return (bytes: number[]) => { + const ctx = { byteOffset: 0, data: null, size: 0 } + const works = reader.readDynamic(new DataView(new Uint8Array(bytes).buffer), ctx) + return { works, data: ctx.data, size: ctx.size } + } +} + +test('fixed, fixed', async t => { + const reader = twoPartReader(uint32, fixedString) + t.equals(reader.fixedSize, false) + t.equals(reader.minSize, 4, 'minSize should be equal partA minSize') + + const read = createRead(reader) + t.deepEquals(read([0, 0, 0, 3].concat(chars('ab'))), { works: false, data: null, size: 0 }, 'too small buffer doesnt work') + t.deepEquals(read([0, 0, 0, 3].concat(chars('abc'))), { works: true, data: 'abc', size: 7 }, 'right size buffer works') +}) + +test('fixed, dynamic', async t => { + const reader = twoPartReader(uint32, num => { + t.equals(num, 4, 'former number properly read') + return varuint32 + }) + t.equals(reader.fixedSize, false) + t.equals(reader.minSize, 4, 'minSize should be equal partA minSize') + + const read = createRead(reader) + t.deepEquals(read([0, 0, 0, 4]), { works: false, data: null, size: 0 }, 'too small buffer doesnt work') + t.deepEquals(read([0, 0, 0, 4, 99]), { works: true, data: 99, size: 5 }, 'right size buffer works') + t.deepEquals(read([0, 0, 0, 4, 255, 12]), { works: true, data: 1663, size: 6 }, 'string successfully read') +}) + +test('dynamic, fixed', async t => { + const reader = twoPartReader(varuint32, fixedString) + t.equals(reader.fixedSize, false, 'isnt fixed size') + t.equals(reader.minSize, 1, 'minsize should be equal partA minSize') + + const read = createRead(reader) + t.deepEquals(read([0]), { works: true, data: '', size: 1 }, 'empty string') + t.deepEquals(read([1, c('a')]), { works: true, data: 'a', size: 2 }, 'single byte-size string') + t.deepEquals(read([2].concat(chars(BYTE_128_STRING))), { works: true, data: '01', size: 3 }, 'single byte-size string, of a larger data set') + t.deepEquals(read([128, 1].concat(chars(BYTE_128_STRING))), { works: true, data: BYTE_128_STRING, size: 130 }, 'double byte-size string') +}) + +test('dynamic, dynamic', async t => { + const reader = twoPartReader(varuint32, num => { + return varuint32 + }) + t.equals(reader.fixedSize, false, 'isnt fixed size') + t.equals(reader.minSize, 1, 'minsize should be equal partA minSize') + + const read = createRead(reader) + t.deepEquals(read([0]), { works: false, data: null, size: 0 }, 'missing second number') + t.deepEquals(read([0, 0]), { works: true, data: 0, size: 2 }, 'working second number') + t.deepEquals(read([128, 1, 128, 1]), { works: true, data: 128, size: 4 }, 'two dynamic sized numbers') +}) From b3c76d6f1f4621b963c4d6ccff2830b6ba16b2ad Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Tue, 21 Aug 2018 14:40:55 +0200 Subject: [PATCH 19/23] Fixed naming of functions. --- ts/reader/type/fixedBytes.ts | 2 +- ts/reader/type/fixedString.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ts/reader/type/fixedBytes.ts b/ts/reader/type/fixedBytes.ts index b65d0f2..724d805 100644 --- a/ts/reader/type/fixedBytes.ts +++ b/ts/reader/type/fixedBytes.ts @@ -1,7 +1,7 @@ import IReader from '../IReader' import createFixedReader from '../util/createFixedReader' -export default function fixedBytesReader (length: number): IReader { +export default function fixedBytes (length: number): IReader { if (isNaN(length)) { throw new Error(`A fixed string needs a length: ${length}`) } diff --git a/ts/reader/type/fixedString.ts b/ts/reader/type/fixedString.ts index d2df7de..5b8645d 100644 --- a/ts/reader/type/fixedString.ts +++ b/ts/reader/type/fixedString.ts @@ -3,7 +3,7 @@ import IReader from '../IReader' import decodeUtf8 from 'decode-utf8' import fixedBytes from './fixedBytes' -export default function fixedStringReader (length: number): IReader { +export default function fixedString (length: number): IReader { if (isNaN(length)) { throw new Error(`A fixed string needs a length: ${length}`) } From b39fc3d28cd47d9b264149e139b2704d72a9b2af Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Tue, 21 Aug 2018 14:41:11 +0200 Subject: [PATCH 20/23] Added length=0 optimization --- ts/reader/type/fixedBytes.ts | 3 +++ ts/reader/type/fixedString.ts | 3 +++ 2 files changed, 6 insertions(+) diff --git a/ts/reader/type/fixedBytes.ts b/ts/reader/type/fixedBytes.ts index 724d805..a232b72 100644 --- a/ts/reader/type/fixedBytes.ts +++ b/ts/reader/type/fixedBytes.ts @@ -2,6 +2,9 @@ import IReader from '../IReader' import createFixedReader from '../util/createFixedReader' export default function fixedBytes (length: number): IReader { + if (length === 0) { + return createFixedReader(0, (view: DataView, byteOffset: number) => new Uint8Array()) + } if (isNaN(length)) { throw new Error(`A fixed string needs a length: ${length}`) } diff --git a/ts/reader/type/fixedString.ts b/ts/reader/type/fixedString.ts index 5b8645d..fe70e7f 100644 --- a/ts/reader/type/fixedString.ts +++ b/ts/reader/type/fixedString.ts @@ -4,6 +4,9 @@ import decodeUtf8 from 'decode-utf8' import fixedBytes from './fixedBytes' export default function fixedString (length: number): IReader { + if (length === 0) { + return createFixedReader(0, (view: DataView, byteOffset: number) => '') + } if (isNaN(length)) { throw new Error(`A fixed string needs a length: ${length}`) } From 98afafb319255b0fdb2e87b2df496c34f5927023 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Tue, 21 Aug 2018 14:42:49 +0200 Subject: [PATCH 21/23] Added limit to how many items to read from a stream. --- ts/reader/readFromStream.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ts/reader/readFromStream.ts b/ts/reader/readFromStream.ts index 0e46066..c0e8779 100644 --- a/ts/reader/readFromStream.ts +++ b/ts/reader/readFromStream.ts @@ -10,8 +10,9 @@ function combine (a: Uint8Array, b: Uint8Array) { return combined } -function readFixedSize (out: Stream, inStream: ReadableStream, reader: IReader) { +function readFixedSize (out: Stream, inStream: ReadableStream, reader: IReader, limit: number = undefined) { let leftOver = null + let count = 0 return inStream.forEach((data: Uint8Array) => { let end = reader.minSize let start = 0 @@ -19,10 +20,11 @@ function readFixedSize (out: Stream, inStream: ReadableStream, data = combine(leftOver, data) } let entries: any[] - while (end <= data.length) { + while (end <= data.length && (limit === undefined || count < limit)) { if (entries === undefined) { entries = [] } + count += 1 entries.push(reader.read(new DataView(data.buffer), start)) start = end end += reader.minSize @@ -45,11 +47,12 @@ const workContext: IDynamicContext = { data: null } -function readDynamicSize (out: Stream, inStream: ReadableStream, reader: IReader) { +function readDynamicSize (out: Stream, inStream: ReadableStream, reader: IReader, limit: number = undefined) { let leftOver = null const context = { byteOffset: 0 } + let count = 0 return inStream.forEach((data: Uint8Array) => { let end = reader.minSize workContext.byteOffset = 0 @@ -57,13 +60,14 @@ function readDynamicSize (out: Stream, inStream: ReadableStream data = combine(leftOver, data) } let entries: any[] - while (end <= data.length) { + while (end <= data.length && (limit === undefined || count < limit)) { if (entries === undefined) { entries = [] } if (!reader.readDynamic(new DataView(data.buffer), workContext)) { break } + count += 1 entries.push(workContext.data) end = workContext.byteOffset + workContext.size + reader.minSize } @@ -79,11 +83,11 @@ function readDynamicSize (out: Stream, inStream: ReadableStream }).then(() => leftOver) } -export default function readFromStream (inStream: ReadableStream, reader: IReader): ReadableStream { +export default function readFromStream (inStream: ReadableStream, reader: IReader, limit: number = undefined): ReadableStream { const out = new Stream() let process = reader.fixedSize - ? readFixedSize(out, inStream, reader) - : readDynamicSize(out, inStream, reader) + ? readFixedSize(out, inStream, reader, limit) + : readDynamicSize(out, inStream, reader, limit) process .then((leftOver) => out.end(null, leftOver)) From 80b82ac71795449c06010e534cb0dcfb6841b3bf Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Tue, 21 Aug 2018 14:43:30 +0200 Subject: [PATCH 22/23] Fixed missing type info in error --- ts/reader/util/readVarUint32.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ts/reader/util/readVarUint32.ts b/ts/reader/util/readVarUint32.ts index 0dad120..d588981 100644 --- a/ts/reader/util/readVarUint32.ts +++ b/ts/reader/util/readVarUint32.ts @@ -6,7 +6,7 @@ export default function readVarUint32 (view: DataView, context: IDynamicContext) return false } if (context.size === 6) { - throw new Error('Invalid int-32 encoding!') + throw new Error('Invalid var-uint-32 encoding!') } return true } From 6385fe3eca506db741213e662991627e3ca7f97c Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Tue, 21 Aug 2018 14:49:27 +0200 Subject: [PATCH 23/23] Extracting AbstractSingleTreeIO that has a tree template. --- ts/raw/RawIO.ts | 37 ++++++++------------------------- ts/util/AbstractSingleTreeIO.ts | 34 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 28 deletions(-) create mode 100644 ts/util/AbstractSingleTreeIO.ts diff --git a/ts/raw/RawIO.ts b/ts/raw/RawIO.ts index ee432f3..be4e635 100644 --- a/ts/raw/RawIO.ts +++ b/ts/raw/RawIO.ts @@ -11,7 +11,7 @@ import IBox3 from '../api/IBox3' import IFeature from '../api/IFeature' import FeatureType from '../api/FeatureType' import Feature from '../api/Feature' -import AbstractIO from '../util/AbstractIO' +import AbstractSingleTreeIO from '../util/AbstractSingleTreeIO' import expandBox from '../util/expandBox' import featureMatch from '../util/featureMatch' @@ -82,47 +82,28 @@ class TreeInfo implements ITree { } } -export default class RawIO extends AbstractIO implements IPNextIO { - info: TreeInfo +export default class RawIO extends AbstractSingleTreeIO implements IPNextIO { pointData: IPoint[][] ids: number[] constructor (id: string, pointData: IPoint[][]) { - super() - this.info = new TreeInfo(id, pointData) + super(Promise.resolve(new TreeInfo(id, pointData))) this.pointData = pointData this.ids = pointData.map((value: IPoint[], index: number) => index) } - getTrees (query?: ITreeQuery): Stream { - const stream = new Stream() - setImmediate(() => { - if (query && query.ids) { - for (const id of query.ids) { - if (id !== this.info.id) { - stream.end(new Error(`Unknown tree ${id}`)).catch(ignoreError) - return - } - } - } - stream.write(this.info) - stream.end() - }) - return stream - } - getNodes (query?: INodeQuery): Stream { const stream = new Stream() let first = true - setImmediate(() => { + this.treeP.then(tree => { let index = 0 for (const points of this.pointData) { const node: INode = { id: index.toString(), - numPoints: this.info.numPoints + numPoints: tree.numPoints } if (first) { - node.treeId = this.info.id + node.treeId = tree.id } stream.write(node) index ++ @@ -134,12 +115,12 @@ export default class RawIO extends AbstractIO implements IPNextIO { getPoints (query?: IPointQuery): Stream<{ [k: string]: any; }> { const stream = new Stream<{ [k: string ]: any }>() - setImmediate(() => { + this.treeP.then(tree => { let ids: number[] = this.ids if (query && query.nodes) { ids = [] for (const node of query.nodes) { - if (node.treeId !== this.info.id) { + if (node.treeId !== tree.id) { stream.end(new Error(`Invalid tree id "${node.treeId}" requested!`)).catch(ignoreError) return } @@ -159,7 +140,7 @@ export default class RawIO extends AbstractIO implements IPNextIO { } } if (query && query.schema) { - const error = featureMatch(this.info.schema, query.schema) + const error = featureMatch(tree.schema, query.schema) if (error) { stream.end(new Error(error)).catch(ignoreError) return diff --git a/ts/util/AbstractSingleTreeIO.ts b/ts/util/AbstractSingleTreeIO.ts new file mode 100644 index 0000000..9eb0d15 --- /dev/null +++ b/ts/util/AbstractSingleTreeIO.ts @@ -0,0 +1,34 @@ +import AbstractIO from '../util/AbstractIO' +import ITree from '../api/ITree' +import ITreeQuery from '../api/ITreeQuery' +import Stream from 'ts-stream' + +function ignoreError () { + return +} + +export default class AbstractSingleTreeIO extends AbstractIO { + treeP: Promise + + constructor (treeP: Promise) { + super() + this.treeP = treeP + } + + getTrees (query?: ITreeQuery): Stream { + const stream = new Stream() + this.treeP.then(tree => { + if (query && query.ids) { + for (const id of query.ids) { + if (id !== tree.id) { + stream.end(new Error(`Unknown tree ${id}`)).catch(ignoreError) + return + } + } + } + stream.write(tree) + stream.end() + }) + return stream + } +}