WIP - Reader system for processing data

package.json

@@ -6,7 +6,7 @@
   "main": "index.js",
   "scripts": {
     "test": "npm run test:lint && npm run test:unit",
-    "test:unit": "tap -- ts_test/*.ts",
+    "test:unit": "tap -- ts_test/*.ts ts_test/**/*.ts",
     "test:lint": "tslint -c .tslintrc.json ts/**/*.ts && standard",
     "build:types-js": "pbjs -t static-module -w commonjs -l \"eslint-disable one-var, no-mixed-operators\" --es6 spec/*.proto -o ts/Types.js && standard --fix ts/Types.js",
     "build:types-ts": "npm run build:types-js && pbts ts/Types.js -o ts/Types.d.ts && tslint -c .tslintrc.json --fix ts/Types.d.ts",
@@ -48,6 +48,8 @@
     "typescript-eslint-parser": "^16.0.1"
   },
   "dependencies": {
+    "bluebird": "^3.5.1",
+    "decode-utf8": "^1.0.1",
     "long": "^4.0.0",
     "protobufjs": "^6.8.6",
     "three": "^0.94.0",

spec/FeatureType.proto

@@ -1,27 +1,37 @@
 enum FeatureType {
-  uint8 = 1;
-  uint16 = 2;
-  uint32 = 3;
-  uint64 = 4;
-  int8 = 5;
-  int16 = 6;
-  int32 = 7;
-  int64 = 8;
-  sint8 = 9;
-  sint16 = 10;
-  sint32 = 11;
-  sint64 = 12;
-  fixedint8 = 13;
-  fixedint16 = 14;
-  fixedint32 = 15;
-  fixedint64 = 16;
-  sfixedint8 = 17;
-  sfixedint16 = 18;
-  sfixedint32 = 19;
-  sfixedint64 = 20;
-  string = 21 [(length) = -1];
-  bytes = 22 [(length) = -1];
-  bool = 23;
-  float = 24;
-  double = 25;
+  int8 = 1;                        // fixed size 1 byte number: -127~128
+  uint8 = 2;                       // fixed size 1 byte number: 0~255
+  int16 = 3;                       // fixed size 2 byte number: -32768~32768
+  int16LE = 4;                     // fixed size 2 byte number: -32768~32768
+  uint16 = 5;                      // fixed size 2 byte number: 0~65536
+  uint16LE = 6;                    // fixed size 2 byte number: 0~255 + (0~255)<<16 => 0~665536
+  int32 = 7;                       // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 -2147483648 => 2147483648~-2147483648
+  int32LE = 8;                     // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 -2147483648 => 2147483648~-2147483648
+  uint32 = 9;                      // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 => 0~4294967296
+  uint32LE = 10;                   // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 => 0~4294967296
+  sint32 = 11;                     // fixed size 4 byte, reverse stored number: -2147483648~2147483648
+  sint32LE = 12;                   // fixed size 4 byte, reverse stored number: -2147483648~2147483648
+  varint32 = 13;                   // variable size 2~5 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15) => 2147483648~-2147483648
+  varuint32 = 14;                  // variable size 2~5 byte, reverse stored number: 0~4294967296
+  varsint32 = 15;                  // variable size 2~5 byte, reverse stored number: -2147483648~2147483648
+  int64 = 16;                      // fixed size 8 byte number: -9223372036854776000~9223372036854776000
+  int64LE = 17;                    // fixed size 8 byte number: -9223372036854776000~9223372036854776000
+  uint64 = 18;                     // fixed size 8 byte number: 0~18446744073709552000
+  uint64LE = 19;                   // fixed size 8 byte number: 0~18446744073709552000
+  sint64 = 20;                     // fixed size 8 byte, reverse stored number: 9223372036854776000~-9223372036854776000
+  sint64LE = 21;                   // fixed size 8 byte, reverse stored number: 9223372036854776000~-9223372036854776000
+  varuint64 = 22;                  // variable size 2~9 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15) => 9223372036854776000~-9223372036854776000
+  varint64 = 23;                   // variable size 2~9 byte number: 0~18446744073709552000
+  varsint64 = 24;                  // variable size 2~9 byte, reverse stored number: -9223372036854776000~9223372036854776000
+  bytes = 25;                      // variable size bytes: uint32-size-number + amount of bytes
+  bytesLE = 26;                    // variable size bytes: uint32LE-size-number + amount of bytes
+  varbytes = 27;                   // variable size bytes with varsize length: varuint32-number + amount of bytes
+  string = 28;                     // variable size string: utf-8 encoded bytes
+  stringLE = 29;                   // variable size string: utf-8 encoded bytes
+  varstring = 30;                  // variable size string with varsize length: utf-8 encoded varbytes
+  bool = 31;                       // 0 = false, 1~7 = true
+  float = 32;                      // fixed size 4 byte floating point
+  double = 33;                     // fixed size 8 byte floating point
+  fixedstring = 34[(length) = -1]; // fixed amount of string
+  fixedbytes = 35[(length) = -1];  // fixed amount of bytes
 }

ts/api/Feature.ts

@@ -3,7 +3,7 @@ import FeatureType from './FeatureType'
 
 const { double, uint8 } = FeatureType
 
-const Feature: { [k: string]: IFeature } = {
+export default {
   x: { name: 'x', type: double },
   y: { name: 'y', type: double },
   z: { name: 'z', type: double },
@@ -12,5 +12,3 @@ const Feature: { [k: string]: IFeature } = {
   b: { name: 'b', type: uint8 },
   a: { name: 'a', type: uint8 }
 }
-
-export default Feature

ts/api/FeatureType.ts

@@ -1,31 +1,75 @@
 enum FeatureType {
-  uint8 = 1,
-  uint16 = 2,
-  uint32 = 3,
-  uint64 = 4,
-  int8 = 5,
-  int16 = 6,
-  int32 = 7,
-  int64 = 8,
-  sint8 = 9,
-  sint16 = 10,
-  sint32 = 11,
-  sint64 = 12,
-  fixedint8 = 13,
-  fixedint16 = 14,
-  fixedint32 = 15,
-  fixedint64 = 16,
-  sfixedint8 = 17,
-  sfixedint16 = 18,
-  sfixedint32 = 19,
-  sfixedint64 = 20,
-  string = 21,
-  bytes = 22,
-  bool = 23,
-  float = 24,
-  double = 25,
-  varstring = 26,
-  varbytes = 27
+  int8 = 1,         // fixed size 1 byte number: -127~128
+  uint8 = 2,        // fixed size 1 byte number: 0~255
+
+  int16 = 3,        // fixed size 2 byte number: -32768~32768
+  int16LE = 4,      // fixed size 2 byte number: -32768~32768 (little endian)
+  uint16 = 5,       // fixed size 2 byte number: 0~255 + (0~255)<<16 => 0~65536
+  uint16LE = 6,     // fixed size 2 byte number: 0~65536 (little endian)
+
+  int32 = 7,        // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 -2147483648 => 2147483648~-2147483648
+  int32LE = 8,      // fixed size 4 byte number: 2147483648~-2147483648 (little endian)
+  uint32 = 9,       // fixed size 4 byte number: 0~255 + (0~255)<<16 + (0~255)<<24 + (0~256)<<32 => 0~4294967296
+  uint32LE = 10,    // fixed size 4 byte number: 0~4294967296 (little endian)
+  sint32 = 11,      // fixed size 4 byte, reverse stored number: -2147483648~2147483648
+  sint32LE = 12,    // fixed size 4 byte, reverse stored number: -2147483648~2147483648 (little endian)
+  varint32 = 13,    // variable size 1~5 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15) => 2147483648~-2147483648
+  varuint32 = 14,   // variable size 1~5 byte number: (same like varint) 0~4294967296
+  varsint32 = 15,   // variable size 1~5 byte, reverse stored number: -2147483648~2147483648
+
+  int64 = 16,       // fixed size 8 byte number: -9223372036854776000~9223372036854776000
+  int64LE = 17,     // fixed size 8 byte number: -9223372036854776000~9223372036854776000 (little endian)
+  uint64 = 18,      // fixed size 8 byte number: 0~18446744073709552000
+  uint64LE = 19,    // fixed size 8 byte number: 0~18446744073709552000 (little endian)
+  sint64 = 20,      // fixed size 8 byte, reverse stored number: 9223372036854776000~-9223372036854776000
+  sint64LE = 21,    // fixed size 8 byte, reverse stored number: 9223372036854776000~-9223372036854776000 (little endian)
+  varint64 = 22,    // variable size 1~9 byte number: [0~127 + (1bit] + [(0~127)<<7) + (1bit] + (0~127)<<14) + (1bit] + [0~127) + (1bit] + 0~15)... => 9223372036854776000~-9223372036854776000
+  varuint64 = 23,   // variable size 1~9 byte number: (same like varint) 0~18446744073709552000
+  varsint64 = 24,   // variable size 1~9 byte, reverse stored number: -9223372036854776000~9223372036854776000
+
+  bytes = 25,       // variable size bytes: uint32-size-number + amount of bytes
+  bytesLE = 26,     // variable size bytes: uint32LE-size-number + amount of bytes
+  varbytes = 27,    // variable size bytes: varint32-size-number + amount of bytes
+  string = 28,      // variable size string: utf-8 encoded bytes
+  stringLE = 29,    // variable size string: utf-8 encoded bytesLE
+  varstring = 30,   // variable size string: utf-8 encoded varbytes
+
+  bool = 31,        // 0 = false, 1~7 = true
+
+  float = 32,       // fixed size 4 byte floating point
+  double = 33,      // fixed size 8 byte floating point
+
+  fixedstring = 34, // fixed amount of string
+  fixedbytes = 35   // fixed amount of bytes
 }
 
 export default FeatureType
+
+export function parseFeatureString (type: string): FeatureType {
+  switch (type) {
+    case 'int8': return FeatureType.int8
+    case 'uint8': return FeatureType.uint8
+    case 'int16': return FeatureType.int16
+    case 'uint16': return FeatureType.uint16
+    case 'int32': return FeatureType.int32
+    case 'uint32': return FeatureType.uint32
+    case 'sint32': return FeatureType.sint32
+    case 'varint32': return FeatureType.varint32
+    case 'varuint32': return FeatureType.varuint32
+    case 'varsint32': return FeatureType.varsint32
+    case 'int64': return FeatureType.int64
+    case 'uint64': return FeatureType.uint64
+    case 'sint64': return FeatureType.sint64
+    case 'varint64': return FeatureType.varint64
+    case 'varuint64': return FeatureType.varuint64
+    case 'varsint64': return FeatureType.varsint64
+    case 'bytes': return FeatureType.bytes
+    case 'string': return FeatureType.string
+    case 'bool': return FeatureType.bool
+    case 'float': return FeatureType.float
+    case 'double': return FeatureType.double
+    case 'fixedstring': return FeatureType.fixedstring
+    case 'fixedbytes': return FeatureType.fixedbytes
+  }
+  throw new Error(`Unknown feature type ${type}`)
+}

ts/raw/RawIO.ts

@@ -11,7 +11,7 @@ import IBox3 from '../api/IBox3'
 import IFeature from '../api/IFeature'
 import FeatureType from '../api/FeatureType'
 import Feature from '../api/Feature'
-import AbstractIO from '../util/AbstractIO'
+import AbstractSingleTreeIO from '../util/AbstractSingleTreeIO'
 import expandBox from '../util/expandBox'
 import featureMatch from '../util/featureMatch'
 
@@ -82,47 +82,28 @@ class TreeInfo implements ITree {
   }
 }
 
-export default class RawIO extends AbstractIO implements IPNextIO {
-  info: TreeInfo
+export default class RawIO extends AbstractSingleTreeIO implements IPNextIO {
   pointData: IPoint[][]
   ids: number[]
 
   constructor (id: string, pointData: IPoint[][]) {
-    super()
-    this.info = new TreeInfo(id, pointData)
+    super(Promise.resolve(new TreeInfo(id, pointData)))
     this.pointData = pointData
     this.ids = pointData.map((value: IPoint[], index: number) => index)
   }
 
-  getTrees (query?: ITreeQuery): Stream<ITree> {
-    const stream = new Stream<ITree>()
-    setImmediate(() => {
-      if (query && query.ids) {
-        for (const id of query.ids) {
-          if (id !== this.info.id) {
-            stream.end(new Error(`Unknown tree ${id}`)).catch(ignoreError)
-            return
-          }
-        }
-      }
-      stream.write(this.info)
-      stream.end()
-    })
-    return stream
-  }
-
   getNodes (query?: INodeQuery): Stream<INode> {
     const stream = new Stream<INode>()
     let first = true
-    setImmediate(() => {
+    this.treeP.then(tree => {
       let index = 0
       for (const points of this.pointData) {
         const node: INode = {
           id: index.toString(),
-          numPoints: this.info.numPoints
+          numPoints: tree.numPoints
         }
         if (first) {
-          node.treeId = this.info.id
+          node.treeId = tree.id
         }
         stream.write(node)
         index ++
@@ -134,12 +115,12 @@ export default class RawIO extends AbstractIO implements IPNextIO {
 
   getPoints (query?: IPointQuery): Stream<{ [k: string]: any; }> {
     const stream = new Stream<{ [k: string ]: any }>()
-    setImmediate(() => {
+    this.treeP.then(tree => {
       let ids: number[] = this.ids
       if (query && query.nodes) {
         ids = []
         for (const node of query.nodes) {
-          if (node.treeId !== this.info.id) {
+          if (node.treeId !== tree.id) {
             stream.end(new Error(`Invalid tree id "${node.treeId}" requested!`)).catch(ignoreError)
             return
           }
@@ -159,7 +140,7 @@ export default class RawIO extends AbstractIO implements IPNextIO {
         }
       }
       if (query && query.schema) {
-        const error = featureMatch(this.info.schema, query.schema)
+        const error = featureMatch(tree.schema, query.schema)
         if (error) {
           stream.end(new Error(error)).catch(ignoreError)
           return

ts/reader/IReader.ts

@@ -0,0 +1,8 @@
+import IDynamicContext from './util/IDynamicContext'
+
+export default interface IReader {
+  fixedSize: boolean
+  minSize: number
+  readDynamic (view: DataView, context: IDynamicContext): boolean
+  read (view: DataView, byteOffset: number): any
+}