[WIP] Ept implementation

package.json

@@ -6,7 +6,7 @@
   "main": "index.js",
   "scripts": {
     "test": "npm run test:lint && npm run test:unit",
-    "test:unit": "tap -- ts_test/*.ts",
+    "test:unit": "tap -- ts_test/*.ts ts_test/**/*.ts",
     "test:lint": "tslint -c .tslintrc.json ts/**/*.ts && standard",
     "build:types-js": "pbjs -t static-module -w commonjs -l \"eslint-disable one-var, no-mixed-operators\" --es6 spec/*.proto -o ts/Types.js && standard --fix ts/Types.js",
     "build:types-ts": "npm run build:types-js && pbts ts/Types.js -o ts/Types.d.ts && tslint -c .tslintrc.json --fix ts/Types.d.ts",
@@ -48,6 +48,7 @@
     "typescript-eslint-parser": "^16.0.1"
   },
   "dependencies": {
+    "bluebird": "^3.5.1",
     "long": "^4.0.0",
     "protobufjs": "^6.8.6",
     "three": "^0.94.0",

ts/api/FeatureType.ts

@@ -29,3 +29,36 @@ enum FeatureType {
 }
 
 export default FeatureType
+
+export function parseFeatureString (type: string): FeatureType {
+  switch (type) {
+    case 'bool': return FeatureType.bool
+    case 'bytes': return FeatureType.bytes
+    case 'double': return FeatureType.double
+    case 'fixedint16': return FeatureType.fixedint16
+    case 'fixedint32': return FeatureType.fixedint32
+    case 'fixedint64': return FeatureType.fixedint64
+    case 'fixedint8': return FeatureType.fixedint8
+    case 'float': return FeatureType.float
+    case 'int16': return FeatureType.int16
+    case 'int32': return FeatureType.int32
+    case 'int64': return FeatureType.int64
+    case 'int8': return FeatureType.int8
+    case 'sfixedint16': return FeatureType.sfixedint16
+    case 'sfixedint32': return FeatureType.sfixedint32
+    case 'sfixedint64': return FeatureType.sfixedint64
+    case 'sfixedint8': return FeatureType.sfixedint8
+    case 'sint16': return FeatureType.sint16
+    case 'sint32': return FeatureType.sint32
+    case 'sint64': return FeatureType.sint64
+    case 'sint8': return FeatureType.sint8
+    case 'string': return FeatureType.string
+    case 'uint16': return FeatureType.uint16
+    case 'uint32': return FeatureType.uint32
+    case 'uint64': return FeatureType.uint64
+    case 'uint8': return FeatureType.uint8
+    case 'varbytes': return FeatureType.varbytes
+    case 'varstring': return FeatureType.varstring
+  }
+  throw new Error(`Unknown feature type ${type}`)
+}

ts/api/INode.ts

@@ -4,4 +4,5 @@ import IBox3 from './IBox3'
 export default interface INode extends INodeSelector {
   readonly numPoints: (number | Long)
   readonly bounds?: IBox3
+  readonly spacing?: number
 }

ts/format/IInput.ts

@@ -0,0 +1,4 @@
+export default interface IInput {
+  id (): string
+  loadJson (filename: string): Promise<any>
+}

ts/format/ept/index.ts

@@ -0,0 +1,104 @@
+import IPNextIO from '../../api/IPNextIO'
+import ITreeQuery from '../../api/ITreeQuery'
+import INodeQuery from '../../api/INodeQuery'
+import IPointQuery from '../../api/IPointQuery'
+import INode from '../../api/INode'
+import ITree from '../../api/ITree'
+import IFeature from '../../api/IFeature'
+import AbstractIO from '../../util/AbstractIO'
+import { Stream, ReadableStream } from 'ts-stream'
+import selectNodes from '../../util/selectNodes'
+import IInput from '../IInput'
+import INodeTree from '../../util/INodeTree'
+import loadTree from './tree'
+import loadRootNodes from './nodes'
+import INodeSelector from '../../api/INodeSelector'
+import IReader from '../../reader/IReader'
+
+function createBinReader (schema: IFeature[], request: IFeature[]): IReader {
+
+}
+
+function createReader (dataType: string, schema: IFeature[]): IReader {
+  if (dataType === 'bin') {
+    return createBinReader(schema)
+  }
+  throw new Error(`Unsupported dataType: ${dataType}`)
+}
+
+export class EPT extends AbstractIO implements IPNextIO {
+
+  tree: Promise<ITree>
+  input: IInput
+  rootNodes: Promise<INodeTree[]>
+
+  constructor (input: IInput) {
+    super()
+    this.input = input
+  }
+
+  getTrees (query?: ITreeQuery): ReadableStream <ITree> {
+    return Stream.from([this.loadTree()])
+  }
+
+  loadTree (): Promise<ITree> {
+    if (!this.tree) {
+      this.tree = loadTree(this.input)
+    }
+    return this.tree
+  }
+
+  loadReader (): Promise<IReader> {
+    return this.loadTree()
+      .then((tree: ITree) => createReader(tree.metadata.dataType, tree.schema))
+  }
+
+  getNodes (query?: INodeQuery): ReadableStream <INode> {
+    const stream = new Stream<INode>()
+    this.loadRootNodes().then(rootTreeNodes => {
+      selectNodes(query, rootTreeNodes, {
+        addNode (node: INode): void {
+          stream.write(node)
+        },
+        isClosed: (): boolean => stream.isEnded(),
+        end () {
+          stream.end()
+        }
+      })
+    })
+    return stream
+  }
+
+  loadRootNodes (): Promise<INodeTree[]> {
+    if (!this.rootNodes) {
+      this.rootNodes = this.loadTree()
+        .then((tree: ITree) => loadRootNodes(this.input, tree))
+    }
+    return this.rootNodes
+  }
+
+  loadNodes (selectors?: INodeSelector[]): ReadableStream<INode> {
+    if (selectors) {
+      // TODO: Implement me!
+    }
+    return this.getNodes()
+  }
+
+  getNodePoints (node: INode, treeFeatures: IReader, output: Stream<{ [k: string]: any }>): void {
+    this.input.binaryStream(`${node.id}`)
+  }
+
+  getPoints (query?: IPointQuery): ReadableStream<{ [k: string]: any }> {
+    const pointStream = new Stream<{ [k: string]: any }>()
+    this.loadReader()
+      .then(reader => {
+        const nodeStream = this.loadNodes(query.nodes)
+        nodeStream.forEach(node =>
+          this.getNodePoints(node, reader, pointStream)
+        ).then(() => {
+          pointStream.end()
+        })
+      })
+    return pointStream
+  }
+}

ts/format/ept/nodes.ts

@@ -0,0 +1,104 @@
+import { slippyToString, SlippyMap, parseSlippy, slippyParentString, slippyBounds, slippySphere, SLIPPY_ZERO } from '../../util/slippy'
+import INodeTree from '../../util/INodeTree'
+import ITree from '../../api/ITree'
+import IInput from '../IInput'
+import IBox3 from '../../api/IBox3'
+import dot from '../../util/dot'
+import addToPoint from '../../util/addToPoint'
+import differencePointPoint from '../../util/differencePointPoint'
+import multiplyWithPoint from '../../util/multiplyWithPoint'
+import IVector3 from '../../api/IVector3'
+import ISphere from '../../util/ISphere'
+import distancePointPoint from '../../util/distancePointPoint'
+
+function nullOp (): null {
+  return null
+}
+
+interface INodeInfo {
+  size: IVector3
+  offset: IVector3
+  sphereRadius: number
+  hierarchyStep: number
+  ticks: number
+}
+
+function scaleAndOffset (point: IVector3, nodeInfo: INodeInfo): IVector3 {
+  return addToPoint(multiplyWithPoint(point, nodeInfo.size), nodeInfo.offset)
+}
+
+function eptSpacing (loc: SlippyMap, nodeInfo: INodeInfo): number {
+
+}
+
+function eptBounds (loc: SlippyMap, nodeInfo: INodeInfo): IBox3 {
+  const bounds = slippyBounds(loc)
+  scaleAndOffset(bounds.min, nodeInfo)
+  scaleAndOffset(bounds.max, nodeInfo)
+  return bounds
+}
+
+function eptSphere (loc: SlippyMap, nodeInfo: INodeInfo): ISphere {
+  const sphere = slippySphere(loc)
+  sphere.radius *= nodeInfo.sphereRadius
+  scaleAndOffset(sphere.center, nodeInfo)
+  return sphere
+}
+
+async function loadNodes (input: IInput, parent: SlippyMap, parentId: string, nodeInfo: INodeInfo): Promise<INodeTree[]> {
+  const id = slippyToString(parent)
+  const byParent: { [k: string]: (INodeTree[] | undefined) } = {}
+  const children: INodeTree[] = []
+  byParent[parentId] = children
+  const dmax: number = parent.d + nodeInfo.hierarchyStep
+  const counts: { [k: string]: number } = await input.loadJson(`${id}.json`)
+  for (const childId in counts) {
+    const numPoints = counts[childId]
+    const loc = parseSlippy(childId)
+    const parentId = slippyParentString(loc)
+    let siblings = byParent[parentId]
+    if (siblings === undefined) {
+      siblings = []
+      byParent[parentId] = siblings
+    }
+    const nodeTree: INodeTree = {
+      node: {
+        numPoints,
+        bounds: eptBounds(loc, nodeInfo),
+        spacing: eptSpacing(loc, nodeInfo)
+      },
+      boundingSphere: eptSphere(loc, nodeInfo),
+      children: nullOp
+    }
+    if (loc.d === dmax) {
+      let subChildren: Promise<INodeTree[]>
+      nodeTree.children = (): Promise<INodeTree[]> => {
+        if (!subChildren) {
+          subChildren = loadNodes(input, loc, childId, nodeInfo)
+        }
+        return subChildren
+      }
+    }
+    siblings.push(nodeTree)
+  }
+  return children
+}
+
+export default function loadRootNodes (input: IInput, tree: ITree): Promise<INodeTree[]> {
+  // There is no deeper hierarchy step
+  let hierarchyStep = Infinity
+  if (tree.metadata && !isNaN(tree.metadata.hierarchyStep)) {
+    hierarchyStep = tree.metadata.hierarchyStep
+  }
+  if (tree.metadata && tree.metadata.hierarchyType && tree.metadata.hierarchyType !== 'json') {
+    throw new Error('Only supported hierarchy type is json')
+  }
+  const nodeInfo = {
+    size: differencePointPoint(tree.bounds.max, tree.bounds.min),
+    sphereRadius: distancePointPoint(tree.bounds.max, tree.bounds.min),
+    offset: tree.bounds.min,
+    hierarchyStep,
+    ticks: tree.metadata.ticks
+  }
+  return loadNodes(input, SLIPPY_ZERO, '0-0-0-0', nodeInfo)
+}

ts/format/ept/tree.ts

@@ -0,0 +1,54 @@
+import IBox3 from '../../api/IBox3'
+import IVector3 from '../../api/IVector3'
+import { parseFeatureString } from '../../api/FeatureType'
+import IFeature from '../../api/IFeature'
+import Feature from '../../api/Feature'
+import ITree from '../../api/ITree'
+import IInput from '../IInput'
+
+function parseFeature (raw: {name: string, type: string }): IFeature {
+  const type = parseFeatureString(raw.type)
+  for (const featName in Feature) {
+    const feat = Feature[featName]
+    if (feat.name === raw.name && feat.type === type) {
+      return feat
+    }
+  }
+  return {
+    name: raw.name,
+    type
+  }
+}
+
+function parseBounds (raw: number[]): IBox3 {
+  return {
+    min: parseVector(raw, 0),
+    max: parseVector(raw, 3)
+  }
+}
+
+function parseVector (raw: number[], offset: number): IVector3 {
+  return { x: raw[offset], y: raw[offset + 1], z: raw[offset + 2] }
+}
+
+export default async function loadTree (input: IInput): Promise<ITree> {
+  const json = await input.loadJson('entwine.json')
+  return {
+    id: input.id(),
+    bounds: parseBounds(json.bounds),
+    boundsConforming: parseBounds(json.boundsConforming),
+    scale: json.scale,
+    offset: parseVector(json.offset, 0),
+    numPoints: json.numPoints,
+    schema: json.schema.map(parseFeature),
+    metadata: {
+      ...json.metadata,
+      dataType: json.dataType,
+      hierarchyStep: json.hierarchyStep,
+      hierarchyType: json.hierarchyType,
+      reprojection: json.reprojection,
+      srs: json.srs,
+      ticks: json.ticks
+    }
+  }
+}

ts/util/INodeTree.ts

@@ -1,8 +1,8 @@
 import INode from '../api/INode'
-import { Sphere } from 'three'
+import ISphere from './ISphere'
 
 export default interface INodeTree {
   node: INode,
-  boundingSphere: Sphere,
-  children: INodeTree[]
+  boundingSphere: ISphere,
+  children (): PromiseLike<INodeTree[]> | null
 }

ts/util/addToPoint.ts

@@ -0,0 +1,8 @@
+import IVector3 from '../api/IVector3'
+
+export default function addToPoint (toPoint: IVector3, add: IVector3): IVector3 {
+  toPoint.x += add.x
+  toPoint.y += add.y
+  toPoint.z += add.z
+  return toPoint
+}

ts/util/differencePointPoint.ts

@@ -0,0 +1,9 @@
+import IVector3 from '../api/IVector3'
+
+export default function distancePointPoint (a: IVector3, b: IVector3): IVector3 {
+  return {
+    x: a.x - b.x,
+    y: a.y - b.y,
+    z: a.z - b.z
+  }
+}

ts/util/multiplyWithPoint.ts

@@ -0,0 +1,8 @@
+import IVector3 from '../api/IVector3'
+
+export default function multiplyWithPoint (toPoint: IVector3, multiply: IVector3): IVector3 {
+  toPoint.x *= multiply.x
+  toPoint.y *= multiply.y
+  toPoint.z *= multiply.z
+  return toPoint
+}