pnext · iosefa · Aug 4, 2018 · Aug 4, 2018 · Aug 31, 2018 · Aug 31, 2018
diff --git a/.gitignore b/.gitignore
@@ -1 +1,3 @@
 build
+data/lidar_raleigh_nc_spm_height_feet.las
+data/lidar_raleigh_nc_spm_height_feet.laz
diff --git a/data/NorthCarolina/0-0-0-0.laz b/data/NorthCarolina/0-0-0-0.laz
diff --git a/data/NorthCarolina/1-0-0-0.laz b/data/NorthCarolina/1-0-0-0.laz
diff --git a/data/NorthCarolina/1-0-0-1.laz b/data/NorthCarolina/1-0-0-1.laz
diff --git a/data/NorthCarolina/1-0-1-0.laz b/data/NorthCarolina/1-0-1-0.laz
diff --git a/data/NorthCarolina/1-0-1-1.laz b/data/NorthCarolina/1-0-1-1.laz
diff --git a/data/NorthCarolina/1-1-0-0.laz b/data/NorthCarolina/1-1-0-0.laz
diff --git a/data/NorthCarolina/1-1-0-1.laz b/data/NorthCarolina/1-1-0-1.laz
diff --git a/data/NorthCarolina/1-1-1-0.laz b/data/NorthCarolina/1-1-1-0.laz
diff --git a/data/NorthCarolina/1-1-1-1.laz b/data/NorthCarolina/1-1-1-1.laz
diff --git a/data/NorthCarolina/2-0-0-1.laz b/data/NorthCarolina/2-0-0-1.laz
diff --git a/data/NorthCarolina/2-0-0-2.laz b/data/NorthCarolina/2-0-0-2.laz
diff --git a/data/NorthCarolina/2-0-1-2.laz b/data/NorthCarolina/2-0-1-2.laz
diff --git a/data/NorthCarolina/2-1-0-1.laz b/data/NorthCarolina/2-1-0-1.laz
diff --git a/data/NorthCarolina/2-1-0-2.laz b/data/NorthCarolina/2-1-0-2.laz
diff --git a/data/NorthCarolina/2-1-1-1.laz b/data/NorthCarolina/2-1-1-1.laz
diff --git a/data/NorthCarolina/2-1-1-2.laz b/data/NorthCarolina/2-1-1-2.laz
diff --git a/data/NorthCarolina/2-1-2-2.laz b/data/NorthCarolina/2-1-2-2.laz
diff --git a/data/NorthCarolina/2-2-1-2.laz b/data/NorthCarolina/2-2-1-2.laz
diff --git a/data/NorthCarolina/2-2-2-1.laz b/data/NorthCarolina/2-2-2-1.laz
diff --git a/data/NorthCarolina/2-2-2-2.laz b/data/NorthCarolina/2-2-2-2.laz
diff --git a/data/NorthCarolina/2-2-3-1.laz b/data/NorthCarolina/2-2-3-1.laz
diff --git a/data/NorthCarolina/2-2-3-2.laz b/data/NorthCarolina/2-2-3-2.laz
diff --git a/data/NorthCarolina/2-3-2-1.laz b/data/NorthCarolina/2-3-2-1.laz
diff --git a/data/NorthCarolina/2-3-2-2.laz b/data/NorthCarolina/2-3-2-2.laz
diff --git a/data/NorthCarolina/2-3-3-1.laz b/data/NorthCarolina/2-3-3-1.laz
diff --git a/data/NorthCarolina/2-3-3-2.laz b/data/NorthCarolina/2-3-3-2.laz
diff --git a/data/NorthCarolina/3-0-0-3.laz b/data/NorthCarolina/3-0-0-3.laz
diff --git a/data/NorthCarolina/3-0-0-4.laz b/data/NorthCarolina/3-0-0-4.laz
diff --git a/data/NorthCarolina/3-1-0-3.laz b/data/NorthCarolina/3-1-0-3.laz
diff --git a/data/NorthCarolina/3-1-0-4.laz b/data/NorthCarolina/3-1-0-4.laz
diff --git a/data/NorthCarolina/3-1-1-3.laz b/data/NorthCarolina/3-1-1-3.laz
diff --git a/data/NorthCarolina/3-1-1-4.laz b/data/NorthCarolina/3-1-1-4.laz
diff --git a/data/NorthCarolina/3-2-0-3.laz b/data/NorthCarolina/3-2-0-3.laz
diff --git a/data/NorthCarolina/3-2-0-4.laz b/data/NorthCarolina/3-2-0-4.laz
diff --git a/data/NorthCarolina/3-2-1-3.laz b/data/NorthCarolina/3-2-1-3.laz
diff --git a/data/NorthCarolina/3-2-1-4.laz b/data/NorthCarolina/3-2-1-4.laz
diff --git a/data/NorthCarolina/3-2-2-3.laz b/data/NorthCarolina/3-2-2-3.laz
diff --git a/data/NorthCarolina/3-2-2-4.laz b/data/NorthCarolina/3-2-2-4.laz
diff --git a/data/NorthCarolina/3-2-3-4.laz b/data/NorthCarolina/3-2-3-4.laz
diff --git a/data/NorthCarolina/3-3-1-3.laz b/data/NorthCarolina/3-3-1-3.laz
diff --git a/data/NorthCarolina/3-3-1-4.laz b/data/NorthCarolina/3-3-1-4.laz
diff --git a/data/NorthCarolina/3-3-2-3.laz b/data/NorthCarolina/3-3-2-3.laz
diff --git a/data/NorthCarolina/3-3-2-4.laz b/data/NorthCarolina/3-3-2-4.laz
diff --git a/data/NorthCarolina/3-3-3-4.laz b/data/NorthCarolina/3-3-3-4.laz
diff --git a/data/NorthCarolina/3-3-4-4.laz b/data/NorthCarolina/3-3-4-4.laz
diff --git a/data/NorthCarolina/3-3-5-4.laz b/data/NorthCarolina/3-3-5-4.laz
diff --git a/data/NorthCarolina/3-4-2-4.laz b/data/NorthCarolina/3-4-2-4.laz
diff --git a/data/NorthCarolina/3-4-3-4.laz b/data/NorthCarolina/3-4-3-4.laz
diff --git a/data/NorthCarolina/3-4-4-3.laz b/data/NorthCarolina/3-4-4-3.laz
diff --git a/data/NorthCarolina/3-4-4-4.laz b/data/NorthCarolina/3-4-4-4.laz
diff --git a/data/NorthCarolina/3-4-5-3.laz b/data/NorthCarolina/3-4-5-3.laz
diff --git a/data/NorthCarolina/3-4-5-4.laz b/data/NorthCarolina/3-4-5-4.laz
diff --git a/data/NorthCarolina/3-4-6-3.laz b/data/NorthCarolina/3-4-6-3.laz
diff --git a/data/NorthCarolina/3-4-6-4.laz b/data/NorthCarolina/3-4-6-4.laz
diff --git a/data/NorthCarolina/3-5-4-3.laz b/data/NorthCarolina/3-5-4-3.laz
diff --git a/data/NorthCarolina/3-5-4-4.laz b/data/NorthCarolina/3-5-4-4.laz
diff --git a/data/NorthCarolina/3-5-5-3.laz b/data/NorthCarolina/3-5-5-3.laz
diff --git a/data/NorthCarolina/3-5-5-4.laz b/data/NorthCarolina/3-5-5-4.laz
diff --git a/data/NorthCarolina/3-5-6-3.laz b/data/NorthCarolina/3-5-6-3.laz
diff --git a/data/NorthCarolina/3-5-6-4.laz b/data/NorthCarolina/3-5-6-4.laz
diff --git a/data/NorthCarolina/3-5-7-3.laz b/data/NorthCarolina/3-5-7-3.laz
diff --git a/data/NorthCarolina/3-5-7-4.laz b/data/NorthCarolina/3-5-7-4.laz
diff --git a/data/NorthCarolina/3-6-6-3.laz b/data/NorthCarolina/3-6-6-3.laz
diff --git a/data/NorthCarolina/3-6-7-3.laz b/data/NorthCarolina/3-6-7-3.laz
diff --git a/data/NorthCarolina/3-7-7-3.laz b/data/NorthCarolina/3-7-7-3.laz
diff --git a/data/NorthCarolina/4-12-14-7.laz b/data/NorthCarolina/4-12-14-7.laz
diff --git a/data/NorthCarolina/4-12-15-7.laz b/data/NorthCarolina/4-12-15-7.laz
diff --git a/data/NorthCarolina/4-13-14-7.laz b/data/NorthCarolina/4-13-14-7.laz
diff --git a/data/NorthCarolina/4-13-15-7.laz b/data/NorthCarolina/4-13-15-7.laz
diff --git a/data/NorthCarolina/4-4-2-8.laz b/data/NorthCarolina/4-4-2-8.laz
diff --git a/data/NorthCarolina/4-4-3-8.laz b/data/NorthCarolina/4-4-3-8.laz
diff --git a/data/NorthCarolina/4-5-2-8.laz b/data/NorthCarolina/4-5-2-8.laz
diff --git a/data/NorthCarolina/4-5-3-8.laz b/data/NorthCarolina/4-5-3-8.laz
diff --git a/data/NorthCarolina/4-6-6-8.laz b/data/NorthCarolina/4-6-6-8.laz
diff --git a/data/NorthCarolina/4-6-7-8.laz b/data/NorthCarolina/4-6-7-8.laz
diff --git a/data/NorthCarolina/4-7-6-8.laz b/data/NorthCarolina/4-7-6-8.laz
diff --git a/data/NorthCarolina/4-7-7-8.laz b/data/NorthCarolina/4-7-7-8.laz
diff --git a/data/NorthCarolina/entwine-build.json b/data/NorthCarolina/entwine-build.json
@@ -0,0 +1,6 @@
+{
+   "overflowDepth" : 0,
+   "overflowThreshold" : 32768,
+   "trustHeaders" : true,
+   "version" : "2.0.0"
+}
diff --git a/data/NorthCarolina/entwine-files.json b/data/NorthCarolina/entwine-files.json
@@ -0,0 +1,13 @@
+[
+   {
+      "bounds" : [ 633370.82000000007, 215000.01000000001, 88.5, 645000, 228500, 558.87 ],
+      "numPoints" : 2269213,
+      "path" : "data/lidar_raleigh_nc_spm_height_feet.laz",
+      "pointStats" : {
+         "inserts" : 2269213,
+         "outOfBounds" : 0
+      },
+      "srs" : "PROJCS[\"NAD83 / North Carolina (ftUS)\",GEOGCS[\"NAD83\",DATUM[\"North_American_Datum_1983\",SPHEROID[\"GRS 1980\",6378137,298.257222101,AUTHORITY[\"EPSG\",\"7019\"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY[\"EPSG\",\"6269\"]],PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4269\"]],PROJECTION[\"Lambert_Conformal_Conic_2SP\"],PARAMETER[\"standard_parallel_1\",36.16666666666666],PARAMETER[\"standard_parallel_2\",34.33333333333334],PARAMETER[\"latitude_of_origin\",33.75],PARAMETER[\"central_meridian\",-79],PARAMETER[\"false_easting\",2000000],PARAMETER[\"false_northing\",0],UNIT[\"US survey foot\",0.3048006096012192,AUTHORITY[\"EPSG\",\"9003\"]],AXIS[\"X\",EAST],AXIS[\"Y\",NORTH],AUTHORITY[\"EPSG\",\"2264\"]]",
+      "status" : "inserted"
+   }
+]
diff --git a/data/NorthCarolina/entwine.json b/data/NorthCarolina/entwine.json
@@ -0,0 +1,81 @@
+{
+   "bounds" : [ 632434, 214999, -6427, 645936, 228501, 7075 ],
+   "boundsConforming" : [ 633370, 215000, 88, 645001, 228501, 559 ],
+   "dataType" : "laszip",
+   "hierarchyType" : "json",
+   "numPoints" : 2269213,
+   "offset" : [ 639185, 221750, 324 ],
+   "scale" : 0.01,
+   "schema" : [
+      {
+         "name" : "X",
+         "type" : "int32"
+      },
+      {
+         "name" : "Y",
+         "type" : "int32"
+      },
+      {
+         "name" : "Z",
+         "type" : "int32"
+      },
+      {
+         "name" : "Intensity",
+         "type" : "uint16"
+      },
+      {
+         "name" : "ReturnNumber",
+         "type" : "uint8"
+      },
+      {
+         "name" : "NumberOfReturns",
+         "type" : "uint8"
+      },
+      {
+         "name" : "ScanDirectionFlag",
+         "type" : "uint8"
+      },
+      {
+         "name" : "EdgeOfFlightLine",
+         "type" : "uint8"
+      },
+      {
+         "name" : "Classification",
+         "type" : "uint8"
+      },
+      {
+         "name" : "ScanAngleRank",
+         "type" : "float"
+      },
+      {
+         "name" : "UserData",
+         "type" : "uint8"
+      },
+      {
+         "name" : "PointSourceId",
+         "type" : "uint16"
+      },
+      {
+         "name" : "GpsTime",
+         "type" : "double"
+      },
+      {
+         "name" : "Red",
+         "type" : "uint16"
+      },
+      {
+         "name" : "Green",
+         "type" : "uint16"
+      },
+      {
+         "name" : "Blue",
+         "type" : "uint16"
+      },
+      {
+         "name" : "OriginId",
+         "type" : "uint32"
+      }
+   ],
+   "srs" : "PROJCS[\"NAD83 / North Carolina (ftUS)\",GEOGCS[\"NAD83\",DATUM[\"North_American_Datum_1983\",SPHEROID[\"GRS 1980\",6378137,298.257222101,AUTHORITY[\"EPSG\",\"7019\"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY[\"EPSG\",\"6269\"]],PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4269\"]],PROJECTION[\"Lambert_Conformal_Conic_2SP\"],PARAMETER[\"standard_parallel_1\",36.16666666666666],PARAMETER[\"standard_parallel_2\",34.33333333333334],PARAMETER[\"latitude_of_origin\",33.75],PARAMETER[\"central_meridian\",-79],PARAMETER[\"false_easting\",2000000],PARAMETER[\"false_northing\",0],UNIT[\"US survey foot\",0.3048006096012192,AUTHORITY[\"EPSG\",\"9003\"]],AXIS[\"X\",EAST],AXIS[\"Y\",NORTH],AUTHORITY[\"EPSG\",\"2264\"]]",
+   "ticks" : 256
+}
diff --git a/data/NorthCarolina/h/0-0-0-0.json b/data/NorthCarolina/h/0-0-0-0.json
@@ -0,0 +1,79 @@
+{
+   "0-0-0-0" : 32615,
+   "1-0-0-0" : 8860,
+   "1-0-0-1" : 63459,
+   "1-0-1-0" : 12764,
+   "1-0-1-1" : 4478,
+   "1-1-0-0" : 15749,
+   "1-1-0-1" : 8194,
+   "1-1-1-0" : 38256,
+   "1-1-1-1" : 32245,
+   "2-0-0-1" : 15416,
+   "2-0-0-2" : 43981,
+   "2-0-1-2" : 787,
+   "2-1-0-1" : 11561,
+   "2-1-0-2" : 82735,
+   "2-1-1-1" : 8914,
+   "2-1-1-2" : 92875,
+   "2-1-2-2" : 15498,
+   "2-2-1-2" : 26930,
+   "2-2-2-1" : 65135,
+   "2-2-2-2" : 46367,
+   "2-2-3-1" : 23473,
+   "2-2-3-2" : 33908,
+   "2-3-2-1" : 6778,
+   "2-3-2-2" : 3725,
+   "2-3-3-1" : 64585,
+   "2-3-3-2" : 49495,
+   "3-0-0-3" : 6354,
+   "3-0-0-4" : 6208,
+   "3-1-0-3" : 50916,
+   "3-1-0-4" : 42594,
+   "3-1-1-3" : 4068,
+   "3-1-1-4" : 39878,
+   "3-2-0-3" : 27987,
+   "3-2-0-4" : 36430,
+   "3-2-1-3" : 1601,
+   "3-2-1-4" : 80681,
+   "3-2-2-3" : 17057,
+   "3-2-2-4" : 68929,
+   "3-2-3-4" : 11663,
+   "3-3-1-3" : 11534,
+   "3-3-1-4" : 24177,
+   "3-3-2-3" : 22477,
+   "3-3-2-4" : 78095,
+   "3-3-3-4" : 73129,
+   "3-3-4-4" : 33875,
+   "3-3-5-4" : 5,
+   "3-4-2-4" : 4938,
+   "3-4-3-4" : 66817,
+   "3-4-4-3" : 88378,
+   "3-4-4-4" : 28557,
+   "3-4-5-3" : 35708,
+   "3-4-5-4" : 45101,
+   "3-4-6-3" : 449,
+   "3-4-6-4" : 10698,
+   "3-5-4-3" : 33711,
+   "3-5-4-4" : 761,
+   "3-5-5-3" : 68152,
+   "3-5-5-4" : 32121,
+   "3-5-6-3" : 41799,
+   "3-5-6-4" : 61444,
+   "3-5-7-3" : 36704,
+   "3-5-7-4" : 3481,
+   "3-6-6-3" : 52016,
+   "3-6-7-3" : 72809,
+   "3-7-7-3" : 33047,
+   "4-12-14-7" : 7740,
+   "4-12-15-7" : 9406,
+   "4-13-14-7" : 8027,
+   "4-13-15-7" : 8616,
+   "4-4-2-8" : 9167,
+   "4-4-3-8" : 9375,
+   "4-5-2-8" : 10537,
+   "4-5-3-8" : 7632,
+   "4-6-6-8" : 9470,
+   "4-6-7-8" : 6463,
+   "4-7-6-8" : 10729,
+   "4-7-7-8" : 8919
+}
diff --git a/data/README.md b/data/README.md
@@ -0,0 +1,21 @@
+# Sample Data in EPT format
+
+The following dataset may be used for visualization and testing purposes. This point cloud sample dataset was originally prepared by Markus Neteler and Helena Mitasova in their book ["Open Source GIS: A GRASS GIS Approach"](https://grassbook.org/datasets/datasets-3rd-edition/). From their description:
+> We developed a completely new free geospatial dataset and substituted all Spearfish (SD) examples in the previous editions with this new, much richer North Carolina (NC, USA) data set. This data set is a comprehensive collection of raster, vector and imagery data covering parts of North Carolina (NC), USA (map), prepared from public data sources provided by the North Carolina state and local government agencies and Global Land Cover Facility (GLCF)
+
+We will only use the sample lidar data.
+
+```bash
+curl https://www.grassbook.org/wp-content/uploads/ncexternal/lidar_raleigh_nc_spm_height_feet.las -o .data/lidar_raleigh_nc_spm_height_feet.las
+```
+Once it is downloaded, we can use pdal (or pdal in docker) to set the srs (as it is undefined in the data itself, but described on database [webpage](https://grassbook.org/datasets/datasets-3rd-edition/)).
+
+```bash
+pdal pipeline data/set_srs.json
+```
+
+And then, using docker we can convert the laz file to the [EPT format](https://github.com/connormanning/ept):
+
+```bash
+docker run -it -v /Users/iosefa/repos/pnext-io/data:/data connormanning/entwine build -i data/lidar_raleigh_nc_spm_height_feet.laz -o /data/NorthCarolina
+```
diff --git a/data/lidar_raleigh_nc_spm_height_feet.las b/data/lidar_raleigh_nc_spm_height_feet.las
diff --git a/data/lidar_raleigh_nc_spm_height_feet.laz b/data/lidar_raleigh_nc_spm_height_feet.laz
diff --git a/data/set_srs.json b/data/set_srs.json
@@ -0,0 +1,11 @@
+{
+  "pipeline":[
+    "data/lidar_raleigh_nc_spm_height_feet.las",
+    {
+      "type":"writers.las",
+      "filename":  "data/lidar_raleigh_nc_spm_height_feet.laz",
+      "a_srs" : "EPSG:2264",
+      "compression": "laszip"
+    }
+  ]
+}
diff --git a/ts_test/format/las/generator/README.md b/ts_test/format/las/generator/README.md
@@ -0,0 +1,20 @@
+# Sample Point Cloud Generator
+
+A bash script that uses pdal to create sample point clouds for testing purposes.
+
+## Requirements
+
+This program requires that you [download and install pdal](https://pdal.io/download.html) (v1.7.1). If you dont have pdal installed, it is suggested that you use [docker](https://pdal.io/quickstart.html).
+
+## Usage
+
+With pdal installed, make the `generator` directory current working directory and run: `./generator.sh`
+
+## Details
+
+A few notes on the naming convention of generated files:
+
+- Files are named with the following convention:`v<LAS minor version number>d<LAS data format>.las`. Therefore, `v1d1.las` is a point cloud with a minor version of `1` and a data format of `1`.
+- Each point of each file has a randomly assigned class that (from class numbers that are not reserved by ASPRS).
+- Each file has a total of 10 points.
+- Point clouds have an SRS assigned.
diff --git a/ts_test/format/las/generator/classify.json b/ts_test/format/las/generator/classify.json
@@ -0,0 +1,17 @@
+{
+  "pipeline":[
+    {
+      "type":"readers.las",
+      "filename":"lasdump/tmp.las"
+    },
+    {
+      "type":"filters.assign",
+      "assignment" : "Classification[0:0]=0"
+    },
+    {
+      "type":"writers.las",
+      "forward": "minor_version, dataformat_id",
+      "filename":"lasdump/tmp.las"
+    }
+  ]
+}
diff --git a/ts_test/format/las/generator/generator.json b/ts_test/format/las/generator/generator.json
@@ -0,0 +1,21 @@
+{
+  "pipeline":[
+    {
+      "type":"readers.faux",
+      "bounds":"([0,1000],[0,1000],[0,1000])",
+      "count":"100",
+      "mode":"random"
+    },
+    {
+      "type":"filters.divider",
+      "capacity":"10"
+    },
+    {
+      "type":"writers.las",
+      "a_srs" : "EPSG:6676",
+      "minor_version": 1,
+      "dataformat_id": 1,
+      "filename":"lasdump/tmp#.las"
+    }
+  ]
+}