added 3d surface reconstruction from sensor image, added point cloud and 3d visualization, 3d surface model added to zoo

Author: Mike Lambeta

.flake8

@@ -3,6 +3,8 @@ exclude =
     .git
     ,.nox
     ,__init__.py
+    ,pytouch/models/pix2pix/thirdparty
+    ,train
 max-line-length = 119
 copyright-check = True
 select = E,F,W,C

AUTHORS.md

@@ -0,0 +1,16 @@
+# PyTouch Team
+
+* Mike Lambeta
+* Huazhe Xu
+* Jingwei Xu
+* Po-Wei Chou
+* Shaoxiong Wang
+* Roberto Calandra
+
+# Additional Authors
+
+* Paloma Sodhi - Surface 3D patch graph algorithms
+
+# Special Thanks To
+
+* Haozhou Wang - for kindly providing the PyPi repository PyTouch.

CONTRIBUTING.md

@@ -4,7 +4,7 @@ We want to make contributing to this project as easy and transparent as possible
 ## Pull Requests
 We actively welcome your pull requests.
 
-1. Fork the repo and create your branch from `master`.
+1. Fork the repo and create your branch from `main`.
 2. If you have added code that should be tested, add tests.
 3. If you have changed APIs, update the documentation.
 4. Ensure the test suite passes.

README.md

@@ -56,4 +56,5 @@ If you use PyTouch in your research please cite the corresponding [paper](https:
 ```
 
 ## Acknowledgments
-We would like to thank Haozhou Wang for kindly providing the PyPi repository PyTouch.
+
+PyTouch would like to acknowledge the [list of contributors](AUTHORS.md).

examples/configs/digit_surface3d.yaml

@@ -0,0 +1,36 @@
+## examples/surface_3d.py
+
+dataset:
+  path: "/path/to/dataset"
+  batch_size: 1
+  shuffle: False
+  num_workers: 8
+
+sensor:
+  max_depth: 0.0198 # max depth value (close to avg gel depth)
+  remove_background_depth: True # sets depth>max_depth to 0 which is filtered out in points3d
+
+  T_cam_offset_sim: [[2.22e-16, 2.22e-16, -1.00e+00, 0.00e+00],
+                [-1.00e+00, 0.00e+00, -2.22e-16, 0.00e+00],
+                [0.00e+00, 1.00e+00, 2.22e-16, 1.50e-02],
+                [0.00e+00, 0.00e+00, 0.00e+00, 1.00e+00]]
+
+  # rotation of +90 degrees about x-axis
+  T_cam_offset_real: [[1., 0., 0., 0.],
+                [0., 0., -1, 0.],
+                [0., 1., 0., 0.],
+                [0., 0., 0., 1.]]
+
+  P:  [[2.30940108e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00],
+       [0.00000000e+00, 1.73205081e+00, 0.00000000e+00, 0.00000000e+00],
+       [0.00000000e+00, 0.00000000e+00, -1.04081633e+00, -2.04081633e-03],
+       [0.00000000e+00, 0.00000000e+00, -1.00000000e+00, 0.00000000e+00]]
+
+  z_near: 0.001
+  z_far: 0.05
+
+  gel min depth: 0.01910434
+  gel_depth_offset: 1e-4
+
+  gel_width: 0.02 # gel width (y-axis) in meters
+  gel_height: 0.03 # gel height (x-axis) in meters

examples/surface_3d.py

@@ -0,0 +1,46 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.
+
+import hydra
+
+from pytouch.common.visualizer import Visualizer3D, Visualizer3DViewParams
+from pytouch.datasets.sequence import ImageSequenceDataset
+from pytouch.sensors import DigitSensor
+from pytouch.tasks import Surface3D
+
+
+@hydra.main(config_path="configs", config_name="digit_surface3d.yaml")
+def visualize_surface_3d(cfg):
+    # load touch sequence dataset
+    img_seq_ds = ImageSequenceDataset(cfg.dataset.path)
+
+    # define a custom camera view
+    view_params = Visualizer3DViewParams(
+        fov=60,  # field of view
+        front=[0.4257, -0.2125, -0.8795],  # front vector
+        lookat=[0.02, 0.0, 0.0],  # look at vector
+        up=[0.9768, -0.0694, 0.2024],  # up vector
+        zoom=0.25,  # zoom
+    )
+
+    # initialize point cloud visualizer
+    visualizer = Visualizer3D(view_params=view_params)
+
+    # initialize surface 3d model
+    surface3d = Surface3D(
+        DigitSensor,
+        sensor_params=cfg.sensor,
+    )
+
+    # get first sequence
+    sequence = img_seq_ds[0]
+    for img in sequence:
+        output = surface3d.point_cloud_3d(img_color=img)
+        visualizer.render(output.points_3d)
+        # you may also plot the color, predicted normal, and predicted depth images
+        # color = output.color
+        # depth = output.depth
+        # normal = output.normal
+
+
+if __name__ == "__main__":
+    visualize_surface_3d()

examples/zoo_models.py

@@ -12,7 +12,7 @@ def main():
 
     # load DIGIT sensor touch detect model from pytouch zoo
     touch_detect_model = pytouch_zoo.load_model_from_zoo(  # noqa: F841
-        "touchdetect_resnet18", sensors.DigitSensor
+        "touchdetect_resnet", sensors.DigitSensor
     )
 
     # load custom PyTorch-Lightning saved model

noxfile.py

@@ -5,7 +5,7 @@
 
 BASE = os.path.abspath(os.path.dirname(__file__))
 
-DEFAULT_PYTHON_VERSIONS = ["3.7", "3.8", "3.9"]
+DEFAULT_PYTHON_VERSIONS = ["3.7", "3.8"]
 
 LINT_SETUP_DEPS = ["black", "flake8", "flake8-copyright", "isort"]
 

pytouch/__init__.py

@@ -5,4 +5,4 @@
 from pytouch.common import SensorDataSources
 from pytouch.models import PyTouchZoo
 
-__version__ = "0.4.0"
+__version__ = "0.4.2"

pytouch/common/__init__.py

@@ -1,3 +1,4 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.
 
 from .constants import SensorDataSources
+from .visualizer import Visualizer3D

pytouch/common/constants.py

@@ -14,3 +14,5 @@ class SensorDataSources(Enum):
     VIDEO = "video"
     # Direct input from sensor
     SENSOR = "sensor"
+    # Simulator input
+    SIM = "simulator"