Match undistorted image names with shot ids (mapillary#706)

Summary:
Pull Request resolved: mapillary#706

Since we do not know on which format the input images are, we were always adding `.jpg` to the undistorted image file name before saving them. This caused a miss-match between the undistorted images' file names and the shot IDs in the undistorted reconstruction.

We fix that by renaming the shot IDs to match the image filename.

Additionally, we do not append the file format to the file name if the filename already ends with that extension. This breaks file-level backwards compatibility.

We also add the `undistorted_shot_ids.json` file to the dataset that maps original shot IDs to the list of corresponding undistorted shot IDs.  Remember that a single shot can have multiple undistorted shots.

Reviewed By: YanNoun

Differential Revision: D26443052

fbshipit-source-id: 4042c93c434ef120cd2c3a13591087ee715a7640

Author: paulinus

CHANGELOG.md

@@ -6,6 +6,11 @@
 ### Breaking
  - Main datastructures moved to C++ with Python bindings
  - Drop Python 2 support.  OpenSfM 0.5.x is the latest to support Python2.
+ - Undistorted image file names only append the image format if it does not match the distorted image source
+ - Undistorted shot ids now match the undistorted image file names and may not match the source shot ids
+
+### Added
+ - The file `undistorted/undistorted_shot_ids.json` stores a map from the original shot ids to their corresponding list of undistorted shot ids.
 
 
 ## 0.5.1

bin/import_colmap.py

@@ -48,6 +48,7 @@
 def compute_and_save_undistorted_reconstruction(
     reconstruction, tracks_manager, data, udata
 ):
+    image_format = data.config["undistorted_image_format"]
     urec = types.Reconstruction()
     utracks_manager = pysfm.TracksManager()
     undistorted_shots = []
@@ -61,7 +62,7 @@ def compute_and_save_undistorted_reconstruction(
         else:
             raise ValueError
         urec.add_camera(ucamera)
-        ushot = osfm_u.get_shot_with_different_camera(urec, shot, ucamera)
+        ushot = osfm_u.get_shot_with_different_camera(urec, shot, ucamera, image_format)
         if tracks_manager:
             osfm_u.add_subshot_tracks(tracks_manager, utracks_manager, shot, ushot)
         undistorted_shots.append(ushot)

opensfm/actions/export_openmvs.py

@@ -21,10 +21,10 @@ def run_dataset(data, image_list):
                 export_only[image.strip()] = True
 
     if reconstructions:
-        export(reconstructions[0], tracks_manager, udata, data, export_only)
+        export(reconstructions[0], tracks_manager, udata, export_only)
 
 
-def export(reconstruction, tracks_manager, udata, data, export_only):
+def export(reconstruction, tracks_manager, udata, export_only):
     exporter = pydense.OpenMVSExporter()
     for camera in reconstruction.cameras.values():
         if camera.projection_type == "perspective":

opensfm/actions/export_ply.py

@@ -26,16 +26,17 @@ def run_dataset(data, no_cameras, no_points, depthmaps, point_num_views):
     if reconstructions:
         data.save_ply(reconstructions[0], tracks_manager, None, no_cameras, no_points, point_num_views)
 
-    if depthmaps and reconstructions:
+    if depthmaps:
         udata = dataset.UndistortedDataSet(data)
-        for id, shot in reconstructions[0].shots.items():
-            rgb = udata.load_undistorted_image(id)
+        urec = udata.load_undistorted_reconstruction()[0]
+        for shot in urec.shots.values():
+            rgb = udata.load_undistorted_image(shot.id)
             for t in ("clean", "raw"):
-                path_depth = udata.depthmap_file(id, t + ".npz")
+                path_depth = udata.depthmap_file(shot.id, t + ".npz")
                 if not os.path.exists(path_depth):
                     continue
                 depth = np.load(path_depth)["depth"]
                 rgb = scale_down_image(rgb, depth.shape[1], depth.shape[0])
                 ply = depthmap_to_ply(shot, depth, rgb)
-                with io.open_wt(udata.depthmap_file(id, t + ".ply")) as fout:
+                with io.open_wt(udata.depthmap_file(shot.id, t + ".ply")) as fout:
                     fout.write(ply)

opensfm/dataset.py

@@ -759,14 +759,23 @@ def __init__(self, base_dataset, undistorted_data_path=None):
         else:
             self.data_path = os.path.join(self.base.data_path, "undistorted")
 
+    def load_undistorted_shot_ids(self):
+        filename = os.path.join(self.data_path, "undistorted_shot_ids.json")
+        with io.open_rt(filename) as fin:
+            return io.json_load(fin)
+
+    def save_undistorted_shot_ids(self, ushot_dict):
+        filename = os.path.join(self.data_path, "undistorted_shot_ids.json")
+        io.mkdir_p(self.data_path)
+        with io.open_wt(filename) as fout:
+            io.json_dump(ushot_dict, fout, minify=False)
+
     def _undistorted_image_path(self):
         return os.path.join(self.data_path, "images")
 
     def _undistorted_image_file(self, image):
         """Path of undistorted version of an image."""
-        image_format = self.config["undistorted_image_format"]
-        filename = image + "." + image_format
-        return os.path.join(self._undistorted_image_path(), filename)
+        return os.path.join(self._undistorted_image_path(), image)
 
     def load_undistorted_image(self, image):
         """Load undistorted image pixels as a numpy array."""

opensfm/undistort.py

@@ -15,6 +15,7 @@
 
 
 def undistort_reconstruction(tracks_manager, reconstruction, data, udata):
+    image_format = data.config["undistorted_image_format"]
     urec = types.Reconstruction()
     urec.points = reconstruction.points
     utracks_manager = pysfm.TracksManager()
@@ -24,18 +25,20 @@ def undistort_reconstruction(tracks_manager, reconstruction, data, udata):
         if shot.camera.projection_type == "perspective":
             camera = perspective_camera_from_perspective(shot.camera)
             urec.add_camera(camera)
-            subshots = [get_shot_with_different_camera(urec, shot, camera)]
+            subshots = [get_shot_with_different_camera(urec, shot, camera, image_format)]
         elif shot.camera.projection_type == "brown":
             camera = perspective_camera_from_brown(shot.camera)
             urec.add_camera(camera)
-            subshots = [get_shot_with_different_camera(urec, shot, camera)]
+            subshots = [get_shot_with_different_camera(urec, shot, camera, image_format)]
         elif shot.camera.projection_type in ["fisheye", "fisheye_opencv"]:
             camera = perspective_camera_from_fisheye(shot.camera)
             urec.add_camera(camera)
-            subshots = [get_shot_with_different_camera(urec, shot, camera)]
+            subshots = [get_shot_with_different_camera(urec, shot, camera, image_format)]
         elif pygeometry.Camera.is_panorama(shot.camera.projection_type):
             subshot_width = int(data.config["depthmap_resolution"])
-            subshots = perspective_views_of_a_panorama(shot, subshot_width, urec)
+            subshots = perspective_views_of_a_panorama(
+                shot, subshot_width, urec, image_format
+            )
 
         for subshot in subshots:
             if tracks_manager:
@@ -46,6 +49,13 @@ def undistort_reconstruction(tracks_manager, reconstruction, data, udata):
     if tracks_manager:
         udata.save_undistorted_tracks_manager(utracks_manager)
 
+    udata.save_undistorted_shot_ids(
+        {
+            shot_id: [ushot.id for ushot in ushots]
+            for shot_id, ushots in undistorted_shots.items()
+        }
+    )
+
     return undistorted_shots
 
 
@@ -122,25 +132,26 @@ def undistort_image(shot, undistorted_shots, original, interpolation, max_size):
 
     projection_type = shot.camera.projection_type
     if projection_type in ["perspective", "brown", "fisheye", "fisheye_opencv"]:
-        new_camera = undistorted_shots[0].camera
+        [undistorted_shot] = undistorted_shots
+        new_camera = undistorted_shot.camera
         height, width = original.shape[:2]
         map1, map2 = pygeometry.compute_camera_mapping(
             shot.camera, new_camera, width, height
         )
         undistorted = cv2.remap(original, map1, map2, interpolation)
-        return {shot.id: scale_image(undistorted, max_size)}
+        return {undistorted_shot.id: scale_image(undistorted, max_size)}
     elif pygeometry.Camera.is_panorama(projection_type):
         subshot_width = undistorted_shots[0].camera.width
         width = 4 * subshot_width
         height = width // 2
         image = cv2.resize(original, (width, height), interpolation=interpolation)
         mint = cv2.INTER_LINEAR if interpolation == cv2.INTER_AREA else interpolation
         res = {}
-        for subshot in undistorted_shots:
+        for undistorted_shot in undistorted_shots:
             undistorted = render_perspective_view_of_a_panorama(
-                image, shot, subshot, mint
+                image, shot, undistorted_shot, mint
             )
-            res[subshot.id] = scale_image(undistorted, max_size)
+            res[undistorted_shot.id] = scale_image(undistorted, max_size)
         return res
     else:
         raise NotImplementedError(
@@ -161,8 +172,16 @@ def scale_image(image, max_size):
     return cv2.resize(image, (width, height), interpolation=cv2.INTER_NEAREST)
 
 
-def get_shot_with_different_camera(urec, shot, camera):
-    new_shot = urec.create_shot(shot.id, shot.camera.id, shot.pose)
+def add_image_format_extension(shot_id, image_format):
+    if shot_id.endswith(f".{image_format}"):
+        return shot_id
+    else:
+        return f"{shot_id}.{image_format}"
+
+
+def get_shot_with_different_camera(urec, shot, camera, image_format):
+    new_shot_id = add_image_format_extension(shot.id, image_format)
+    new_shot = urec.create_shot(new_shot_id, shot.camera.id, shot.pose)
     new_shot.metadata = shot.metadata
     return new_shot
 
@@ -206,6 +225,7 @@ def perspective_camera_from_fisheye_opencv(fisheye_opencv):
     camera.height = fisheye_opencv.height
     return camera
 
+
 def perspective_camera_from_fisheye62(fisheye62):
     """Create a perspective camera from a fisheye extended."""
     camera = pygeometry.Camera.create_perspective(
@@ -216,7 +236,8 @@ def perspective_camera_from_fisheye62(fisheye62):
     camera.height = fisheye62.height
     return camera
 
-def perspective_views_of_a_panorama(spherical_shot, width, reconstruction):
+
+def perspective_views_of_a_panorama(spherical_shot, width, reconstruction, image_format):
     """Create 6 perspective views of a panorama."""
     camera = pygeometry.Camera.create_perspective(0.5, 0.0, 0.0)
     camera.id = "perspective_panorama_camera"
@@ -240,9 +261,12 @@ def perspective_views_of_a_panorama(spherical_shot, width, reconstruction):
         pose = pygeometry.Pose()
         pose.set_rotation_matrix(R)
         pose.set_origin(o)
+        shot_id = add_image_format_extension(
+            f"{spherical_shot.id}_perspective_view_{name}", image_format
+        )
         shots.append(
             reconstruction.create_shot(
-                "{}_perspective_view_{}".format(spherical_shot.id, name),
+                shot_id,
                 camera.id,
                 pose,
             )