Merge pull request #901 from StanfordVL/fix/pre-release-bug-fixes

Pre-release bug fixes

omnigibson/configs/default_cfg.yaml

@@ -1,6 +1,4 @@
 env:
-  action_frequency: 60              # (int): environment executes action at the action_frequency rate
-  physics_frequency: 60             # (int): physics frequency (1 / physics_timestep for physx)
   device: null                      # (None or str): specifies the device to be used if running on the gpu with torch backend
 
 render:

omnigibson/configs/fetch_behavior.yaml

@@ -1,6 +1,4 @@
 env:
-  action_frequency: 60                  # (int): environment executes action at the action_frequency rate
-  physics_frequency: 60                 # (int): physics frequency (1 / physics_timestep for physx)
   device: null                          # (None or str): specifies the device to be used if running on the gpu with torch backend
   automatic_reset: false                # (bool): whether to automatic reset after an episode finishes
   flatten_action_space: false           # (bool): whether to flatten the action space as a sinle 1D-array

omnigibson/configs/turtlebot_nav.yaml

@@ -1,6 +1,4 @@
 env:
-  action_frequency: 60                  # (int): environment executes action at the action_frequency rate
-  physics_frequency: 60                 # (int): physics frequency (1 / physics_timestep for physx)
   device: null                          # (None or str): specifies the device to be used if running on the gpu with torch backend
   automatic_reset: false                # (bool): whether to automatic reset after an episode finishes
   flatten_action_space: false           # (bool): whether to flatten the action space as a sinle 1D-array

omnigibson/envs/env_base.py

@@ -1,3 +1,5 @@
+from collections import OrderedDict
+from collections.abc import Iterable
 from copy import deepcopy
 
 import gymnasium as gym
@@ -18,7 +20,7 @@
     recursively_generate_compatible_dict,
     recursively_generate_flat_dict,
 )
-from omnigibson.utils.numpy_utils import NumpyTypes
+from omnigibson.utils.numpy_utils import NumpyTypes, list_to_np_array
 from omnigibson.utils.python_utils import (
     Recreatable,
     assert_valid_key,
@@ -403,8 +405,8 @@ def _load_action_space(self):
                 lows.append(space.low)
                 highs.append(space.high)
             action_space = gym.spaces.Box(
-                th.tensor(lows, dtype=th.float32).cpu().numpy(),
-                th.tensor(highs, dtype=th.float32).cpu().numpy(),
+                list_to_np_array(lows),
+                list_to_np_array(highs),
                 dtype=NumpyTypes.FLOAT32,
             )
 
@@ -617,6 +619,10 @@ def step(self, action, n_render_iterations=1):
                 - dict: info, i.e. dictionary with any useful information
         """
         # Pre-processing before stepping simulation
+        if isinstance(action, Iterable) and not isinstance(action, (dict, OrderedDict)):
+            # Convert numpy arrays and lists to tensors
+            # Skip dict action
+            action = th.as_tensor(action).flatten()
         self._pre_step(action)
 
         # Step simulation

omnigibson/examples/object_states/folded_unfolded_state_demo.py

@@ -4,6 +4,7 @@
 from omnigibson.macros import gm
 from omnigibson.object_states import Folded, Unfolded
 from omnigibson.utils.constants import PrimType
+from omnigibson.utils.python_utils import multi_dim_linspace
 
 # Make sure object states and GPU dynamics are enabled (GPU dynamics needed for cloth)
 gm.ENABLE_OBJECT_STATES = True
@@ -111,7 +112,9 @@ def print_state():
             end[:, 0] += x_extent * 0.9
 
             increments = 25
-            for ctrl_pts in th.cat([th.linspace(start, mid, increments), th.linspace(mid, end, increments)]):
+            for ctrl_pts in th.cat(
+                [multi_dim_linspace(start, mid, increments), multi_dim_linspace(mid, end, increments)]
+            ):
                 obj.root_link.set_particle_positions(ctrl_pts, idxs=indices)
                 og.sim.step()
                 print_state()
@@ -137,7 +140,9 @@ def print_state():
             end[:, 1] += direction * y_extent * 0.4
 
             increments = 25
-            for ctrl_pts in th.cat([th.linspace(start, mid, increments), th.linspace(mid, end, increments)]):
+            for ctrl_pts in th.cat(
+                [multi_dim_linspace(start, mid, increments), multi_dim_linspace(mid, end, increments)]
+            ):
                 obj.root_link.set_particle_positions(ctrl_pts, idxs=indices)
                 env.step(th.empty(0))
                 print_state()

omnigibson/objects/primitive_object.py

@@ -82,7 +82,7 @@ def __init__(
         """
         # Compose load config and add rgba values
         load_config = dict() if load_config is None else load_config
-        load_config["color"] = th.tensor(rgba[:3])
+        load_config["color"] = rgba[:3]
         load_config["opacity"] = rgba[3]
         load_config["radius"] = radius
         load_config["height"] = height
@@ -211,7 +211,7 @@ def radius(self, radius):
         """
         assert_valid_key(key=self._primitive_type, valid_keys=VALID_RADIUS_OBJECTS, name="primitive object with radius")
         # Update the extents variable
-        original_extent = th.tensor(self._extents)
+        original_extent = self._extents.clone()
         self._extents = (
             th.ones(3) * radius * 2.0
             if self._primitive_type == "Sphere"
@@ -269,7 +269,7 @@ def height(self, height):
         """
         assert_valid_key(key=self._primitive_type, valid_keys=VALID_HEIGHT_OBJECTS, name="primitive object with height")
         # Update the extents variable
-        original_extent = th.tensor(self._extents)
+        original_extent = self._extents.clone()
         self._extents[2] = height
 
         # Calculate the correct scaling factor and scale the points and normals appropriately
@@ -316,7 +316,7 @@ def size(self, size):
         assert_valid_key(key=self._primitive_type, valid_keys=VALID_SIZE_OBJECTS, name="primitive object with size")
 
         # Update the extents variable
-        original_extent = th.tensor(self._extents)
+        original_extent = self._extents.clone()
         self._extents = th.ones(3) * size
 
         # Calculate the correct scaling factor and scale the points and normals appropriately

omnigibson/prims/cloth_prim.py

@@ -134,7 +134,7 @@ def _post_load(self):
         self._centroid_idx = th.argmin(dists)
 
         # Store the default position of the points in the local frame
-        self._default_positions = th.tensor(self.get_attribute(attr="points"))
+        self._default_positions = vtarray_to_torch(self.get_attribute(attr="points"))
 
     @property
     def visual_aabb(self):
@@ -184,7 +184,7 @@ def compute_particle_positions(self, idxs=None):
         scale = self.scale
 
         # Don't copy to save compute, since we won't be returning a reference to the underlying object anyways
-        p_local = th.as_tensor(self.get_attribute(attr="points"), dtype=th.float32)
+        p_local = vtarray_to_torch(self.get_attribute(attr="points"))
         p_local = p_local[idxs] if idxs is not None else p_local
         p_world = (ori @ (p_local * scale).T).T + pos
 
@@ -211,7 +211,7 @@ def set_particle_positions(self, positions, idxs=None):
 
         # Fill the idxs if requested
         if idxs is not None:
-            p_local_old = th.as_tensor(self.get_attribute(attr="points"), dtype=th.float32)
+            p_local_old = vtarray_to_torch(self.get_attribute(attr="points"))
             p_local_old[idxs] = p_local
             p_local = p_local_old
 
@@ -288,7 +288,7 @@ def particle_velocities(self):
                 cartesian coordinates with respect to the world frame.
         """
         # the velocities attribute is w.r.t the world frame already
-        return th.tensor(self.get_attribute(attr="velocities"))
+        return vtarray_to_torch(self.get_attribute(attr="velocities"))
 
     @particle_velocities.setter
     def particle_velocities(self, vel):

omnigibson/prims/geom_prim.py

@@ -88,6 +88,7 @@ def color(self, rgb):
         Args:
             3-array: The default RGB color used for this visual geom
         """
+        rgb = th.as_tensor(rgb)
         if self.has_material():
             self.material.diffuse_color_constant = rgb
         else:

omnigibson/sensors/vision_sensor.py

@@ -382,7 +382,7 @@ def _remap_semantic_segmentation(self, img, id_to_labels):
 
         image_keys = th.unique(img)
         if not set(image_keys.tolist()).issubset(set(replicator_mapping.keys())):
-            log.warning(
+            log.debug(
                 "Some semantic IDs in the image are not in the id_to_labels mapping. This is a known issue with the replicator and should only affect a few pixels. These pixels will be marked as unlabelled."
             )
 
@@ -482,6 +482,8 @@ def _remap_bounding_box_semantic_ids(self, bboxes, id_to_labels):
         replicator_mapping = self._preprocess_semantic_labels(id_to_labels)
         for bbox in bboxes:
             bbox["semanticId"] = semantic_class_name_to_id()[replicator_mapping[bbox["semanticId"]]]
+        # Replicator returns each box as a numpy.void; we convert them to tuples here
+        bboxes = [box.tolist() for box in bboxes]
         info = {semantic_class_name_to_id()[val]: val for val in replicator_mapping.values()}
         return bboxes, info
 

omnigibson/simulator.py

@@ -72,7 +72,6 @@
 m.KIT_FILES = {
     (4, 0, 0): "omnigibson_4_0_0.kit",
     (4, 1, 0): "omnigibson_4_1_0.kit",
-    (2023, 1, 1): "omnigibson_2023_1_1.kit",
 }
 
 

omnigibson/systems/micro_particle_system.py

@@ -219,7 +219,7 @@ def particle_positions(self):
             th.tensor: (N, 3) numpy array, where each of the N particles' positions are expressed in (x,y,z)
                 cartesian coordinates relative to this instancer's parent prim
         """
-        return th.tensor(self.get_attribute(attr="positions"))
+        return vtarray_to_torch(self.get_attribute(attr="positions"))
 
     @particle_positions.setter
     def particle_positions(self, pos):
@@ -273,7 +273,7 @@ def particle_velocities(self):
             th.tensor: (N, 3) numpy array, where each of the N particles' velocities are expressed in (x,y,z)
                 cartesian coordinates relative to this instancer's parent prim
         """
-        return th.tensor(self.get_attribute(attr="velocities"))
+        return vtarray_to_torch(self.get_attribute(attr="velocities"))
 
     @particle_velocities.setter
     def particle_velocities(self, vel):
@@ -296,7 +296,7 @@ def particle_scales(self):
             th.tensor: (N, 3) numpy array, where each of the N particles' scales are expressed in (x,y,z)
                 cartesian coordinates relative to this instancer's parent prim
         """
-        return th.tensor(self.get_attribute(attr="scales"))
+        return vtarray_to_torch(self.get_attribute(attr="scales"))
 
     @particle_scales.setter
     def particle_scales(self, scales):
@@ -319,7 +319,7 @@ def particle_prototype_ids(self):
             th.tensor: (N,) numpy array, where each of the N particles' prototype_id (i.e.: which prototype is being used
                 for that particle)
         """
-        return th.tensor(self.get_attribute(attr="protoIndices"))
+        return vtarray_to_torch(self.get_attribute(attr="protoIndices"))
 
     @particle_prototype_ids.setter
     def particle_prototype_ids(self, prototype_ids):

omnigibson/utils/deprecated_utils.py

@@ -956,12 +956,12 @@ def colorize_bboxes(bboxes_2d_data, bboxes_2d_rgb, num_channels=3):
     rgb_img = Image.fromarray(bboxes_2d_rgb)
     rgb_img_draw = ImageDraw.Draw(rgb_img)
     for bbox_2d in bboxes_2d_data:
-        semantic_id_list.append(bbox_2d["semanticId"])
+        semantic_id_list.append(bbox_2d[0])
         bbox_2d_list.append(bbox_2d)
     semantic_id_list_np = np.unique(np.array(semantic_id_list))
     color_list = random_colours(len(semantic_id_list_np.tolist()), True, num_channels)
     for bbox_2d in bbox_2d_list:
-        index = np.where(semantic_id_list_np == bbox_2d["semanticId"])[0][0]
+        index = np.where(semantic_id_list_np == bbox_2d[0])[0][0]
         bbox_color = color_list[index]
         outline = (bbox_color[0], bbox_color[1], bbox_color[2])
         if num_channels == 4:
@@ -971,9 +971,7 @@ def colorize_bboxes(bboxes_2d_data, bboxes_2d_rgb, num_channels=3):
                 bbox_color[2],
                 bbox_color[3],
             )
-        rgb_img_draw.rectangle(
-            [(bbox_2d["x_min"], bbox_2d["y_min"]), (bbox_2d["x_max"], bbox_2d["y_max"])], outline=outline, width=2
-        )
+        rgb_img_draw.rectangle([(bbox_2d[1], bbox_2d[2]), (bbox_2d[3], bbox_2d[4])], outline=outline, width=2)
     bboxes_2d_rgb = np.array(rgb_img)
     return bboxes_2d_rgb
 

omnigibson/utils/gym_utils.py

@@ -1,4 +1,5 @@
 from abc import ABCMeta, abstractmethod
+from collections.abc import Iterable
 
 import gymnasium as gym
 import torch as th
@@ -56,6 +57,9 @@ def recursively_generate_compatible_dict(dic):
         elif isinstance(v, th.Tensor) and v.dim() > 1:
             # Map to list of tuples
             out[k] = tuple(tuple(row.tolist()) for row in v)
+        elif isinstance(v, Iterable):
+            # bounding box modalities give a list of tuples
+            out[k] = tuple(v)
         else:
             # Preserve the key-value pair
             out[k] = v

omnigibson/utils/numpy_utils.py

@@ -19,3 +19,7 @@ def vtarray_to_torch(vtarray, dtype=th.float32, device="cpu"):
 
 def pil_to_tensor(pil_image):
     return th.tensor(np.array(pil_image), dtype=th.uint8)
+
+
+def list_to_np_array(list):
+    return np.array(list)

omnigibson/utils/python_utils.py

@@ -790,3 +790,39 @@ def h5py_group_to_torch(group):
         else:
             state[key] = th.tensor(value[()], dtype=th.float32)
     return state
+
+
+@th.jit.script
+def multi_dim_linspace(start: th.Tensor, stop: th.Tensor, num: int) -> th.Tensor:
+    """
+    Generate a tensor with evenly spaced values along multiple dimensions.
+    This function creates a tensor where each slice along the first dimension
+    contains values linearly interpolated between the corresponding elements
+    of 'start' and 'stop'. It's similar to numpy.linspace but works with
+    multi-dimensional inputs in PyTorch.
+    Args:
+        start (th.Tensor): Starting values for each dimension.
+        stop (th.Tensor): Ending values for each dimension.
+        num (int): Number of samples to generate along the interpolated dimension.
+    Returns:
+        th.Tensor: A tensor of shape (num, *start.shape) containing the interpolated values.
+    Example:
+        >>> start = th.tensor([0, 10, 100])
+        >>> stop = th.tensor([1, 20, 200])
+        >>> result = multi_dim_linspace(start, stop, num=5)
+        >>> print(result.shape)
+        torch.Size([5, 3])
+        >>> print(result)
+        tensor([[  0.0000,  10.0000, 100.0000],
+                [  0.2500,  12.5000, 125.0000],
+                [  0.5000,  15.0000, 150.0000],
+                [  0.7500,  17.5000, 175.0000],
+                [  1.0000,  20.0000, 200.0000]])
+    """
+    steps = th.linspace(0, 1, num, dtype=start.dtype, device=start.device)
+
+    # Create a new shape for broadcasting
+    new_shape = [num] + [1] * start.dim()
+    steps = steps.reshape(new_shape)
+
+    return start + steps * (stop - start)

omnigibson/utils/vision_utils.py

@@ -156,7 +156,7 @@ def randomize_colors(N, bright=True):
     hsv = [(1.0 * i / N, 1, brightness) for i in range(N)]
     colors = th.tensor(list(map(lambda c: colorsys.hsv_to_rgb(*c), hsv)))
     colors = colors[th.randperm(colors.size(0))]
-    colors[0] = [0, 0, 0]  # First color is black
+    colors[0] = th.tensor([0, 0, 0], dtype=th.float32)  # First color is black
     return colors
 
 
@@ -173,7 +173,7 @@ def segmentation_to_rgb(seg_im, N, colors=None):
             to different segmentation IDs. Otherwise, will be generated randomly
     """
     # ensure all values lie within [0, N]
-    seg_im = th.fmod(seg_im, N)
+    seg_im = th.fmod(seg_im, N).cpu()
 
     if colors is None:
         use_colors = randomize_colors(N=N, bright=True)