Feature/isaac demo

.gitmodules

@@ -13,3 +13,21 @@
 [submodule "third_party/ObjectClear"] # for object clearing [reconstruction]
 	path = third_party/ObjectClear
 	url = [email protected]:PointsCoder/ObjectClear.git
+[submodule "third_party/graspness_unofficial"] # for grasp generation [motion]
+	path = third_party/graspness_unofficial
+	url = [email protected]:PointsCoder/graspness_unofficial.git
+[submodule "third_party/graspnetAPI"] # for grasp evaluation and utilities [motion]
+	path = third_party/graspnetAPI
+	url = [email protected]:PointsCoder/graspnetAPI.git
+[submodule "third_party/MinkowskiEngine"] # for sparse convolution operations [motion]
+	path = third_party/MinkowskiEngine
+	url = [email protected]:DynamoDuan/MinkowskiEngine.git
+[submodule "third_party/WiLoR"] # for hand extraction [motion]
+	path = third_party/WiLoR
+	url = [email protected]:DynamoDuan/WiLoR.git
+[submodule "third_party/curobo"]
+	path =  third_party/curobo
+	url = [email protected]:DynamoDuan/curobo.git
+[submodule "third_party/IsaacLab"]
+	path =  third_party/IsaacLab
+	url = [email protected]:PointsCoder/IsaacLab.git

config/config.yaml

@@ -1,7 +1,7 @@
 keys:
-  - demo_image
+#
   - demo_video
-  - demo_genvideo
+  # - demo_genvideo
 
 # global settings, can be overridden by local settings
 global:

docker/isaaclab/Dockerfile

@@ -25,62 +25,41 @@ ENV CMAKE_BUILD_PARALLEL_LEVEL=2
 ENV CUDA_HOME=/usr/local/cuda
 ENV FORCE_CUDA=1
 
-RUN useradd -ms /bin/bash isaac && chown -R isaac:isaac /app
-ENV HOME=/home/isaac
-USER isaac
-ENV PATH=$HOME/.local/bin:$PATH
-
 # RUN pip install --no-cache-dir --index-url https://download.pytorch.org/whl/cu118 \
 #     torch==2.5.1+cu118 torchvision==0.20.1+cu118 torchaudio==2.5.1+cu118
 
-RUN pip install torch==2.7.0 torchvision==0.22.0 --index-url https://download.pytorch.org/whl/cu128
-
-# graspnetapi (will install open3d & change numpy version from 1.26.4 to 1.23.4)
-RUN git clone https://github.com/PointsCoder/graspnetAPI.git "$HOME/graspnetAPI"
-RUN cd "$HOME/graspnetAPI" && pip install -e .
-
+RUN pip install torch==2.7.0 torchvision==0.22.0 --index-url https://download.pytorch.org/whl/cu128 
+RUN pip install transforms3d
 
-ENV TORCH_CUDA_ARCH_LIST="7.5;8.0;8.6;8.9;9.0;12.0"
-# graspness (will compile pointnet2, knn)
-RUN git clone https://github.com/PointsCoder/graspness_unofficial.git "$HOME/third_party/graspness_unofficial"
-RUN cd "$HOME/third_party/graspness_unofficial/pointnet2" && python -m pip install --user -v --no-build-isolation .
-# RUN cd "$HOME/third_party/graspness_unofficial/knn" && python -m pip install --user -v --no-build-isolation . # (knn is not necessary)
 
 USER root
 # Patch shared_ptr_base.h
 RUN sed -i 's|auto __raw = __to_address(__r.get())|auto __raw = std::__to_address(__r.get())|' \
     /usr/include/c++/11/bits/shared_ptr_base.h
 
-USER isaac
-# MinkowskiEngine
-RUN git clone https://github.com/songlin/MinkowskiEngine.git "$HOME/MinkowskiEngine"
-RUN cd "$HOME/MinkowskiEngine" && git checkout cuda-12.8-cc-12.0  && export CUDA_HOME=/usr/local/cuda; python setup.py install --user --blas=openblas --force_cuda
-
-# checkpoints
-RUN pip install gdown && \
-    mkdir -p "$HOME/third_party/graspness_unofficial/ckpt" && \
-    gdown 10o5fc8LQsbI8H0pIC2RTJMNapW9eczqF -O "$HOME/third_party/graspness_unofficial/ckpt/minkuresunet_kinect.tar" && \
-    gdown 1RfdpEM2y0x98rV28d7B2Dg8LLFKnBkfL -O "$HOME/third_party/graspness_unofficial/ckpt/minkuresunet_realsense.tar"
-
 ENV TORCH_CUDA_ARCH_LIST="8.9+PTX"
-# curobo
-RUN git clone https://github.com/NVlabs/curobo.git "$HOME/curobo" && \
-    cd "$HOME/curobo" && pip install -e . --no-build-isolation --user
+# curobo - install to /opt to avoid being overwritten by volume mount
+COPY third_party/curobo /app/third_party/curobo
+RUN cd /app/third_party/curobo && \
+     pip install -e . --no-build-isolation
 
 # Isaac Sim 4.5.0
 RUN pip install 'isaacsim[all,extscache]==4.5.0' --extra-index-url https://pypi.nvidia.com
 
 # Revert the fixed pytorch version
+# Install IsaacLab to /opt to avoid being overwritten by volume mount
 ENV TERM=xterm-256color
-RUN git clone https://github.com/PointsCoder/IsaacLab.git "$HOME/IsaacLab" && \
-    cd "$HOME/IsaacLab" && \
+COPY third_party/IsaacLab /app/third_party/IsaacLab
+RUN cd /app/third_party/IsaacLab && \
     sed -i 's|torch==2.5.1|torch==2.7.0|' source/isaaclab/setup.py && \
     sed -i 's|torch==2.5.1|torch==2.7.0|' source/isaaclab_tasks/setup.py && \
     sed -i 's|torch==2.5.1|torch==2.7.0|' source/isaaclab_rl/setup.py && \
     ./isaaclab.sh --install
 
 # usd conversion 
 # curobo changes the numpy version from 1.23.4 to 2.x, need to get it back
-RUN pip install loguru rich numpy==1.23.4
+RUN pip install loguru rich numpy==1.23.4 transforms3d
 
-ENV PYTHONPATH=/app:$HOME/third_party:$HOME/third_party/graspness_unofficial/pointnet2
+# Ensure isaaclab can be imported
+# IsaacLab is installed to /opt/IsaacLab to avoid being overwritten by volume mount
+ENV PYTHONPATH=/app:/app/third_party:/app/third_party/IsaacLab/source

docker/real2sim/Dockerfile

@@ -8,7 +8,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 # -------------------------------------------------------------------------
 RUN apt-get update && apt-get install -y --no-install-recommends \
     python3 python3-dev python3-pip python3-venv python-is-python3 \
-    build-essential cmake ninja-build pkg-config \
+    build-essential cmake ninja-build pkg-config libopenblas-dev \
     git curl wget ffmpeg bash libspatialindex-dev \
     && python3 -m pip install --upgrade --no-cache-dir pip wheel setuptools packaging \
     && rm -rf /var/lib/apt/lists/*
@@ -46,6 +46,9 @@ RUN pip install --no-cache-dir torch-scatter \
 COPY docker/real2sim/requirements.docker.txt         /tmp/requirements.docker.txt
 RUN pip install --no-cache-dir -r /tmp/requirements.docker.txt
 
+# Install chumpy separately (setup.py needs pip module, ensure build deps are available)
+RUN pip install --no-build-isolation --no-cache-dir "git+https://github.com/mattloper/chumpy"
+
 
 # -------------------------------------------------------------------------
 # Third-Party compiled Python deps
@@ -82,7 +85,18 @@ RUN cd / && wget https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.t
     cmake .. &&\
     make install
 
+RUN cd /
+COPY third_party/graspnetAPI /app/third_party/graspnetAPI
+RUN cd /app/third_party/graspnetAPI && pip install -e .
+
+COPY third_party/graspness_unofficial /app/third_party/graspness_unofficial
+RUN cd /app/third_party/graspness_unofficial/pointnet2 && python -m pip install --no-cache-dir -v --no-build-isolation .
+RUN cd /app/third_party/graspness_unofficial/knn && python -m pip install --no-cache-dir -v --no-build-isolation .
+
+COPY third_party/MinkowskiEngine /app/third_party/MinkowskiEngine
+RUN cd /app/third_party/MinkowskiEngine && python setup.py install --blas=openblas --force_cuda
+
 # -------------------------------------------------------------------------
 # Runtime env
 # -------------------------------------------------------------------------
-ENV PYTHONPATH=/app
+ENV PYTHONPATH=/app:/app/third_party:/app/third_party/graspness_unofficial/pointnet2:

docker/real2sim/requirements.docker.txt

@@ -20,6 +20,7 @@ scipy==1.15.3
 wandb==0.21.3
 gdown
 
+
 #__________________________________________________________________________________
 # 2) Related deps (torch-scatter is separate)
 ninja==1.11.1
@@ -61,4 +62,27 @@ Rtree
 # 8) Foundation Pose
 transformations==2024.6.1
 ruamel.yaml==0.18.6
-h5py==3.10.0
+h5py==3.10.0
+
+# 9) PromptDA
+pyliblzfse
+lightning==2.1.3
+termcolor
+tyro==0.9.2
+spaces
+trimesh 
+
+# 10) WiLoR
+pyrender
+pytorch-lightning
+scikit-image
+smplx==0.1.28
+yacs
+xtcocotools
+pandas
+hydra-submitit-launcher
+hydra-colorlog
+pyrootutils
+webdataset
+ultralytics==8.1.34
+# Note: chumpy installed separately in Dockerfile due to build isolation issues

openreal2sim/simulation/isaaclab-old/README.md

@@ -0,0 +1,52 @@
+# IsaacLab
+
+## Installation
+
+**Pull the pre-built docker image**
+
+If you want to use a pre-built image from Docker Hub, you can pull it directly:
+   ```bash
+   docker pull ghcr.io/pointscoder/isaaclab:dev
+   docker tag ghcr.io/pointscoder/isaaclab:dev isaaclab:dev
+   ```
+
+**On the host machine and before entering the container**, run
+```
+xhost +local:
+```
+This is to allow the container to access the host's X server for IsaacSim GUI.
+
+Then, launch the container:
+```
+HOST_UID=$(id -u) HOST_GID=$(id -g) docker compose -p "$USER" -f docker/compose.yml up -d isaaclab 
+```
+and enter it:
+```
+HOST_UID=$(id -u) HOST_GID=$(id -g) docker compose -p "$USER" -f docker/compose.yml exec isaaclab bash
+```
+
+## Preprocessing
+
+We need to convert the textured meshes to USD format. This is done by running the following script:
+```
+python openreal2sim/simulation/isaaclab/sim_preprocess/usd_conversion.py
+```
+
+Then, we generate grasp proposals for each object in the scene:
+```
+python openreal2sim/simulation/isaaclab/sim_preprocess/grasp_generation.py
+```
+
+## Running the simulation
+
+We provide heuristic policies using grasping and motion planning:
+```
+python openreal2sim/simulation/isaaclab/sim_heuristic_manip.py
+```
+
+## Replay Robot Trajectories
+
+We can also replay the recorded robot trajectories in IsaacSim:
+```
+python openreal2sim/simulation/isaaclab/sim_replay_trajectories.py --demo_dir <path_to_demo_directory>
+```

openreal2sim/simulation/isaaclab-old/demo/README.md

@@ -0,0 +1,137 @@
+### Data Collection and Generation ###
+This folder is used for collecting manipulation data and augmenting it with spatial transformation.
+
+The demo pipeline consists of three main stages:
+1. **USD Conversion** - Converts scene data to USD format for simulation
+2. **Heuristic Manipulation** - Performs object manipulation using heuristic policies with grasping and motion planning. This will create a new folder in `task` and `h5py` folder for each key. In the task folder, a json is formulated, and required assets for the scene are structured. 
+3. **Randomized Rollout** - Generates randomized scene configurations and collects successful manipulation trajectories. This will add demos to the `task` folder and trajectory configs to the task config, and add new h5py file to the `h5py` folder.
+
+### Code Structure ###
+
+```
+demo/
+├── sim_agent.sh                    # Main pipeline script that runs all stages
+├── sim_heuristic_manip.py          # Heuristic manipulation policy implementation
+├── sim_randomize_rollout.py        # Randomized rollout for data augmentation
+└── envs/                           # Environment configuration and utilities
+    ├── task_cfg.py                 # Task configuration classes (TaskCfg, TrajectoryCfg, etc.)
+    ├── task_construct.py           # Task construction and serialization utilities
+    ├── randomizer.py               # Scene randomization for data augmentation
+    └── running_cfg.py              # Runtime configuration management
+```
+
+### Running Script ###
+Please run ` bash openreal2sim/simulation/isaaclab/demo/sim_agent.sh` to collect the demo. This will iterate through the data folder and run the three stages for each input key.
+
+### Running Config Explanation ###
+There are two editable configs for this script:
+
+First, running device and headless option are defined in the bash script.
+
+Second, the randomizer config, environment num and total trajectry num are all defined in the `running_cfg.py` file. Please edit that file for your own needs.
+
+
+#### TaskCfg (Main Task Configuration)
+
+The top-level configuration that defines a complete manipulation task:
+
+- **task_key** (str): Unique identifier for the task
+- **task_id** (int): Sequential task ID
+- **task_desc** (List[str]): Description of the task
+- **task_type** (TaskType): Type of task (SIMPLE_PICK, SIMPLE_PICK_PLACE, TARGETTED_PICK_PLACE)
+- **bg_rgb_path** (str): Path to background RGB image
+- **background_path** (str): Path to background mesh (GLB format)
+- **background_usd_path** (str): Path to background USD file
+- **camera_info** (CameraInfo): Camera intrinsic and extrinsic parameters
+- **manipulated_oid** (int): Object ID of the manipulated object
+- **start_related** (List[int]): List of object IDs related to the start state
+- **end_related** (List[int]): List of object IDs related to the end state
+- **objects** (List[ObjectCfg]): List of all objects in the scene
+- **reference_trajectory** (TrajectoryCfg, optional): Reference trajectory generated by heuristic manipulation
+- **generated_trajectories** (List[TrajectoryCfg]): List of randomized trajectories for data augmentation
+
+#### TrajectoryCfg (Trajectory Configuration)
+
+Defines a specific manipulation trajectory:
+
+- **robot_pose** (List[float]): Robot base pose [x, y, z, qw, qx, qy, qz] in world frame
+- **object_poses** (Dict[int, List[float]]): Initial poses of all objects, keyed by object ID
+- **object_trajectory** (List[List[float]]): Object trajectory as sequence of poses [x, y, z, qw, qx, qy, qz]
+- **final_gripper_close** (bool): Whether gripper should be closed at the end
+- **success_metric** (SuccessMetric): Criteria for task success
+- **pregrasp_pose** (List[float], optional): Pre-grasp end-effector pose in world frame
+- **grasp_pose** (List[float], optional): Grasp end-effector pose in world frame
+- **robot_type** (RobotType, optional): Type of robot (FRANKA or UR5)
+
+#### SuccessMetric (Success Criteria)
+
+Defines how task success is evaluated:
+
+- **success_metric_type** (SuccessMetricType): Type of success metric
+  - `SIMPLE_LIFT`: Object must be lifted by a certain height
+  - `TARGET_POINT`: Object must reach a specific target pose
+  - `TARGET_PLANE`: Object must reach a target plane (height)
+- **final_gripper_close** (bool): Whether gripper should be closed at success
+- **lift_height** (float, optional): Required lift height for SIMPLE_LIFT
+- **ground_value** (float, optional): Target height for TARGET_PLANE
+- **end_pose** (List[float], optional): Target pose [x, y, z, qw, qx, qy, qz] for TARGET_POINT
+
+#### ObjectCfg (Object Configuration)
+
+Defines an object in the scene:
+
+- **object_id** (int): Unique object identifier
+- **object_name** (str): Name of the object
+- **mesh_path** (str): Path to object mesh file (GLB format)
+- **usd_path** (str): Path to object USD file
+
+#### CameraInfo (Camera Configuration)
+
+Camera intrinsic and extrinsic parameters:
+
+- **width, height** (float): Image dimensions
+- **fx, fy** (float): Focal lengths
+- **cx, cy** (float): Principal point
+- **camera_opencv_to_world** (List[List[float]]): 4x4 transformation matrix from OpenCV to world frame
+- **camera_position** (List[float]): Camera position [x, y, z]
+- **camera_heading_wxyz** (List[float]): Camera orientation quaternion [w, x, y, z]
+
+#### Configuration Workflow
+
+1. **Initial Task Creation**: `construct_task_config()` creates a `TaskCfg` from scene reconstruction data
+2. **Reference Trajectory**: `add_reference_trajectory()` adds the first successful trajectory from heuristic manipulation
+3. **Data Augmentation**: `add_generated_trajectories()` adds randomized trajectories from rollout stage
+4. **Serialization**: Task configs are saved as JSON files in `tasks/<key>/task.json`
+5. **Loading**: `load_task_cfg()` reconstructs `TaskCfg` objects from JSON files
+
+#### Example Task Configuration Structure
+
+```json
+{
+  "task_key": "demo_video",
+  "task_id": 0,
+  "task_type": "TARGETTED_PICK_PLACE",
+  "manipulated_oid": 2,
+  "objects": [
+    {
+      "object_id": 2,
+      "object_name": "pen",
+      "mesh_path": "tasks/demo_video/object_2.glb",
+      "usd_path": "tasks/demo_video/object_2.usd"
+    }
+  ],
+  "reference_trajectory": {
+    "robot_pose": [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0],
+    "object_poses": {"2": [0.1, 0.2, 0.05, 1.0, 0.0, 0.0, 0.0]},
+    "object_trajectory": [[...], [...], ...],
+    "pregrasp_pose": [0.1, 0.2, 0.15, 1.0, 0.0, 0.0, 0.0],
+    "grasp_pose": [0.1, 0.2, 0.1, 1.0, 0.0, 0.0, 0.0],
+    "success_metric": {
+      "success_metric_type": "TARGET_POINT",
+      "end_pose": [0.3, 0.4, 0.1, 1.0, 0.0, 0.0, 0.0],
+      "final_gripper_close": false
+    }
+  },
+  "generated_trajectories": [...]
+}
+```