Structural changes for BundleSDF-PLL cyclic pipeline

.gitignore

@@ -137,19 +137,27 @@ dmypy.json
 .pyre/
 
 # Datasets
+data_by*
 data/
+data_cube_2/
 logs/
 results/
+results_by*
+geometry/
 output/
 dataset/
 !BundleTrack/XMem/inference/data
 run_used.yml
 cn_traj/
+frames_annotated*
+frames_to_annotate*
 
 # IDE
 .vscode
 
 # Data files
 *.npy
 *.png
-*.obj
+*.obj
+!media/*
+!assets/**/*

BundleTrack/.gitignore

@@ -27,4 +27,6 @@ lf-net-release/release
 masks/
 *.tar.gz
 *.pth
-!SwiftNet/swiftnet_resnet18_old.pth
+!SwiftNet/swiftnet_resnet18_old.pth
+
+LoFTR/weights

BundleTrack/LoFTR/demo/demo_loftr.py

@@ -15,7 +15,7 @@
 import numpy as np
 import matplotlib.cm as cm
 
-os.sys.path.append("../")  # Add the project directory
+os.sys.path.append("../")  # Add the BundleTrack project directory
 from src.loftr import LoFTR, default_cfg
 from src.config.default import get_cfg_defaults
 try:

BundleTrack/config_behave.yml

@@ -1,4 +1,10 @@
-data_dir: /mnt/9a72c439-d0a7-45e8-8d20-d7a235d02763/DATASET/behave/BOWEN_ADDON/Date03_Sub03_boxmedium.2.color
+# NOTE: This is NOT the BundleSDF configuration file for BundleSDF/PLL
+# collaboration project.  Make edits in assets/config_toss_nerf.yml instead for
+# that project.
+
+bundlesdf_run_id: null    # To be filled in for an experiment.
+
+video_dir: /mnt/9a72c439-d0a7-45e8-8d20-d7a235d02763/DATASET/behave/BOWEN_ADDON/Date03_Sub03_boxmedium.2.color
 model_name: textured_simple
 model_dir: ""
 debug_dir: /home/bowen/debug/BundleTrack/

BundleTrack/config_ho3d.yml

@@ -1,4 +1,10 @@
-data_dir: /mnt/9a72c439-d0a7-45e8-8d20-d7a235d02763/DATASET/HO3D_v3/evaluation/SM1
+# NOTE: This is NOT the BundleSDF configuration file for BundleSDF/PLL
+# collaboration project.  Make edits in assets/config_toss_nerf.yml instead for
+# that project.
+
+bundlesdf_run_id: null    # To be filled in for an experiment.
+
+video_dir: /mnt/9a72c439-d0a7-45e8-8d20-d7a235d02763/DATASET/HO3D_v3/evaluation/SM1
 model_name: textured_simple
 model_dir: ""
 debug_dir: /home/bowen/debug/BundleTrack/

BundleTrack/config_toss.yml

@@ -0,0 +1,121 @@
+# BundleSDF configuration file for BundleSDF/PLL collaboration project.
+
+bundlesdf_run_id: null    # To be filled in for an experiment.
+
+video_dir: /mnt/9a72c439-d0a7-45e8-8d20-d7a235d02763/DATASET/HO3D_v3/evaluation/SM1
+model_name: textured_simple
+model_dir: ""
+debug_dir: /home/bowen/debug/BundleTrack/
+init_pose_dir: ""
+SPDLOG: 2
+port: '5555'
+seg_port: '1111'
+nerf_port: "9999"
+
+toss_frames: []
+
+visible_angle: 70   # Angle between normal and point to camera origin within XXX
+                    # is regarded as visible
+erode_mask: 3
+
+segmentation:
+  ob_scales: [0.3,0.3,0.3]
+  tolerance: 0.03
+
+depth_processing:
+  zfar: 2.0 # 1 is too small for cube dataset
+  erode:
+    radius: 1
+    diff: 0.001
+    ratio: 0.8   #if ratio larger than this, depth set to 0
+  bilateral_filter:
+    radius: 2
+    sigma_D: 2
+    sigma_R: 100000
+  outlier_removal:
+    num: 30
+    std_mul: 3
+  edge_normal_thres: 10  #deg between normal and ray
+  denoise_cloud: False
+  percentile: 95
+
+bundle:
+  num_iter_outter: 7
+  num_iter_inner: 5
+  window_size: 5  #exclude keyframes, include new frame
+  max_BA_frames: 10
+  subset_selection_method: normal_orientation_nearest
+  depth_association_radius: 5   # 0: findDenseCorr;
+                                # 1: findDenseCorrNearestNeighbor3D
+  non_neighbor_max_rot: 90
+  non_neighbor_min_visible: 0.1   # ratio of pixel visible
+  icp_pose_rot_thres: 60    # Rotation larger than XX deg is ignored for icp
+  w_rpi: 0
+  w_p2p: 1    # Used in loss.cpp
+  w_fm: 1
+  w_sdf: 0
+  w_pm: 0
+  robust_delta: 0.005
+  min_fm_edges_newframe: 15
+  image_downscale: [4]
+  feature_edge_dist_thres: 0.01
+  feature_edge_normal_thres: 30   # Normal angle should be within this range
+  max_optimized_feature_loss: 0.03
+
+keyframe:
+  min_interval: 1
+  min_feat_num: 0
+  min_trans: 0
+  min_rot: 10  # 5
+  min_visible: 1
+
+sift:
+  scales: [2,4,8]
+  max_match_per_query: 5
+  nOctaveLayers: 3
+  contrastThreshold: 0.01
+  edgeThreshold: 50
+  sigma: 1.6
+
+feature_corres:
+  mutual: True
+  map_points: True
+  max_dist_no_neighbor: 0.02  # 0.01
+  max_normal_no_neighbor: 45  # 20
+  max_dist_neighbor: 0.03  # 0.02
+  max_normal_neighbor: 45  # 30
+  suppression_patch_size: 5
+  max_view_normal_angle: 180
+  min_match_with_ref: 5
+  resize: 400
+  rematch_after_nerf: True
+
+ransac:
+  max_iter: 2000
+  num_sample: 3
+  inlier_dist: 0.01
+  inlier_normal_angle: 20
+  desired_succ_rate: 0.99
+  max_trans_neighbor: 0.02   #ransac model estimated pose shouldnt be too far
+  max_rot_deg_neighbor: 30
+  max_trans_no_neighbor: 0.01
+  max_rot_no_neighbor: 10
+  epipolar_thres: 1
+  min_match_after_ransac: 5
+
+p2p:
+  projective: false
+  max_dist: 0.02
+  max_normal_angle: 45
+
+sdf_edge:
+  max_dist: 0.02
+
+shape:
+  res: 0.005
+  xrange: [-0.2,0.2]
+  yrange: [-0.2,0.2]
+  zrange: [-0.2,0.2]
+  max_weight: 100
+  truncate_dist: 0.005
+

BundleTrack/config_ycbineoat.yml

@@ -1,4 +1,10 @@
-data_dir: /home/bowen/debug/ycbineoat
+# NOTE: This is NOT the BundleSDF configuration file for BundleSDF/PLL
+# collaboration project.  Make edits in assets/config_toss_nerf.yml instead for
+# that project.
+
+bundlesdf_run_id: null    # To be filled in for an experiment.
+
+video_dir: /home/bowen/debug/ycbineoat
 model_name: textured_simple
 model_dir: ""
 debug_dir: /home/bowen/debug/BundleTrack/

BundleTrack/scripts/data_reader.py

@@ -13,16 +13,25 @@
 yaml = ruamel.yaml.YAML()
 code_dir = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(f'{code_dir}/../../')
-from Utils import *
+from bundlenets.Utils import *
 
 HO3D_ROOT = '/mnt/9a72c439-d0a7-45e8-8d20-d7a235d02763/DATASET/HO3D_v3'
 
 
 class YcbineoatReader:
-  def __init__(self,video_dir, downscale=1, shorter_side=None):
+  def __init__(self,video_dir, downscale=1, shorter_side=None, occluded=None,offset_frames=1):
     self.video_dir = video_dir
     self.downscale = downscale
-    self.color_files = sorted(glob.glob(f"{self.video_dir}/rgb/*.png"))
+    occ_label = '' if occluded is None else f'_{occluded}'
+    offset_label = '' if offset_frames == 1 else f'_offset_{offset_frames}'
+    self.rgb_folder = f'rgb{occ_label}{offset_label}'
+    self.mask_folder = f'masks{occ_label}{offset_label}'
+    self.depth_folder = f'depth{offset_label}'
+    self.gt_pose_folder = f'annotated_poses{offset_label}'
+    # Check if depth folder exists
+    if not os.path.exists(f"{self.video_dir}/{self.depth_folder}"):
+      self.depth_folder = 'depth'
+    self.color_files = sorted(glob.glob(f"{self.video_dir}/{self.rgb_folder}/*.png"))
     self.K = np.loadtxt(f'{video_dir}/cam_K.txt').reshape(3,3)
     self.id_strs = []
     for color_file in self.color_files:
@@ -37,7 +46,7 @@ def __init__(self,video_dir, downscale=1, shorter_side=None):
     self.W = int(self.W*self.downscale)
     self.K[:2] *= self.downscale
 
-    self.gt_pose_files = sorted(glob.glob(f'{self.video_dir}/annotated_poses/*'))
+    self.gt_pose_files = sorted(glob.glob(f'{self.video_dir}/{self.gt_pose_folder}/*'))
 
     self.videoname_to_object = {
       'bleach0': "021_bleach_cleanser",
@@ -73,14 +82,14 @@ def get_color(self,i):
     return color
 
   def get_mask(self,i):
-    mask = cv2.imread(self.color_files[i].replace('rgb','masks'),-1)
+    mask = cv2.imread(self.color_files[i].replace(self.rgb_folder, self.mask_folder),-1)
     if len(mask.shape)==3:
       mask = (mask.sum(axis=-1)>0).astype(np.uint8)
     mask = cv2.resize(mask, (self.W,self.H), interpolation=cv2.INTER_NEAREST)
     return mask
 
   def get_depth(self,i):
-    depth = cv2.imread(self.color_files[i].replace('rgb','depth'),-1)/1e3
+    depth = cv2.imread(self.color_files[i].replace(self.rgb_folder, self.depth_folder),-1)/1e3
     depth = cv2.resize(depth, (self.W,self.H), interpolation=cv2.INTER_NEAREST)
     return depth
 
@@ -91,12 +100,12 @@ def get_xyz_map(self,i):
     return xyz_map
 
   def get_occ_mask(self,i):
-    hand_mask_file = self.color_files[i].replace('rgb','masks_hand')
+    hand_mask_file = self.color_files[i].replace(self.rgb_folder,'masks_hand')
     occ_mask = np.zeros((self.H,self.W), dtype=bool)
     if os.path.exists(hand_mask_file):
       occ_mask = occ_mask | (cv2.imread(hand_mask_file,-1)>0)
 
-    right_hand_mask_file = self.color_files[i].replace('rgb','masks_hand_right')
+    right_hand_mask_file = self.color_files[i].replace(self.rgb_folder,'masks_hand_right')
     if os.path.exists(right_hand_mask_file):
       occ_mask = occ_mask | (cv2.imread(right_hand_mask_file,-1)>0)
 
@@ -181,5 +190,5 @@ def get_gt_pose(self,i):
     else:
       ob_in_cam_gt[:3,3] = meta['objTrans']
       ob_in_cam_gt[:3,:3] = cv2.Rodrigues(meta['objRot'].reshape(3))[0]
-      ob_in_cam_gt = glcam_in_cvcam@ob_in_cam_gt
+      ob_in_cam_gt = GLCAM_IN_CVCAM@ob_in_cam_gt
     return ob_in_cam_gt

BundleTrack/src/Bundler.cpp

@@ -262,7 +262,9 @@ void Bundler::processNewFrame(std::shared_ptr<Frame> frame)
 
 bool Bundler::checkAndAddKeyframe(std::shared_ptr<Frame> frame)
 {
-  if (frame->_id==0)
+  const auto toss_frames = (*yml)["toss_frames"].as<std::vector<int>>();
+  int cnt = count(toss_frames.begin(), toss_frames.end(), frame->_id + 1); 
+  if (frame->_id==0 || cnt>0)
   {
     _keyframes.push_back(frame);
     SPDLOG("Added frame {} as keyframe, current #keyframe: {}", frame->_id_str, _keyframes.size());