Code release.

.gitignore

@@ -0,0 +1,23 @@
+*.pyc
+data
+output
+build
+diff_rasterization/diff_rast.egg-info
+diff_rasterization/dist
+submodules/
+SIBR_viewers/
+submodules/*
+*.dump
+pointops2/pointops2.egg-info
+simple-knn/simple_knn.egg-info
+__pycache__/
+*.so
+.Python
+.vscode/
+build/
+develop-eggs/
+dist/
+eggs/
+.eggs/
+*.egg
+*.sh

README.md

@@ -1,16 +1,54 @@
 # Real-time Photorealistic Dynamic Scene Representation and Rendering with 4D Gaussian Splatting
-### [Paper](https://arxiv.org/abs/)
+### [Project page](https://fudan-zvg.github.io/4d-gaussian-splatting/) | [Paper](https://arxiv.org/abs/2310.10642)
 > [**Real-time Photorealistic Dynamic Scene Representation and Rendering with 4D Gaussian Splatting**](https://arxiv.org/abs/2310.10642),            
 > Zeyu Yang, Hongye Yang, Zijie Pan, [Xiatian Zhu](https://surrey-uplab.github.io/), [Li Zhang](https://lzrobots.github.io)  
 > **Arxiv preprint**
 
 **This repository is the official implementation of "Real-time Photorealistic Dynamic Scene Representation and Rendering with 4D Gaussian Splatting".** In this paper, we propose coherent integrated modeling of the space and time dimensions for dynamic scenes by formulating unbiased 4D Gaussian primitives along with a dedicated rendering pipeline.
 
+
 ## 🛠️ Pipeline
 <div align="center">
   <img src="assets/pipeline.png"/>
 </div><br/>
 
+
+## Get started
+
+### Environment
+
+The hardware and software requirements are the same as those of the [3D Gaussian Splatting](https://github.com/graphdeco-inria/gaussian-splatting), which this code is built upon. To setup the environment, please run the following command:
+
+```shell
+git clone https://github.com/fudan-zvg/4d-gaussian-splatting
+cd 4d-gaussian-splatting
+conda env create --file environment.yml
+conda activate 4dgs
+```
+
+### Data preparation
+
+**DyNeRF dataset:**
+
+Download the [Neural 3D Video dataset](https://fudan-zvg.github.io/4d-gaussian-splatting/) and extract each scene to `data/N3V`. After that, preprocess the raw video by executing:
+
+```shell
+python scripts/n3v2blender.py data/N3V/$scene_name
+```
+
+**DNeRF dataset:**
+
+The dataset can be downloaded from [drive](https://drive.google.com/file/d/19Na95wk0uikquivC7uKWVqllmTx-mBHt/view?usp=sharing) or [dropbox](https://www.dropbox.com/s/0bf6fl0ye2vz3vr/data.zip?dl=0). Then, unzip each scene into `data/dnerf`.
+
+
+### Running
+
+After the installation and data preparation, you can train the model by running:
+
+```shell
+python train.py --config $config_path
+```
+
 ## 🎥 Videos
 
 ### 🎞️ Dynamic novel view synthesis
@@ -30,7 +68,6 @@ https://github.com/fudan-zvg/4d-gaussian-splatting/assets/45744267/6bd0b57b-4857
 https://github.com/fudan-zvg/4d-gaussian-splatting/assets/45744267/2c79974c-1867-4ce6-848b-5d31679b6067
 
 
-
 ## 📜 Reference
 ```bibtex
 @article{yang2023gs4d,

arguments/__init__.py

@@ -0,0 +1,129 @@
+#
+# Copyright (C) 2023, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use 
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact  [email protected]
+#
+
+from argparse import ArgumentParser, Namespace
+import sys
+import os
+
+class GroupParams:
+    pass
+
+class ParamGroup:
+    def __init__(self, parser: ArgumentParser, name : str, fill_none = False):
+        group = parser.add_argument_group(name)
+        for key, value in vars(self).items():
+            shorthand = False
+            if key.startswith("_"):
+                shorthand = True
+                key = key[1:]
+            t = type(value)
+            value = value if not fill_none else None 
+            if shorthand:
+                if t == bool:
+                    group.add_argument("--" + key, ("-" + key[0:1]), default=value, action="store_true")
+                else:
+                    group.add_argument("--" + key, ("-" + key[0:1]), default=value, type=t)
+            else:
+                if t == bool:
+                    group.add_argument("--" + key, default=value, action="store_true")
+                else:
+                    group.add_argument("--" + key, default=value, type=t)
+
+    def extract(self, args):
+        group = GroupParams()
+        for arg in vars(args).items():
+            if arg[0] in vars(self) or ("_" + arg[0]) in vars(self):
+                setattr(group, arg[0], arg[1])
+        return group
+
+class ModelParams(ParamGroup): 
+    def __init__(self, parser, sentinel=False):
+        self.sh_degree = 3
+        self._source_path = ""
+        self._model_path = ""
+        self._images = "images"
+        self._resolution = -1
+        self._white_background = False
+        self.data_device = "cuda"
+        self.eval = False
+        self.extension = ".png"
+        self.num_extra_pts = 0
+        self.loaded_pth = ""
+        self.frame_ratio = 1
+        self.dataloader = False
+        super().__init__(parser, "Loading Parameters", sentinel)
+
+    def extract(self, args):
+        g = super().extract(args)
+        g.source_path = os.path.abspath(g.source_path)
+        return g
+
+class PipelineParams(ParamGroup):
+    def __init__(self, parser):
+        self.convert_SHs_python = False
+        self.compute_cov3D_python = False
+        self.debug = False
+        self.env_map_res = 0
+        self.env_optimize_until = 1000000000
+        self.env_optimize_from = 0
+        self.eval_shfs_4d = False
+        super().__init__(parser, "Pipeline Parameters")
+
+class OptimizationParams(ParamGroup):
+    def __init__(self, parser):
+        self.iterations = 30_000
+        self.position_lr_init = 0.00016
+        self.position_t_lr_init = -1.0
+        self.position_lr_final = 0.0000016
+        self.position_lr_delay_mult = 0.01
+        self.position_lr_max_steps = 30_000
+        self.feature_lr = 0.0025
+        self.opacity_lr = 0.05
+        self.scaling_lr = 0.005
+        self.rotation_lr = 0.001
+        self.percent_dense = 0.01
+        self.lambda_dssim = 0.2
+        self.thresh_opa_prune = 0.005
+        self.densification_interval = 100
+        self.opacity_reset_interval = 3000
+        self.densify_from_iter = 500
+        self.densify_until_iter = 15_000
+        self.densify_grad_threshold = 0.0002
+        self.densify_grad_t_threshold = 0.0002 / 40
+        self.densify_until_num_points = -1
+        self.final_prune_from_iter = -1
+        self.sh_increase_interval = 1000
+        self.lambda_opa_mask = 0.0
+        self.lambda_rigid = 0.0
+        self.lambda_motion = 0.0
+        super().__init__(parser, "Optimization Parameters")
+
+def get_combined_args(parser : ArgumentParser):
+    cmdlne_string = sys.argv[1:]
+    cfgfile_string = "Namespace()"
+    args_cmdline = parser.parse_args(cmdlne_string)
+
+    try:
+        cfgfilepath = os.path.join(args_cmdline.model_path, "cfg_args")
+        print("Looking for config file in", cfgfilepath)
+        with open(cfgfilepath) as cfg_file:
+            print("Config file found: {}".format(cfgfilepath))
+            cfgfile_string = cfg_file.read()
+    except TypeError:
+        print("Config file not found at")
+        pass
+    args_cfgfile = eval(cfgfile_string)
+
+    merged_dict = vars(args_cfgfile).copy()
+    for k,v in vars(args_cmdline).items():
+        if v != None:
+            merged_dict[k] = v
+    return Namespace(**merged_dict)

configs/dnerf/bouncingballs.yaml

@@ -0,0 +1,59 @@
+gaussian_dim: 4
+time_duration: [0.0, 1.0]
+num_pts: 100_000
+num_pts_ratio: 1.0
+rot_4d: True
+force_sh_3d: False
+batch_size: 2
+exhaust_test: True
+
+ModelParams:
+  sh_degree: 3
+  source_path: "data/dnerf/bouncingballs"
+  model_path: "output/dnerf/bouncingballs/"
+  images: "images"
+  resolution: 2
+  white_background: False
+  data_device: "cuda"
+  eval: True
+  extension: ".png"
+  num_extra_pts: 0
+  loaded_pth: ""
+  frame_ratio: 1
+  dataloader: False
+
+PipelineParams:
+  convert_SHs_python: False
+  compute_cov3D_python: False
+  debug: False
+  env_map_res: 0
+  env_optimize_until: 1000000000
+  env_optimize_from: 0
+  eval_shfs_4d: True
+
+OptimizationParams:
+  iterations: 20_000
+  position_lr_init: 0.00016
+  position_t_lr_init: -1.0
+  position_lr_final: 0.0000016
+  position_lr_delay_mult: 0.01
+  position_lr_max_steps: 15_000
+  feature_lr: 0.0025
+  opacity_lr: 0.05
+  scaling_lr: 0.005
+  rotation_lr: 0.001
+  percent_dense: 0.01
+  lambda_dssim: 0.2
+  thresh_opa_prune: 0.005
+  densification_interval: 200
+  opacity_reset_interval: 3000
+  densify_from_iter: 500
+  densify_until_iter: 10_000
+  densify_grad_threshold: 0.0002
+  densify_grad_t_threshold: 0.0002 / 40 # 想办法用上
+  densify_until_num_points: -1
+  final_prune_from_iter: -1
+  sh_increase_interval: 1000
+  lambda_opa_mask: 0.0
+  lambda_rigid: 0.0
+  lambda_motion: 0.0

configs/dnerf/hellwarrior.yaml

@@ -0,0 +1,59 @@
+gaussian_dim: 4
+time_duration: [0.0, 1.0]
+num_pts: 100_000
+num_pts_ratio: 1.0
+rot_4d: True
+force_sh_3d: False
+batch_size: 1
+exhaust_test: True
+
+ModelParams:
+  sh_degree: 3
+  source_path: "data/dnerf/hellwarrior"
+  model_path: "output/dnerf/hellwarrior"
+  images: "images"
+  resolution: 2
+  white_background: False
+  data_device: "cuda"
+  eval: True
+  extension: ".png"
+  num_extra_pts: 0
+  loaded_pth: ""
+  frame_ratio: 1
+  dataloader: False
+
+PipelineParams:
+  convert_SHs_python: False
+  compute_cov3D_python: False
+  debug: False
+  env_map_res: 0
+  env_optimize_until: 1000000000
+  env_optimize_from: 0
+  eval_shfs_4d: True
+
+OptimizationParams:
+  iterations: 30_000
+  position_lr_init: 0.00016
+  position_t_lr_init: -1.0
+  position_lr_final: 0.0000016
+  position_lr_delay_mult: 0.01
+  position_lr_max_steps: 30_000
+  feature_lr: 0.0025
+  opacity_lr: 0.05
+  scaling_lr: 0.005
+  rotation_lr: 0.001
+  percent_dense: 0.01
+  lambda_dssim: 0.2
+  thresh_opa_prune: 0.005
+  densification_interval: 100
+  opacity_reset_interval: 3000
+  densify_from_iter: 500
+  densify_until_iter: 15_000
+  densify_grad_threshold: 0.0002
+  densify_grad_t_threshold: 0.0002 / 40 # 想办法用上
+  densify_until_num_points: -1
+  final_prune_from_iter: -1
+  sh_increase_interval: 1000
+  lambda_opa_mask: 0.0
+  lambda_rigid: 0.0
+  lambda_motion: 0.0

configs/dnerf/hook.yaml

@@ -0,0 +1,59 @@
+gaussian_dim: 4
+time_duration: [0.0, 1.0]
+num_pts: 300_000
+num_pts_ratio: 1.0
+rot_4d: True
+force_sh_3d: False
+batch_size: 8
+exhaust_test: True
+
+ModelParams:
+  sh_degree: 3
+  source_path: "data/dnerf/hook"
+  model_path: "output/dnerf/hook"
+  images: "images"
+  resolution: 2
+  white_background: False
+  data_device: "cuda"
+  eval: True
+  extension: ".png"
+  num_extra_pts: 0
+  loaded_pth: ""
+  frame_ratio: 1
+  dataloader: False
+
+PipelineParams:
+  convert_SHs_python: False
+  compute_cov3D_python: False
+  debug: False
+  env_map_res: 0
+  env_optimize_until: 1000000000
+  env_optimize_from: 0
+  eval_shfs_4d: True
+
+OptimizationParams:
+  iterations: 30_000
+  position_lr_init: 0.00016
+  position_t_lr_init: -1.0
+  position_lr_final: 0.0000016
+  position_lr_delay_mult: 0.01
+  position_lr_max_steps: 30_000
+  feature_lr: 0.0025
+  opacity_lr: 0.05
+  scaling_lr: 0.005
+  rotation_lr: 0.001
+  percent_dense: 0.01
+  lambda_dssim: 0.2
+  thresh_opa_prune: 0.005
+  densification_interval: 100
+  opacity_reset_interval: 3000
+  densify_from_iter: 500
+  densify_until_iter: 15_000
+  densify_grad_threshold: 0.0002
+  densify_grad_t_threshold: 0.0002 / 40 # 想办法用上
+  densify_until_num_points: -1
+  final_prune_from_iter: -1
+  sh_increase_interval: 1000
+  lambda_opa_mask: 0.0
+  lambda_rigid: 0.0
+  lambda_motion: 0.0