Upload 243 files

.gitattributes

@@ -33,3 +33,32 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/demo_gradio.gif filter=lfs diff=lfs merge=lfs -text
+assets/pipeline.jpg filter=lfs diff=lfs merge=lfs -text
+examples/video/bungeenerf_colosseum.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/video/dtu_scan_106.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/video/fillerbuster_hand_hand.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/video/fillerbuster_ramen.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/video/fox.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/video/horizongs_hillside_summer.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/video/kitti360.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/video/llff_fortress.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/video/llff_horns.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/video/matrixcity_street.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/video/meganerf_rubble.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/video/vrnerf_apartment.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/video/vrnerf_kitchen.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/video/vrnerf_riverview.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/video/vrnerf_workshop.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/vrnerf/riverview/21_DSC0001.jpg filter=lfs diff=lfs merge=lfs -text
+examples/vrnerf/riverview/21_DSC0010.jpg filter=lfs diff=lfs merge=lfs -text
+examples/vrnerf/riverview/21_DSC0019.jpg filter=lfs diff=lfs merge=lfs -text
+examples/vrnerf/riverview/21_DSC0028.jpg filter=lfs diff=lfs merge=lfs -text
+examples/vrnerf/riverview/21_DSC0037.jpg filter=lfs diff=lfs merge=lfs -text
+examples/vrnerf/riverview/21_DSC0046.jpg filter=lfs diff=lfs merge=lfs -text
+examples/vrnerf/riverview/21_DSC0055.jpg filter=lfs diff=lfs merge=lfs -text
+examples/vrnerf/riverview/21_DSC0064.jpg filter=lfs diff=lfs merge=lfs -text
+examples/vrnerf/riverview/21_DSC0073.jpg filter=lfs diff=lfs merge=lfs -text
+examples/vrnerf/riverview/21_DSC0082.jpg filter=lfs diff=lfs merge=lfs -text
+examples/vrnerf/riverview/21_DSC0091.jpg filter=lfs diff=lfs merge=lfs -text
+examples/vrnerf/riverview/21_DSC0100.jpg filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Lihan Jiang and Yucheng Mao
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

config/compute_metrics.yaml

@@ -0,0 +1,28 @@
+defaults:
+  - model/encoder: noposplat
+  - loss: []
+  - override dataset/[email protected]_sampler: evaluation
+
+dataset:
+  re10k:
+    view_sampler:
+      index_path: assets/evaluation_index_re10k.json
+
+data_loader:
+  train:
+    num_workers: 0
+    persistent_workers: true
+    batch_size: 1
+    seed: 1234
+  test:
+    num_workers: 4
+    persistent_workers: false
+    batch_size: 1
+    seed: 2345
+  val:
+    num_workers: 0
+    persistent_workers: true
+    batch_size: 1
+    seed: 3456
+
+seed: 111123

config/dataset/base_dataset.yaml

@@ -0,0 +1,7 @@
+make_baseline_1: true
+relative_pose: true
+augment: true
+background_color: [1.0, 1.0, 1.0]
+overfit_to_scene: null
+skip_bad_shape: true
+rescale_to_1cube: false

config/dataset/co3d.yaml

@@ -0,0 +1,15 @@
+defaults:
+  - base_dataset
+  - view_sampler: rank
+
+name: co3d
+roots: [datasets/co3dv2]
+
+input_image_shape: [256, 256]
+original_image_shape: [540, 960]
+cameras_are_circular: false
+
+baseline_min: 1e-3
+baseline_max: 1e2
+max_fov: 110.0
+avg_pose: false

config/dataset/dl3dv.yaml

@@ -0,0 +1,19 @@
+defaults:
+  - base_dataset
+  - view_sampler: bounded
+
+name: dl3dv
+roots: [datasets/dl3dv]
+
+input_image_shape: [256, 256]
+original_image_shape: [540, 960]
+cameras_are_circular: false
+
+baseline_min: 1e-3
+baseline_max: 1e2
+max_fov: 100.0
+avg_pose: false
+
+rescale_to_1cube: true
+make_baseline_1: false
+intr_augment: true

config/dataset/scannetpp.yaml

@@ -0,0 +1,22 @@
+defaults:
+  - base_dataset
+  - view_sampler: rank
+
+name: scannetpp
+roots: [datasets/scannetpp]
+
+input_image_shape: [256, 256]
+original_image_shape: [690, 1035]
+cameras_are_circular: false
+
+baseline_min: 1e-3
+baseline_max: 1e2
+max_fov: 130.0 # 120.0
+metric_thre: 0.5 # aggressive metric threshold!!
+
+skip_bad_shape: true # if use dlsr and iphone, set to false
+
+rescale_to_1cube: true
+make_baseline_1: false
+intr_augment: true
+normalize_by_pts3d: false

config/dataset/view_sampler/all.yaml

@@ -0,0 +1 @@
+name: all

config/dataset/view_sampler/arbitrary.yaml

@@ -0,0 +1,7 @@
+name: arbitrary
+
+num_target_views: 1
+num_context_views: 2
+
+# If you want to hard-code context views, do so here.
+context_views: null

config/dataset/view_sampler/bounded.yaml

@@ -0,0 +1,16 @@
+name: bounded
+
+num_target_views: 1
+num_context_views: 24
+
+min_distance_between_context_views: 2
+max_distance_between_context_views: 6
+min_distance_to_context_views: 0
+
+warm_up_steps: 0
+initial_min_distance_between_context_views: 2
+initial_max_distance_between_context_views: 6
+
+max_img_per_gpu: 24
+min_gap_multiplier: 3
+max_gap_multiplier: 5

config/dataset/view_sampler/evaluation.yaml

@@ -0,0 +1,4 @@
+name: evaluation
+
+index_path: assets/evaluation_index_re10k.json
+num_context_views: 2

config/dataset/view_sampler/rank.yaml

@@ -0,0 +1,14 @@
+name: rank
+
+num_target_views: 4
+num_context_views: 24
+
+min_distance_between_context_views: 8
+max_distance_between_context_views: 22
+min_distance_to_context_views: 0
+
+warm_up_steps: 0
+initial_min_distance_between_context_views: 5
+initial_max_distance_between_context_views: 7
+
+max_img_per_gpu: 24

config/experiment/co3d.yaml

@@ -0,0 +1,90 @@
+# @package _global_
+
+defaults:
+  - /dataset@_group_.co3d: co3d
+  - override /model/encoder: anysplat
+  - override /model/encoder/backbone: croco
+  - override /loss: [mse, lpips, depth_consis] # ablate: opacity loss
+
+wandb:
+  name: co3d
+  tags: [co3d, 448x448]
+  
+model:
+  encoder:
+    gs_params_head_type: dpt_gs
+    pose_free: true
+    intrinsics_embed_loc: encoder
+    intrinsics_embed_type: token
+    pretrained_weights: ''
+    voxel_size: 0.002
+    pred_pose: true
+    anchor_feat_dim: 128
+    gs_prune: false # ablate: opacity loss
+    pred_head_type: depth
+    freeze_backbone: false
+    distill: true
+    render_conf: false
+    conf_threshold: 0.1
+    freeze_module: patch_embed
+    voxelize: true
+    intermediate_layer_idx: [4, 11, 17, 23]
+    
+dataset:
+  co3d:
+    input_image_shape: [224, 448]
+    view_sampler:
+      num_context_views: 24
+      num_target_views: 1
+      min_distance_between_context_views: 32
+      max_distance_between_context_views: 256
+      max_img_per_gpu: 24 # keep the same as num_context_views
+    avg_pose: false
+    intr_augment: true
+    normalize_by_pts3d: false
+    rescale_to_1cube: false
+
+optimizer:
+  lr: 2e-4
+  warm_up_steps: 1000
+  backbone_lr_multiplier: 0.1
+
+data_loader:
+  train:
+    batch_size: 1 # not used here
+    
+trainer:
+  max_steps: 30000
+  val_check_interval: 500
+  num_nodes: 1
+  accumulate_grad_batches: 1
+  precision: bf16-mixed
+
+checkpointing:
+  load: null
+  every_n_train_steps: 200
+  save_weights_only: false
+  save_top_k: 5
+  
+train:
+  pose_loss_alpha: 1.0
+  pose_loss_delta: 1.0
+  cxt_depth_weight: 0.0
+  weight_pose: 10.0
+  weight_depth: 0.0
+  weight_normal: 0.0
+
+hydra:
+  run:
+    dir: output/exp_${wandb.name}/${now:%Y-%m-%d_%H-%M-%S}
+
+loss:
+  mse:
+    conf: false
+  lpips:
+    conf: false
+  depth_consis:
+    weight: 0.1
+    loss_type: MSE
+
+

config/experiment/dl3dv.yaml

@@ -0,0 +1,92 @@
+# @package _global_
+
+defaults:
+  - /dataset@_group_.dl3dv: dl3dv
+  - override /model/encoder: anysplat
+  - override /model/encoder/backbone: croco
+  - override /loss: [mse, lpips, depth_consis] # ablate: opacity loss
+
+wandb:
+  name: dl3dv
+  tags: [dl3dv, 448x448]
+  
+model:
+  encoder:
+    gs_params_head_type: dpt_gs
+    pose_free: true
+    intrinsics_embed_loc: encoder
+    intrinsics_embed_type: token
+    pretrained_weights: ''
+    voxel_size: 0.002
+    pred_pose: true
+    anchor_feat_dim: 128
+    gs_prune: false # ablate: opacity loss
+    pred_head_type: depth
+    freeze_backbone: false
+    distill: true
+    render_conf: false
+    conf_threshold: 0.1
+    freeze_module: patch_embed
+    voxelize: true
+    intermediate_layer_idx: [4, 11, 17, 23]
+    
+dataset:
+  dl3dv:
+    input_image_shape: [224, 448]
+    view_sampler:
+
+      num_target_views: 2
+      min_distance_between_context_views: 32
+      max_distance_between_context_views: 256
+
+      min_gap_multiplier: 3
+      max_gap_multiplier: 5
+    avg_pose: false
+    intr_augment: true
+    normalize_by_pts3d: false
+    rescale_to_1cube: false
+
+optimizer:
+  lr: 2e-4
+  warm_up_steps: 1000
+  backbone_lr_multiplier: 0.1
+
+data_loader:
+  train:
+    batch_size: 1 # not used here
+    
+trainer:
+  max_steps: 30000
+  val_check_interval: 500
+  num_nodes: 1
+  accumulate_grad_batches: 1
+  precision: bf16-mixed
+
+checkpointing:
+  load: null
+  every_n_train_steps: 200
+  save_weights_only: false
+  save_top_k: 5
+  
+train:
+  pose_loss_alpha: 1.0
+  pose_loss_delta: 1.0
+  cxt_depth_weight: 0.0
+  weight_pose: 10.0
+  weight_depth: 1.0
+  weight_normal: 0.0
+
+hydra:
+  run:
+    dir: output/exp_${wandb.name}/${now:%Y-%m-%d_%H-%M-%S}
+
+loss:
+  mse:
+    conf: false
+  lpips:
+    conf: false
+  depth_consis:
+    weight: 0.1
+    loss_type: MSE
+
+

config/experiment/multi-dataset.yaml

@@ -0,0 +1,121 @@
+# @package _global_
+
+defaults:
+  - /dataset@_group_.dl3dv: dl3dv
+  - /dataset@_group_.co3d: co3d
+  - /dataset@_group_.scannetpp: scannetpp
+  - override /model/encoder: anysplat
+  - override /model/encoder/backbone: croco
+  - override /loss: [mse, lpips, depth_consis] # ablate: opacity loss
+
+wandb:
+  name: multidataset-16gpu
+  tags: [multidataset, 448x448]
+  
+model:
+  encoder:
+    gs_params_head_type: dpt_gs
+    pose_free: true
+    intrinsics_embed_loc: encoder
+    intrinsics_embed_type: token
+    pretrained_weights: ''
+    voxel_size: 0.002
+    pred_pose: true
+    anchor_feat_dim: 128
+    gs_prune: false # ablate: opacity loss
+    pred_head_type: depth
+    freeze_backbone: false
+    distill: true
+    render_conf: false
+    conf_threshold: 0.1
+    freeze_module: patch_embed
+    voxelize: true
+    intermediate_layer_idx: [4, 11, 17, 23]
+    
+dataset:
+  dl3dv:
+    input_image_shape: [224, 448]
+    view_sampler:
+
+      num_target_views: 2
+      min_distance_between_context_views: 32
+      max_distance_between_context_views: 256
+
+      min_gap_multiplier: 3
+      max_gap_multiplier: 5
+      max_img_per_gpu: 24
+    avg_pose: false
+    intr_augment: true
+    normalize_by_pts3d: false
+    rescale_to_1cube: false
+
+  co3d:
+    input_image_shape: [224, 448]
+    view_sampler:
+
+      num_target_views: 1
+      min_distance_between_context_views: 32
+      max_distance_between_context_views: 256
+      max_img_per_gpu: 24
+
+    avg_pose: false
+    intr_augment: true
+    normalize_by_pts3d: false
+    rescale_to_1cube: false
+
+  scannetpp:
+    input_image_shape: [224, 448]
+    view_sampler:
+      num_target_views: 2
+      min_distance_between_context_views: 128
+      max_distance_between_context_views: 512
+      max_img_per_gpu: 24
+    avg_pose: false
+    intr_augment: true
+    normalize_by_pts3d: false
+    rescale_to_1cube: false
+
+optimizer:
+  lr: 2e-4
+  warm_up_steps: 1000
+  backbone_lr_multiplier: 0.1
+
+data_loader:
+  train:
+    batch_size: 1 # not used here
+    
+trainer:
+  max_steps: 30000
+  val_check_interval: 500
+  num_nodes: 2
+  accumulate_grad_batches: 1
+  precision: bf16-mixed
+
+checkpointing:
+  load: null 
+  every_n_train_steps: 200
+  save_weights_only: false
+  save_top_k: 5
+  
+train:
+  pose_loss_alpha: 1.0
+  pose_loss_delta: 1.0
+  cxt_depth_weight: 0.0
+  weight_pose: 10.0
+  weight_depth: 0.0
+  weight_normal: 0.0
+
+hydra:
+  run:
+    dir: output/exp_${wandb.name}/${now:%Y-%m-%d_%H-%M-%S}
+
+loss:
+  mse:
+    conf: false
+  lpips:
+    conf: false
+  depth_consis:
+    weight: 0.1
+    loss_type: MSE
+
+

config/experiment/scannetpp.yaml

@@ -0,0 +1,90 @@
+# @package _global_
+
+defaults:
+  - /dataset@_group_.scannetpp: scannetpp
+  - override /model/encoder: anysplat
+  - override /model/encoder/backbone: croco
+  - override /loss: [mse, lpips, depth_consis] # ablate: opacity loss
+
+wandb:
+  name: vggt-mdataset-new-scannetpp-dynamic_batchsampler
+  tags: [multidataset, 448x448]
+  
+model:
+  encoder:
+    gs_params_head_type: dpt_gs
+    pose_free: true
+    intrinsics_embed_loc: encoder
+    intrinsics_embed_type: token
+    pretrained_weights: ''
+    voxel_size: 0.002
+    pred_pose: true
+    anchor_feat_dim: 128
+    gs_prune: false # ablate: opacity loss
+    pred_head_type: depth
+    freeze_backbone: false
+    distill: true
+    render_conf: false
+    conf_threshold: 0.1
+    freeze_module: patch_embed
+    voxelize: true
+    intermediate_layer_idx: [4, 11, 17, 23]
+    
+dataset:
+  scannetpp:
+    input_image_shape: [224, 448]
+    view_sampler:
+      num_context_views: 24
+      num_target_views: 2
+      min_distance_between_context_views: 128
+      max_distance_between_context_views: 512
+      max_img_per_gpu: 24 # keep the same as num_context_views
+    avg_pose: false
+    intr_augment: true
+    normalize_by_pts3d: false
+    rescale_to_1cube: false
+
+optimizer:
+  lr: 2e-4
+  warm_up_steps: 1000
+  backbone_lr_multiplier: 0.1
+
+data_loader:
+  train:
+    batch_size: 1 # not used here
+    
+trainer:
+  max_steps: 30000
+  val_check_interval: 500
+  num_nodes: 1
+  accumulate_grad_batches: 1
+  precision: bf16-mixed
+
+checkpointing:
+  load: null
+  every_n_train_steps: 200
+  save_weights_only: false
+  save_top_k: 5
+  
+train:
+  pose_loss_alpha: 1.0
+  pose_loss_delta: 1.0
+  cxt_depth_weight: 0.0
+  weight_pose: 10.0
+  weight_depth: 0.0
+  weight_normal: 0.0
+
+hydra:
+  run:
+    dir: output/exp_${wandb.name}/${now:%Y-%m-%d_%H-%M-%S}
+
+loss:
+  mse:
+    conf: false
+  lpips:
+    conf: false
+  depth_consis:
+    weight: 0.1
+    loss_type: MSE
+
+

config/generate_evaluation_index.yaml

@@ -0,0 +1,36 @@
+defaults:
+  - dataset: re10k
+  - optional dataset/view_sampler_dataset_specific_config: ${dataset/view_sampler}_${dataset}
+  - override dataset/view_sampler: all
+
+dataset:
+  overfit_to_scene: null
+
+data_loader:
+  train:
+    num_workers: 0
+    persistent_workers: true
+    batch_size: 1
+    seed: 1234
+  test:
+    num_workers: 8
+    persistent_workers: false
+    batch_size: 1
+    seed: 2345
+  val:
+    num_workers: 0
+    persistent_workers: true
+    batch_size: 1
+    seed: 3456
+
+index_generator:
+  num_target_views: 3
+  min_overlap: 0.6
+  max_overlap: 1.0
+  min_distance: 45
+  max_distance: 135
+  output_path: outputs/evaluation_index_re10k
+  save_previews: false
+  seed: 123
+
+seed: 456

config/loss/chamfer_distance.yaml

@@ -0,0 +1,5 @@
+chamfer_distance:
+  weight: 0.01
+  down_sample_ratio: 0.1
+  sigma_image: null
+

config/loss/depth.yaml

@@ -0,0 +1,4 @@
+depth:
+  weight: 0.01
+  sigma_image: null
+  use_second_derivative: false

config/loss/depth_consis.yaml

@@ -0,0 +1,4 @@
+depth_consis:
+  weight: 1.0
+  sigma_image: null
+  use_second_derivative: false

config/loss/depthgt.yaml

@@ -0,0 +1,3 @@
+depthgt:
+  weight: 0.1
+  type: l1+gradient

config/loss/lod.yaml

@@ -0,0 +1,3 @@
+lod:
+  mse_weight: 1.0
+  lpips_weight: 0.05

config/loss/lpips.yaml

@@ -0,0 +1,3 @@
+lpips:
+  weight: 0.05
+  apply_after_step: 0

config/loss/mse.yaml

@@ -0,0 +1,2 @@
+mse:
+  weight: 1.0

config/loss/normal_consis.yaml

@@ -0,0 +1,5 @@
+normal_consis:
+  normal_weight: 1.0
+  smooth_weight: 1.0
+  sigma_image: null
+  use_second_derivative: false

config/loss/opacity.yaml

@@ -0,0 +1,3 @@
+opacity:
+  weight: 0.1
+  type: exp+mean

config/main.yaml

@@ -0,0 +1,81 @@
+defaults:
+  - model/encoder: anysplat
+  - model/decoder: splatting_cuda
+  - loss: [mse]
+  
+wandb:
+  project: anysplat
+  entity: scene-representation-group
+  name: debug
+  mode: online
+mode: train
+
+#dataset:
+#  overfit_to_scene: null
+
+data_loader:
+  # Avoid having to spin up new processes to print out visualizations.
+  train:
+    num_workers: 16 # 16
+    persistent_workers: true
+    batch_size: 4
+    seed: 1234
+  test:
+    num_workers: 4
+    persistent_workers: false
+    batch_size: 1
+    seed: 2345
+  val:
+    num_workers: 1
+    persistent_workers: true
+    batch_size: 1
+    seed: 3456
+
+optimizer:
+  lr: 1.5e-4
+  warm_up_steps: 2000
+  backbone_lr_multiplier: 0.1
+
+checkpointing:
+  load: null
+  every_n_train_steps: 5000
+  save_top_k: 1
+  save_weights_only: true
+
+train:
+  output_path: ${hydra.run.dir}
+  depth_mode: null
+  extended_visualization: false
+  print_log_every_n_steps: 10
+  distiller: ''
+  distill_max_steps: 1000000
+  random_context_views: false
+  
+test:
+  output_path: outputs/test-nopo
+  align_pose: true
+  pose_align_steps: 100
+  rot_opt_lr:  0.005
+  trans_opt_lr:  0.005
+  compute_scores: true
+  save_image: true
+  save_video: false
+  save_compare: true
+  generate_video: false
+  mode: inference
+  image_folder: examples/bungeenerf
+
+seed: 111123
+
+trainer:
+  max_steps: -1
+  val_check_interval: 250
+  gradient_clip_val: 0.5
+  num_nodes: 1
+  accumulate_grad_batches: 1
+
+hydra:
+  run:
+    dir: output-debug/exp_${wandb.name}/${now:%Y-%m-%d_%H-%M-%S}
+  # run:
+    # dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}_rank${oc.env:LOCAL_RANK,0}

config/model/decoder/splatting_cuda.yaml

@@ -0,0 +1,3 @@
+name: splatting_cuda
+background_color: [1.0, 1.0, 1.0]
+make_scale_invariant: false

config/model/encoder/anysplat.yaml

@@ -0,0 +1,62 @@
+defaults:
+  - backbone: croco
+
+name: anysplat
+
+opacity_mapping:
+  initial: 0.0
+  final: 0.0
+  warm_up: 1
+
+num_monocular_samples: 32
+num_surfaces: 1
+predict_opacity: false
+
+gaussians_per_pixel: 1
+
+gaussian_adapter:
+  gaussian_scale_min: 0.5
+  gaussian_scale_max: 15.0
+  sh_degree: 4
+
+d_feature: 32
+
+visualizer:
+  num_samples: 8
+  min_resolution: 256
+  export_ply: false
+
+apply_bounds_shim: true
+
+gs_params_head_type: dpt_gs
+pose_free: true
+pretrained_weights: ""
+scale_align: false
+
+voxel_size: 0.001
+n_offsets: 2
+anchor_feat_dim: 83 # 32
+add_view: false
+color_attr: 3D # 3D or RGB
+mlp_type: unified
+scaffold: true
+
+# unet3d:
+#   # lifter_params:
+#   #   img_in_dim: 32
+#   #   voxel_out_dim: 32
+#   img_feature_source: dino
+#   in_channels: 83 # 32  keep same as anchor_feat_dim
+#   num_blocks: 2 # 512 -> 128 
+#   f_maps: 83 # 32
+#   # f_maps_2d: 32
+#   neck_dense_type: "UNCHANGED"
+#   neck_bound: 4
+#   use_attention: true
+#   gs_enhanced: "original"
+#   gsplat_upsample: 4
+#   occ_upsample: 1
+#   max_scaling: 10
+#   max_return: 2
+#   feature_pooling_2d: "max"
+#   gs_free_space: "free-1"

config/model/encoder/backbone/croco.yaml

@@ -0,0 +1,9 @@
+name: croco
+
+model: ViTLarge_BaseDecoder
+patch_embed_cls: PatchEmbedDust3R
+asymmetry_decoder: true
+
+intrinsics_embed_loc: 'encoder'
+intrinsics_embed_degree: 4
+intrinsics_embed_type: 'token'

demo_gradio.py

@@ -0,0 +1,459 @@
+#!/usr/bin/env python3
+import functools
+import gc
+import os
+import shutil
+import sys
+import tempfile
+import time
+from datetime import datetime
+from pathlib import Path
+
+import cv2
+import gradio as gr
+import torch
+from huggingface_hub import hf_hub_download
+from PIL import Image
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.misc.image_io import save_interpolated_video
+from src.model.model.anysplat import AnySplat
+from src.model.ply_export import export_ply
+from src.utils.image import process_image
+
+
+# 1) Core model inference
+def get_reconstructed_scene(outdir, model, device):
+    # Load Images
+    image_files = sorted(
+        [
+            os.path.join(outdir, "images", f)
+            for f in os.listdir(os.path.join(outdir, "images"))
+        ]
+    )
+    images = [process_image(img_path) for img_path in image_files]
+    images = torch.stack(images, dim=0).unsqueeze(0).to(device)  # [1, K, 3, 448, 448]
+    b, v, c, h, w = images.shape
+
+    assert c == 3, "Images must have 3 channels"
+
+    # Run Inference
+    gaussians, pred_context_pose = model.inference((images + 1) * 0.5)
+
+    # Save the results
+    pred_all_extrinsic = pred_context_pose["extrinsic"]
+    pred_all_intrinsic = pred_context_pose["intrinsic"]
+    video, depth_colored = save_interpolated_video(
+        pred_all_extrinsic,
+        pred_all_intrinsic,
+        b,
+        h,
+        w,
+        gaussians,
+        outdir,
+        model.decoder,
+    )
+
+    plyfile = os.path.join(outdir, "gaussians.ply")
+    export_ply(
+        gaussians.means[0],
+        gaussians.scales[0],
+        gaussians.rotations[0],
+        gaussians.harmonics[0],
+        gaussians.opacities[0],
+        Path(plyfile),
+        save_sh_dc_only=True,
+    )
+
+    # Clean up
+    torch.cuda.empty_cache()
+    return plyfile, video, depth_colored
+
+
+# 2) Handle uploaded video/images --> produce target_dir + images
+def handle_uploads(input_video, input_images):
+    """
+    Create a new 'target_dir' + 'images' subfolder, and place user-uploaded
+    images or extracted frames from video into it. Return (target_dir, image_paths).
+    """
+    start_time = time.time()
+    gc.collect()
+    torch.cuda.empty_cache()
+
+    # Create a unique folder name
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+    target_dir = f"input_images_{timestamp}"
+    target_dir_images = os.path.join(target_dir, "images")
+
+    # Clean up if somehow that folder already exists
+    if os.path.exists(target_dir):
+        shutil.rmtree(target_dir)
+    os.makedirs(target_dir)
+    os.makedirs(target_dir_images)
+
+    image_paths = []
+
+    # --- Handle images ---
+    if input_images is not None:
+        for file_data in input_images:
+            if isinstance(file_data, dict) and "name" in file_data:
+                file_path = file_data["name"]
+            else:
+                file_path = file_data
+            dst_path = os.path.join(target_dir_images, os.path.basename(file_path))
+            shutil.copy(file_path, dst_path)
+            image_paths.append(dst_path)
+
+    # --- Handle video ---
+    if input_video is not None:
+        if isinstance(input_video, dict) and "name" in input_video:
+            video_path = input_video["name"]
+        else:
+            video_path = input_video
+
+        vs = cv2.VideoCapture(video_path)
+        fps = vs.get(cv2.CAP_PROP_FPS)
+        frame_interval = int(fps * 1)  # 1 frame/sec
+
+        count = 0
+        video_frame_num = 0
+        while True:
+            gotit, frame = vs.read()
+            if not gotit:
+                break
+            count += 1
+            if count % frame_interval == 0:
+                image_path = os.path.join(
+                    target_dir_images, f"{video_frame_num:06}.png"
+                )
+                cv2.imwrite(image_path, frame)
+                image_paths.append(image_path)
+                video_frame_num += 1
+
+    # Sort final images for gallery
+    image_paths = sorted(image_paths)
+
+    end_time = time.time()
+    print(
+        f"Files copied to {target_dir_images}; took {end_time - start_time:.3f} seconds"
+    )
+    return target_dir, image_paths
+
+
+# 3) Update gallery on upload
+def update_gallery_on_upload(input_video, input_images):
+    """
+    Whenever user uploads or changes files, immediately handle them
+    and show in the gallery. Return (target_dir, image_paths).
+    If nothing is uploaded, returns "None" and empty list.
+    """
+    if not input_video and not input_images:
+        return None, None, None
+    target_dir, image_paths = handle_uploads(input_video, input_images)
+    return None, target_dir, image_paths
+
+
+# 4) Reconstruction: uses the target_dir plus any viz parameters
+def gradio_demo(
+    target_dir,
+):
+    """
+    Perform reconstruction using the already-created target_dir/images.
+    """
+    if not os.path.isdir(target_dir) or target_dir == "None":
+        return None, None, None
+
+    start_time = time.time()
+    gc.collect()
+    torch.cuda.empty_cache()
+    
+    # Prepare frame_filter dropdown
+    target_dir_images = os.path.join(target_dir, "images")
+    all_files = (
+        sorted(os.listdir(target_dir_images))
+        if os.path.isdir(target_dir_images)
+        else []
+    )
+    all_files = [f"{i}: {filename}" for i, filename in enumerate(all_files)]
+
+    print("Running run_model...")
+    with torch.no_grad():
+        plyfile, video, depth_colored = get_reconstructed_scene(
+            target_dir, model, device
+        )
+
+    end_time = time.time()
+    print(f"Total time: {end_time - start_time:.2f} seconds (including IO)")
+
+    return plyfile, video, depth_colored
+
+
+def clear_fields():
+    """
+    Clears the 3D viewer, the stored target_dir, and empties the gallery.
+    """
+    return None, None, None
+
+
+if __name__ == "__main__":
+    server_name = "127.0.0.1"
+    server_port = None
+    share = True
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    
+    # Load model
+    model = AnySplat.from_pretrained(
+        "lhjiang/anysplat"
+    )
+    model = model.to(device)
+    model.eval()
+    for param in model.parameters():
+        param.requires_grad = False
+
+    theme = gr.themes.Ocean()
+    theme.set(
+        checkbox_label_background_fill_selected="*button_primary_background_fill",
+        checkbox_label_text_color_selected="*button_primary_text_color",
+    )
+    css = """
+        .custom-log * {
+            font-style: italic;
+            font-size: 22px !important;
+            background-image: linear-gradient(120deg, #0ea5e9 0%, #6ee7b7 60%, #34d399 100%);
+            -webkit-background-clip: text;
+            background-clip: text;
+            font-weight: bold !important;
+            color: transparent !important;
+            text-align: center !important;
+        }
+        
+        .example-log * {
+            font-style: italic;
+            font-size: 16px !important;
+            background-image: linear-gradient(120deg, #0ea5e9 0%, #6ee7b7 60%, #34d399 100%);
+            -webkit-background-clip: text;
+            background-clip: text;
+            color: transparent !important;
+        }
+        
+        #my_radio .wrap {
+            display: flex;
+            flex-wrap: nowrap;
+            justify-content: center;
+            align-items: center;
+        }
+
+        #my_radio .wrap label {
+            display: flex;
+            width: 50%;
+            justify-content: center;
+            align-items: center;
+            margin: 0;
+            padding: 10px 0;
+            box-sizing: border-box;
+        }
+        """
+    with gr.Blocks(css=css, title="AnySplat Demo", theme=theme) as demo:
+        gr.Markdown(
+            """
+            <h1 style='text-align: center;'>AnySplat: Feed-forward 3D Gaussian Splatting from Unconstrained Views</h1>
+            """
+        )
+
+        with gr.Row():
+            gr.Markdown(
+                """
+                        <p align="center">
+                        <a title="Website" href="https://city-super.github.io/anysplat/" target="_blank" rel="noopener noreferrer" style="display: inline-block;">
+                            <img src="https://www.obukhov.ai/img/badges/badge-website.svg">
+                        </a>
+                        <a title="arXiv" href="https://arxiv.org/pdf/2505.23716" target="_blank" rel="noopener noreferrer" style="display: inline-block;">
+                            <img src="https://www.obukhov.ai/img/badges/badge-pdf.svg">
+                        </a>
+                        <a title="Github" href="https://github.com/OpenRobotLab/AnySplat" target="_blank" rel="noopener noreferrer" style="display: inline-block;">
+                            <img src="https://img.shields.io/badge/Github-Page-black" alt="badge-github-stars">
+                        </a>
+                
+                        </p>
+                        """
+            )
+        with gr.Row():
+            gr.Markdown(
+                """
+            ### Getting Started:
+
+            1. Upload Your Data: Use the "Upload Video" or "Upload Images" buttons on the left to provide your input. Videos will be automatically split into individual frames (one frame per second).
+
+            2. Preview: Your uploaded images will appear in the gallery on the left.
+
+            3. Reconstruct: Click the "Reconstruct" button to start the 3D reconstruction process.
+
+            4. Visualize: The reconstructed 3D Gaussian Splat will appear in the viewer on the right, along with the rendered RGB and depth videos. The trajectory of the rendered video is obtained by interpolating the estimated input image poses.
+            
+            <strong style="color: #0ea5e9;">Please note:</strong> <span style="color: #0ea5e9; font-weight: bold;">The generated splats are large in size, so they may not load successfully in the Hugging Face demo. You can download the .ply file and render it using other viewers, such as [SuperSplat](https://playcanvas.com/supersplat/editor).</span>
+            """
+            )
+
+        target_dir_output = gr.Textbox(label="Target Dir", visible=False, value="None")
+        is_example = gr.Textbox(label="is_example", visible=False, value="None")
+        num_images = gr.Textbox(label="num_images", visible=False, value="None")
+        dataset_name = gr.Textbox(label="dataset_name", visible=False, value="None")
+        scene_name = gr.Textbox(label="scene_name", visible=False, value="None")
+        image_type = gr.Textbox(label="image_type", visible=False, value="None")
+
+        with gr.Row():
+            with gr.Column(scale=2):
+                with gr.Tabs():
+                    with gr.Tab("Input Data"):
+                        input_video = gr.Video(label="Upload Video", interactive=True)
+                        input_images = gr.File(
+                            file_count="multiple",
+                            label="Upload Images",
+                            interactive=True,
+                        )
+
+                        image_gallery = gr.Gallery(
+                            label="Preview",
+                            columns=4,
+                            height="300px",
+                            show_download_button=True,
+                            object_fit="contain",
+                            preview=True,
+                        )
+
+            with gr.Column(scale=4):
+                with gr.Tabs():
+                    with gr.Tab("AnySplat Output"):
+                        with gr.Column():
+                            reconstruction_output = gr.Model3D(
+                                label="3D Reconstructed Gaussian Splat",
+                                height=540,
+                                zoom_speed=0.5,
+                                pan_speed=0.5,
+                                camera_position=[20, 20, 20],
+                            )
+
+                        with gr.Row():
+                            with gr.Row():
+                                rgb_video = gr.Video(
+                                    label="RGB Video", interactive=False, autoplay=True
+                                )
+                                depth_video = gr.Video(
+                                    label="Depth Video",
+                                    interactive=False,
+                                    autoplay=True,
+                                )
+
+                        with gr.Row():
+                            submit_btn = gr.Button(
+                                "Reconstruct", scale=1, variant="primary"
+                            )
+                            clear_btn = gr.ClearButton(
+                                [
+                                    input_video,
+                                    input_images,
+                                    reconstruction_output,
+                                    target_dir_output,
+                                    image_gallery,
+                                    rgb_video,
+                                    depth_video,
+                                ],
+                                scale=1,
+                            )
+
+        # ---------------------- Examples section ----------------------
+
+        examples = [
+            [None, "examples/video/re10k_1eca36ec55b88fe4.mp4", "re10k", "1eca36ec55b88fe4", "2", "Real", "True",],
+            [None, "examples/video/bungeenerf_colosseum.mp4", "bungeenerf", "colosseum", "8", "Synthetic", "True",],
+            [None, "examples/video/fox.mp4", "InstantNGP", "fox", "14", "Real", "True",],
+            [None, "examples/video/matrixcity_street.mp4", "matrixcity", "street", "32", "Synthetic", "True",],
+            [None, "examples/video/vrnerf_apartment.mp4", "vrnerf", "apartment", "32", "Real", "True",],
+            [None, "examples/video/vrnerf_kitchen.mp4", "vrnerf", "kitchen", "17", "Real", "True",],
+            [None, "examples/video/vrnerf_riverview.mp4", "vrnerf", "riverview", "12", "Real", "True",],
+            [None, "examples/video/vrnerf_workshop.mp4", "vrnerf", "workshop", "32", "Real", "True",],
+            [None, "examples/video/fillerbuster_ramen.mp4", "fillerbuster", "ramen", "32", "Real", "True",],
+            [None, "examples/video/meganerf_rubble.mp4", "meganerf", "rubble", "10", "Real", "True",],
+            [None, "examples/video/llff_horns.mp4", "llff", "horns", "12", "Real", "True",],
+            [None, "examples/video/llff_fortress.mp4", "llff", "fortress", "7", "Real", "True",],
+            [None, "examples/video/dtu_scan_106.mp4", "dtu", "scan_106", "20", "Real", "True",],
+            [None, "examples/video/horizongs_hillside_summer.mp4", "horizongs", "hillside_summer", "55", "Synthetic", "True",],
+            [None, "examples/video/kitti360.mp4", "kitti360", "kitti360", "64", "Real", "True",],
+        ]
+
+        def example_pipeline(
+            input_images,
+            input_video,
+            dataset_name,
+            scene_name,
+            num_images_str,
+            image_type,
+            is_example,
+        ):
+            """
+            1) Copy example images to new target_dir
+            2) Reconstruct
+            3) Return model3D + logs + new_dir + updated dropdown + gallery
+            We do NOT return is_example. It's just an input.
+            """
+            target_dir, image_paths = handle_uploads(input_video, input_images)
+            plyfile, video, depth_colored = gradio_demo(target_dir)
+            return plyfile, video, depth_colored, target_dir, image_paths
+
+        gr.Markdown("Click any row to load an example.", elem_classes=["example-log"])
+
+        gr.Examples(
+            examples=examples,
+            inputs=[
+                input_images,
+                input_video,
+                dataset_name,
+                scene_name,
+                num_images,
+                image_type,
+                is_example,
+            ],
+            outputs=[
+                reconstruction_output,
+                rgb_video,
+                depth_video,
+                target_dir_output,
+                image_gallery,
+            ],
+            fn=example_pipeline,
+            cache_examples=False,
+            examples_per_page=50,
+        )
+
+        gr.Markdown("<p style='text-align: center; font-style: italic; color: #666;'>We thank VGGT for their excellent gradio implementation!</p>")
+
+        submit_btn.click(
+            fn=clear_fields,
+            inputs=[],
+            outputs=[reconstruction_output, rgb_video, depth_video],
+        ).then(
+            fn=gradio_demo,
+            inputs=[
+                target_dir_output,
+            ],
+            outputs=[reconstruction_output, rgb_video, depth_video],
+        ).then(
+            fn=lambda: "False", inputs=[], outputs=[is_example]
+        )
+
+        input_video.change(
+            fn=update_gallery_on_upload,
+            inputs=[input_video, input_images],
+            outputs=[reconstruction_output, target_dir_output, image_gallery],
+        )
+        input_images.change(
+            fn=update_gallery_on_upload,
+            inputs=[input_video, input_images],
+            outputs=[reconstruction_output, target_dir_output, image_gallery],
+        )
+
+        # demo.launch(share=share, server_name=server_name, server_port=server_port)
+        demo.queue(max_size=20).launch(show_error=True, share=True)
+
+        # We thank VGGT for their excellent gradio implementation

examples/video/bungeenerf_colosseum.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:416b6af945547b5d19476823672de552944c7b5a147d29e9e8243e91a16aee3e
+size 329073

examples/video/dtu_scan_106.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16d7a06325cd368b134908e600a6c0741c7d0d188f1db690532b8ac85d65fef5
+size 352188

examples/video/fillerbuster_hand_hand.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b4ca982672bc92342b3e722c171d9d2e4d67a5a8116cd9f346956fbe01e253f
+size 319404

examples/video/fillerbuster_ramen.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d60346a64a0a0d6805131d0d57edeeb0dae24f24c3f10560e95df65531221229
+size 660736

examples/video/fox.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3fa2ccff78e5d8085bb58f3def2d482e8df285ced5ef1b56abfe3766f0d90e0
+size 2361921

examples/video/horizongs_hillside_summer.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5dff78d9c00b3776bfca3a370061698bddead2ae940fe5a42d082ccf2ca80d1
+size 1606537

examples/video/kitti360.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c6b13929b2c2aae8b95921d8626f5be06f6afffe05ea4e47940ffeb9906f9fc
+size 1843629

examples/video/llff_fortress.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90ea046a0ec78651975529ebe6b9c72b60c19561fe61b15b15b9df0e44d9fe9a
+size 196243

examples/video/llff_horns.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bc4c443c2a3f889f0c1283e98bd6a7026c36858fb37808bb2e8699ad1a2c1d8
+size 372570

examples/video/matrixcity_street.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa415f27177398b4e06f580beb3778701ca55784afade2fd6a058212213febc8
+size 3163684

examples/video/meganerf_rubble.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3410c759eb73ca2403ab8fe35d5ebabdbc25e3a0e67d8670a89fe17686246ed0
+size 450116

examples/video/vrnerf_apartment.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fdd5f165a4293cd95e3dd88d84b1f370decdd86308aa67a9d3832e01f4d6906
+size 2076392

examples/video/vrnerf_kitchen.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3db5d766ec86a7abdfe1f033b252337e6d934ea15035fafb4d0fc0c0e9e9740a
+size 775715

examples/video/vrnerf_riverview.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b8187936cc49910ef330a37b1bbdab0076096d6c01f33b097c11937184de168
+size 768290

examples/video/vrnerf_workshop.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0f1334acc74bd70086a9be94d0c36838ebd7499af27f942c315e1ba282e285b
+size 1718918