change gsplat to diff

.gitignore

@@ -20,4 +20,6 @@ build/
 *.mp4
 .vs
 /exp/
-/dev/
+/dev/
+gradio_cached_examples/
+gradio_cache_folder/

app.py

@@ -12,6 +12,7 @@ import gradio as gr
 import uuid
 import spaces
 
+subprocess.run(shlex.split("pip install wheel/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl"))
 subprocess.run(shlex.split("pip install wheel/simple_knn-0.0.0-cp310-cp310-linux_x86_64.whl"))
 subprocess.run(shlex.split("pip install wheel/curope-0.0.0-cp310-cp310-linux_x86_64.whl"))
 

command

@@ -1,33 +0,0 @@
-conda activate feat2gs
-cd Feat2GS/
-
-bash scripts/run_feat2gs_eval_parallel.sh
-bash scripts/run_feat2gs_eval.sh
-bash scripts/run_instantsplat_eval_parallel.sh
-bash scripts/run_feat2gs_eval_dtu_parallel.sh
-
-python video/generate_video.py
-
-bash scripts/run_all_trajectories.sh
-bash scripts/run_video_render.sh
-bash scripts/run_video_render_instantsplat.sh
-bash scripts/run_video_render_dtu.sh
-
-tensorboard --logdir=/home/chenyue/output/Feat2gs/output/eval/ --port=7001
-
-cd /home/chenyue/output/Feat2gs/output/eval/Tanks/Train/6_views/feat2gs-G/dust3r/
-tensorboard --logdir_spec \
-radio:radio,\
-dust3r:dust3r,\
-dino_b16:dino_b16,\
-mast3r:mast3r,\
-dift:dift,\
-dinov2:dinov2_b14,\
-clip:clip_b16,\
-mae:mae_b16,\
-midas:midas_l16,\
-sam:sam_base,\
-iuvrgb:iuvrgb \
---port 7002
-
-CUDA_VISIBLE_DEVICES=7 gradio demo.py

gaussian_renderer/__init__.py

@@ -18,142 +18,140 @@ from utils.graphics_utils import depth_to_normal
 
 ### if use [diff-gaussian-rasterization](https://github.com/graphdeco-inria/diff-gaussian-rasterization)
 
-# from diff_gaussian_rasterization import (
-#     GaussianRasterizationSettings,
-#     GaussianRasterizer,
-# )
-# from utils.sh_utils import eval_sh
-
-# def render(
-#     viewpoint_camera,
-#     pc: GaussianModel,
-#     pipe,
-#     bg_color: torch.Tensor,
-#     scaling_modifier=1.0,
-#     override_color=None,
-#     camera_pose=None,
-# ):
-#     """
-#     Render the scene.
-
-#     Background tensor (bg_color) must be on GPU!
-#     """
-
-#     # Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means
-#     screenspace_points = (
-#         torch.zeros_like(
-#             pc.get_xyz, dtype=pc.get_xyz.dtype, requires_grad=True, device="cuda"
-#         )
-#         + 0
-#     )
-#     try:
-#         screenspace_points.retain_grad()
-#     except:
-#         pass
-
-#     # Set up rasterization configuration
-#     tanfovx = math.tan(viewpoint_camera.FoVx * 0.5)
-#     tanfovy = math.tan(viewpoint_camera.FoVy * 0.5)
-
-#     # Set camera pose as identity. Then, we will transform the Gaussians around camera_pose
-#     w2c = torch.eye(4).cuda()
-#     projmatrix = (
-#         w2c.unsqueeze(0).bmm(viewpoint_camera.projection_matrix.unsqueeze(0))
-#     ).squeeze(0)
-#     camera_pos = w2c.inverse()[3, :3]
-#     raster_settings = GaussianRasterizationSettings(
-#         image_height=int(viewpoint_camera.image_height),
-#         image_width=int(viewpoint_camera.image_width),
-#         tanfovx=tanfovx,
-#         tanfovy=tanfovy,
-#         bg=bg_color,
-#         scale_modifier=scaling_modifier,
-#         # viewmatrix=viewpoint_camera.world_view_transform,
-#         # projmatrix=viewpoint_camera.full_proj_transform,
-#         viewmatrix=w2c,
-#         projmatrix=projmatrix,
-#         sh_degree=pc.active_sh_degree,
-#         # campos=viewpoint_camera.camera_center,
-#         campos=camera_pos,
-#         prefiltered=False,
-#         debug=pipe.debug,
-#     )
-
-#     rasterizer = GaussianRasterizer(raster_settings=raster_settings)
-
-#     # means3D = pc.get_xyz
-#     rel_w2c = get_camera_from_tensor(camera_pose)
-#     # Transform mean and rot of Gaussians to camera frame
-#     gaussians_xyz = pc._xyz.clone()
-#     gaussians_rot = pc._rotation.clone()
-
-#     xyz_ones = torch.ones(gaussians_xyz.shape[0], 1).cuda().float()
-#     xyz_homo = torch.cat((gaussians_xyz, xyz_ones), dim=1)
-#     gaussians_xyz_trans = (rel_w2c @ xyz_homo.T).T[:, :3]
-#     gaussians_rot_trans = quadmultiply(camera_pose[:4], gaussians_rot)
-#     means3D = gaussians_xyz_trans
-#     means2D = screenspace_points
-#     opacity = pc.get_opacity
-
-#     # If precomputed 3d covariance is provided, use it. If not, then it will be computed from
-#     # scaling / rotation by the rasterizer.
-#     scales = None
-#     rotations = None
-#     cov3D_precomp = None
-#     if pipe.compute_cov3D_python:
-#         cov3D_precomp = pc.get_covariance(scaling_modifier)
-#     else:
-#         scales = pc.get_scaling
-#         rotations = gaussians_rot_trans  # pc.get_rotation
-
-#     # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors
-#     # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer.
-#     shs = None
-#     colors_precomp = None
-#     if override_color is None:
-#         if pipe.convert_SHs_python:
-#             shs_view = pc.get_features.transpose(1, 2).view(
-#                 -1, 3, (pc.max_sh_degree + 1) ** 2
-#             )
-#             dir_pp = pc.get_xyz - viewpoint_camera.camera_center.repeat(
-#                 pc.get_features.shape[0], 1
-#             )
-#             dir_pp_normalized = dir_pp / dir_pp.norm(dim=1, keepdim=True)
-#             sh2rgb = eval_sh(pc.active_sh_degree, shs_view, dir_pp_normalized)
-#             colors_precomp = torch.clamp_min(sh2rgb + 0.5, 0.0)
-#         else:
-#             shs = pc.get_features
-#     else:
-#         colors_precomp = override_color
-
-#     # Rasterize visible Gaussians to image, obtain their radii (on screen).
-#     rendered_image, radii = rasterizer(
-#         means3D=means3D,
-#         means2D=means2D,
-#         shs=shs,
-#         colors_precomp=colors_precomp,
-#         opacities=opacity,
-#         scales=scales,
-#         rotations=rotations,
-#         cov3D_precomp=cov3D_precomp,
-#     )
-
-#     # Those Gaussians that were frustum culled or had a radius of 0 were not visible.
-#     # They will be excluded from value updates used in the splitting criteria.
-#     return {
-#         "render": rendered_image,
-#         "viewspace_points": screenspace_points,
-#         "visibility_filter": radii > 0,
-#         "radii": radii,
-#     }
+from diff_gaussian_rasterization import (
+    GaussianRasterizationSettings,
+    GaussianRasterizer,
+)
+from utils.sh_utils import eval_sh
+
+def render(
+    viewpoint_camera,
+    pc: GaussianModel,
+    pipe,
+    bg_color: torch.Tensor,
+    scaling_modifier=1.0,
+    override_color=None,
+    camera_pose=None,
+):
+    """
+    Render the scene.
+
+    Background tensor (bg_color) must be on GPU!
+    """
+
+    # Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means
+    screenspace_points = (
+        torch.zeros_like(
+            pc.get_xyz, dtype=pc.get_xyz.dtype, requires_grad=True, device="cuda"
+        )
+        + 0
+    )
+    try:
+        screenspace_points.retain_grad()
+    except:
+        pass
+
+    # Set up rasterization configuration
+    tanfovx = math.tan(viewpoint_camera.FoVx * 0.5)
+    tanfovy = math.tan(viewpoint_camera.FoVy * 0.5)
+
+    # Set camera pose as identity. Then, we will transform the Gaussians around camera_pose
+    w2c = torch.eye(4).cuda()
+    projmatrix = (
+        w2c.unsqueeze(0).bmm(viewpoint_camera.projection_matrix.unsqueeze(0))
+    ).squeeze(0)
+    camera_pos = w2c.inverse()[3, :3]
+    raster_settings = GaussianRasterizationSettings(
+        image_height=int(viewpoint_camera.image_height),
+        image_width=int(viewpoint_camera.image_width),
+        tanfovx=tanfovx,
+        tanfovy=tanfovy,
+        bg=bg_color,
+        scale_modifier=scaling_modifier,
+        # viewmatrix=viewpoint_camera.world_view_transform,
+        # projmatrix=viewpoint_camera.full_proj_transform,
+        viewmatrix=w2c,
+        projmatrix=projmatrix,
+        sh_degree=pc.active_sh_degree,
+        # campos=viewpoint_camera.camera_center,
+        campos=camera_pos,
+        prefiltered=False,
+        debug=pipe.debug,
+    )
+
+    rasterizer = GaussianRasterizer(raster_settings=raster_settings)
+
+    # means3D = pc.get_xyz
+    rel_w2c = get_camera_from_tensor(camera_pose)
+    # Transform mean and rot of Gaussians to camera frame
+    gaussians_xyz = pc._xyz.clone()
+    gaussians_rot = pc._rotation.clone()
+
+    xyz_ones = torch.ones(gaussians_xyz.shape[0], 1).cuda().float()
+    xyz_homo = torch.cat((gaussians_xyz, xyz_ones), dim=1)
+    gaussians_xyz_trans = (rel_w2c @ xyz_homo.T).T[:, :3]
+    gaussians_rot_trans = quadmultiply(camera_pose[:4], gaussians_rot)
+    means3D = gaussians_xyz_trans
+    means2D = screenspace_points
+    opacity = pc.get_opacity
+
+    # If precomputed 3d covariance is provided, use it. If not, then it will be computed from
+    # scaling / rotation by the rasterizer.
+    scales = None
+    rotations = None
+    cov3D_precomp = None
+    if pipe.compute_cov3D_python:
+        cov3D_precomp = pc.get_covariance(scaling_modifier)
+    else:
+        scales = pc.get_scaling
+        rotations = gaussians_rot_trans  # pc.get_rotation
+
+    # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors
+    # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer.
+    shs = None
+    colors_precomp = None
+    if override_color is None:
+        if pipe.convert_SHs_python:
+            shs_view = pc.get_features.transpose(1, 2).view(
+                -1, 3, (pc.max_sh_degree + 1) ** 2
+            )
+            dir_pp = pc.get_xyz - viewpoint_camera.camera_center.repeat(
+                pc.get_features.shape[0], 1
+            )
+            dir_pp_normalized = dir_pp / dir_pp.norm(dim=1, keepdim=True)
+            sh2rgb = eval_sh(pc.active_sh_degree, shs_view, dir_pp_normalized)
+            colors_precomp = torch.clamp_min(sh2rgb + 0.5, 0.0)
+        else:
+            shs = pc.get_features
+    else:
+        colors_precomp = override_color
+
+    # Rasterize visible Gaussians to image, obtain their radii (on screen).
+    rendered_image, radii = rasterizer(
+        means3D=means3D,
+        means2D=means2D,
+        shs=shs,
+        colors_precomp=colors_precomp,
+        opacities=opacity,
+        scales=scales,
+        rotations=rotations,
+        cov3D_precomp=cov3D_precomp,
+    )
+
+    # Those Gaussians that were frustum culled or had a radius of 0 were not visible.
+    # They will be excluded from value updates used in the splitting criteria.
+    return {
+        "render": rendered_image,
+        "viewspace_points": screenspace_points,
+        "visibility_filter": radii > 0,
+        "radii": radii,
+    }
 
 
 ### if use [gsplat](https://github.com/nerfstudio-project/gsplat)
 
 from gsplat import rasterization
-import spaces
 
-@spaces.GPU(duration=150)
 def render_gsplat(
         viewpoint_camera, 
         pc : GaussianModel, 

run_video.py

@@ -18,7 +18,7 @@ import torch
 from scene import Scene
 import os
 from tqdm import tqdm
-from gaussian_renderer import render_gsplat
+from gaussian_renderer import render, render_gsplat
 from argparse import ArgumentParser
 from arguments import ModelParams, PipelineParams, get_combined_args
 from gaussian_renderer import GaussianModel
@@ -201,7 +201,7 @@ def render_sets(dataset: ModelParams, iteration: int, pipeline: PipelineParams,
         if args.resize:
             view = resize_render(view)
             
-        rendering = render_gsplat(
+        rendering = render(
             view, gaussians, pipeline, background, camera_pose=camera_pose
         )["render"]
         

train_feat2gs.py

@@ -14,7 +14,7 @@ import numpy as np
 import torch
 from random import randint
 from utils.loss_utils import l1_loss, ssim
-from gaussian_renderer import render_gsplat
+from gaussian_renderer import render, render_gsplat
 import sys
 from scene import Scene, Feat2GaussianModel
 from argparse import ArgumentParser
@@ -133,7 +133,7 @@ def training(dataset, opt, pipe, testing_iterations, saving_iterations, checkpoi
             Ll1 = torch.tensor(0)
 
         if iteration > warm_iter:
-            render_pkg = render_gsplat(viewpoint_cam, gaussians, pipe, bg, camera_pose=pose)
+            render_pkg = render(viewpoint_cam, gaussians, pipe, bg, camera_pose=pose)
             image, viewspace_point_tensor, visibility_filter, radii = render_pkg["render"], render_pkg["viewspace_points"], render_pkg["visibility_filter"], render_pkg["radii"]
 
             # Loss