gen3c / gui /src /camera_path.cu
elungky's picture
Initial commit for new Space - pre-built Docker image
28451f7
/*
* SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/** @file camera_path.cpp
* @author Thomas Müller & Alex Evans, NVIDIA
*/
#include <neural-graphics-primitives/camera_path.h>
#include <neural-graphics-primitives/common.h>
#include <neural-graphics-primitives/json_binding.h>
#ifdef NGP_GUI
# include <imgui/imgui.h>
# include <imguizmo/ImGuizmo.h>
#endif
#include <fstream>
#include <json/json.hpp>
using namespace nlohmann;
namespace ngp {
CameraKeyframe lerp(const CameraKeyframe& p0, const CameraKeyframe& p1, float t, float t0, float t1) {
t = (t - t0) / (t1 - t0);
quat R1 = p1.R;
// take the short path
if (dot(R1, p0.R) < 0.0f) {
R1 = -R1;
}
return {
normalize(slerp(p0.R, R1, t)),
p0.T + (p1.T - p0.T) * t,
p0.fov + (p1.fov - p0.fov) * t,
p0.timestamp + (p1.timestamp - p0.timestamp) * t,
};
}
CameraKeyframe normalize(const CameraKeyframe& p0) {
CameraKeyframe result = p0;
result.R = normalize(result.R);
return result;
}
CameraKeyframe spline_cm(float t, const CameraKeyframe& p0, const CameraKeyframe& p1, const CameraKeyframe& p2, const CameraKeyframe& p3) {
CameraKeyframe q0 = lerp(p0, p1, t, -1.f, 0.f);
CameraKeyframe q1 = lerp(p1, p2, t, 0.f, 1.f);
CameraKeyframe q2 = lerp(p2, p3, t, 1.f, 2.f);
CameraKeyframe r0 = lerp(q0, q1, t, -1.f, 1.f);
CameraKeyframe r1 = lerp(q1, q2, t, 0.f, 2.f);
return lerp(r0, r1, t, 0.f, 1.f);
}
CameraKeyframe spline_cubic(float t, const CameraKeyframe& p0, const CameraKeyframe& p1, const CameraKeyframe& p2, const CameraKeyframe& p3) {
float tt = t * t;
float ttt = t * t * t;
float a = (1 - t) * (1 - t) * (1 - t) * (1.f / 6.f);
float b = (3.f * ttt - 6.f * tt + 4.f) * (1.f / 6.f);
float c = (-3.f * ttt + 3.f * tt + 3.f * t + 1.f) * (1.f / 6.f);
float d = ttt * (1.f / 6.f);
return normalize(p0 * a + p1 * b + p2 * c + p3 * d);
}
CameraKeyframe spline_quadratic(float t, const CameraKeyframe& p0, const CameraKeyframe& p1, const CameraKeyframe& p2) {
float tt = t * t;
float a = (1 - t) * (1 - t) * 0.5f;
float b = (-2.f * tt + 2.f * t + 1.f) * 0.5f;
float c = tt * 0.5f;
return normalize(p0 * a + p1 * b + p2 * c);
}
CameraKeyframe spline_linear(float t, const CameraKeyframe& p0, const CameraKeyframe& p1) { return normalize(p0 * (1.0f - t) + p1 * t); }
void to_json(json& j, const CameraKeyframe& p) {
j = json{
{"R", p.R },
{"T", p.T },
{"fov", p.fov },
{"timestamp", p.timestamp},
};
}
bool load_relative_to_first = false; // set to true when using a camera path that is aligned with the first training image, such that it is
// invariant to changes in the space of the training data
void from_json(bool is_first, const json& j, CameraKeyframe& p, const CameraKeyframe& first, const mat4x3& ref) {
if (is_first && load_relative_to_first) {
p.from_m(ref);
} else {
p.R = j.at("R");
p.T = j.at("T");
if (load_relative_to_first) {
mat4 ref4 = {ref};
mat4 first4 = {first.m()};
mat4 p4 = {p.m()};
p.from_m(mat4x3(ref4 * inverse(first4) * p4));
}
}
j.at("fov").get_to(p.fov);
if (j.contains("timestamp")) {
j.at("timestamp").get_to(p.timestamp);
} else {
p.timestamp = 0.f;
}
}
void CameraPath::save(const fs::path& path) {
json j = {
{"loop", loop },
{"time", play_time },
{"path", keyframes },
{"duration_seconds", duration_seconds()},
{"spline_order", spline_order },
};
std::ofstream f(native_string(path));
f << j;
}
void CameraPath::load(const fs::path& path, const mat4x3& first_xform) {
std::ifstream f{native_string(path)};
if (!f) {
throw std::runtime_error{fmt::format("Camera path {} does not exist.", path.str())};
}
json j;
f >> j;
CameraKeyframe first;
keyframes.clear();
if (j.contains("loop")) {
loop = j["loop"];
}
if (j.contains("time")) {
play_time = j["time"];
}
if (j.contains("path")) {
for (auto& el : j["path"]) {
CameraKeyframe p;
bool is_first = keyframes.empty();
from_json(is_first, el, p, first, first_xform);
if (is_first) {
first = p;
}
keyframes.push_back(p);
}
}
spline_order = j.value("spline_order", 3);
sanitize_keyframes();
play_time = 0.0f;
if (keyframes.size() >= 16) {
keyframe_subsampling = keyframes.size() - 1;
editing_kernel_type = EEditingKernel::Gaussian;
}
}
void CameraPath::add_camera(const mat4x3& camera, float fov, float timestamp) {
int n = std::max(0, int(keyframes.size()) - 1);
int i = (int)ceil(play_time * (float)n + 0.001f);
if (i > keyframes.size()) {
i = keyframes.size();
}
if (i < 0) {
i = 0;
}
keyframes.insert(keyframes.begin() + i, CameraKeyframe(camera, fov, timestamp));
update_cam_from_path = false;
play_time = get_playtime(i);
sanitize_keyframes();
}
float editing_kernel(float x, EEditingKernel kernel) {
x = kernel == EEditingKernel::Gaussian ? x : clamp(x, -1.0f, 1.0f);
switch (kernel) {
case EEditingKernel::Gaussian: return expf(-2.0f * x * x);
case EEditingKernel::Quartic: return (1.0f - x * x) * (1.0f - x * x);
case EEditingKernel::Hat: return 1.0f - fabsf(x);
case EEditingKernel::Box: return x > -1.0f && x < 1.0f ? 1.0f : 0.0f;
case EEditingKernel::None: return fabs(x) < 0.0001f ? 1.0f : 0.0f;
default: throw std::runtime_error{"Unknown editing kernel"};
}
}
#ifdef NGP_GUI
int CameraPath::imgui(char path_filename_buf[1024], float frame_milliseconds, const mat4x3& camera, float fov, const mat4x3& first_xform) {
int n = std::max(0, int(keyframes.size()) - 1);
int read = 0; // 1=smooth, 2=hard
ImGui::InputText("##PathFile", path_filename_buf, 1024);
ImGui::SameLine();
static std::string camera_path_load_error_string = "";
if (rendering) {
ImGui::BeginDisabled();
}
if (ImGui::Button("Load")) {
try {
load(path_filename_buf, first_xform);
} catch (const std::exception& e) {
ImGui::OpenPopup("Camera path load error");
camera_path_load_error_string = std::string{"Failed to load camera path: "} + e.what();
}
}
if (rendering) {
ImGui::EndDisabled();
}
if (ImGui::BeginPopupModal("Camera path load error", NULL, ImGuiWindowFlags_AlwaysAutoResize)) {
ImGui::Text("%s", camera_path_load_error_string.c_str());
if (ImGui::Button("OK", ImVec2(120, 0))) {
ImGui::CloseCurrentPopup();
}
ImGui::EndPopup();
}
if (!keyframes.empty()) {
ImGui::SameLine();
if (ImGui::Button("Save")) {
save(path_filename_buf);
}
}
if (rendering) {
ImGui::BeginDisabled();
}
if (ImGui::Button("Add from cam")) {
const float duration = duration_seconds();
add_camera(camera, fov, 0.0f);
make_keyframe_timestamps_equidistant(duration);
read = 2;
}
auto p = get_pos(play_time);
if (!keyframes.empty()) {
ImGui::SameLine();
if (ImGui::Button("Split")) {
update_cam_from_path = false;
int i = clamp(p.kfidx + 1, 0, (int)keyframes.size());
const float duration = duration_seconds();
keyframes.insert(keyframes.begin() + i, eval_camera_path(play_time));
make_keyframe_timestamps_equidistant(duration);
play_time = get_playtime(i);
read = 2;
}
ImGui::SameLine();
int i = p.kfidx;
if (!loop) {
i += (int)round(p.t);
}
if (ImGui::Button("|<")) {
play_time = 0.f;
read = 2;
}
ImGui::SameLine();
if (ImGui::Button("<")) {
play_time = n ? (get_playtime(i - 1) + 0.0001f) : 0.f;
read = 2;
}
ImGui::SameLine();
if (ImGui::Button(update_cam_from_path ? "Stop" : "Read")) {
update_cam_from_path = !update_cam_from_path;
read = 2;
}
ImGui::SameLine();
if (ImGui::Button(">")) {
play_time = n ? (get_playtime(i + 1) + 0.0001f) : 1.0f;
read = 2;
}
ImGui::SameLine();
if (ImGui::Button(">|")) {
play_time = 1.0f;
read = 2;
}
ImGui::SameLine();
if (ImGui::Button("Dup")) {
update_cam_from_path = false;
const float duration = duration_seconds();
keyframes.insert(keyframes.begin() + i, keyframes[i]);
make_keyframe_timestamps_equidistant(duration);
play_time = get_playtime(i);
read = 2;
}
ImGui::SameLine();
if (ImGui::Button("Del")) {
update_cam_from_path = false;
const float duration = duration_seconds();
keyframes.erase(keyframes.begin() + i);
make_keyframe_timestamps_equidistant(duration);
play_time = get_playtime(i - 1);
read = 2;
}
ImGui::SameLine();
if (ImGui::Button("Set")) {
keyframes[i] = CameraKeyframe(camera, fov, keyframes[i].timestamp);
read = 2;
if (n) {
play_time = get_playtime(i);
}
}
if (ImGui::RadioButton("Translate", m_gizmo_op == ImGuizmo::TRANSLATE)) {
m_gizmo_op = ImGuizmo::TRANSLATE;
}
ImGui::SameLine();
if (ImGui::RadioButton("Rotate", m_gizmo_op == ImGuizmo::ROTATE)) {
m_gizmo_op = ImGuizmo::ROTATE;
}
ImGui::SameLine();
if (ImGui::RadioButton("Local", m_gizmo_mode == ImGuizmo::LOCAL)) {
m_gizmo_mode = ImGuizmo::LOCAL;
}
ImGui::SameLine();
if (ImGui::RadioButton("World", m_gizmo_mode == ImGuizmo::WORLD)) {
m_gizmo_mode = ImGuizmo::WORLD;
}
ImGui::SameLine();
ImGui::Checkbox("Loop path", &loop);
if (ImGui::Button("Start") && !keyframes.empty()) {
auto_play_speed = 0.0f;
play_time = 0.0f;
read = 2;
}
ImGui::SameLine();
if (ImGui::Button("Rev") && !keyframes.empty()) {
auto_play_speed = -1.0f / duration_seconds();
}
ImGui::SameLine();
if (ImGui::Button(auto_play_speed != 0 ? "Pause" : "Play") && !keyframes.empty()) {
auto_play_speed = auto_play_speed == 0.0f ? (1.0f / duration_seconds()) : 0.0f;
}
ImGui::SameLine();
if (ImGui::Button("End") && !keyframes.empty()) {
auto_play_speed = 0.0f;
play_time = 1.0f;
read = 2;
}
ImGui::SliderFloat("Playback speed", &auto_play_speed, -1.0f, 1.0f);
if (auto_play_speed != 0.0f) {
float prev = play_time;
play_time = clamp(play_time + auto_play_speed * (frame_milliseconds / 1000.f), 0.0f, 1.0f);
if (play_time != prev) {
read = 1;
}
}
if (ImGui::SliderFloat("Camera path time", &play_time, 0.0f, 1.0f)) {
read = 1;
}
ImGui::Text("Current keyframe %d/%d:", i, n + 1);
if (ImGui::SliderFloat("Field of view", &keyframes[i].fov, 0.0f, 120.0f)) {
read = 2;
}
if (ImGui::Button("Apply to all keyframes")) {
for (auto& k : keyframes) {
k.fov = keyframes[i].fov;
}
}
if (ImGui::TreeNodeEx("Batch keyframe editing")) {
ImGui::Combo("Editing kernel", (int*)&editing_kernel_type, EditingKernelStr);
ImGui::SliderFloat(
"Editing kernel radius", &editing_kernel_radius, 0.001f, 10.0f, "%.4f", ImGuiSliderFlags_Logarithmic | ImGuiSliderFlags_NoRoundToFormat
);
ImGui::TreePop();
}
if (ImGui::TreeNodeEx("Advanced camera path settings")) {
ImGui::SliderInt("Spline order", &spline_order, 0, 3);
ImGui::SliderInt("Keyframe subsampling", &keyframe_subsampling, 1, max((int)keyframes.size() - 1, 1));
ImGui::TreePop();
}
}
if (rendering) {
ImGui::EndDisabled();
}
return keyframes.empty() ? 0 : read;
}
bool debug_project(const mat4& proj, vec3 p, ImVec2& o) {
vec4 ph{p.x, p.y, p.z, 1.0f};
vec4 pa = proj * ph;
if (pa.w <= 0.f) {
return false;
}
o.x = pa.x / pa.w;
o.y = pa.y / pa.w;
return true;
}
void add_debug_line(ImDrawList* list, const mat4& proj, vec3 a, vec3 b, uint32_t col, float thickness) {
ImVec2 aa, bb;
if (debug_project(proj, a, aa) && debug_project(proj, b, bb)) {
list->AddLine(aa, bb, col, thickness * 2.0f);
}
}
void visualize_cube(ImDrawList* list, const mat4& world2proj, const vec3& a, const vec3& b, const mat3& render_aabb_to_local) {
mat3 m = transpose(render_aabb_to_local);
add_debug_line(list, world2proj, m * vec3{a.x, a.y, a.z}, m * vec3{a.x, a.y, b.z}, 0xffff4040); // Z
add_debug_line(list, world2proj, m * vec3{b.x, a.y, a.z}, m * vec3{b.x, a.y, b.z}, 0xffffffff);
add_debug_line(list, world2proj, m * vec3{a.x, b.y, a.z}, m * vec3{a.x, b.y, b.z}, 0xffffffff);
add_debug_line(list, world2proj, m * vec3{b.x, b.y, a.z}, m * vec3{b.x, b.y, b.z}, 0xffffffff);
add_debug_line(list, world2proj, m * vec3{a.x, a.y, a.z}, m * vec3{b.x, a.y, a.z}, 0xff4040ff); // X
add_debug_line(list, world2proj, m * vec3{a.x, b.y, a.z}, m * vec3{b.x, b.y, a.z}, 0xffffffff);
add_debug_line(list, world2proj, m * vec3{a.x, a.y, b.z}, m * vec3{b.x, a.y, b.z}, 0xffffffff);
add_debug_line(list, world2proj, m * vec3{a.x, b.y, b.z}, m * vec3{b.x, b.y, b.z}, 0xffffffff);
add_debug_line(list, world2proj, m * vec3{a.x, a.y, a.z}, m * vec3{a.x, b.y, a.z}, 0xff40ff40); // Y
add_debug_line(list, world2proj, m * vec3{b.x, a.y, a.z}, m * vec3{b.x, b.y, a.z}, 0xffffffff);
add_debug_line(list, world2proj, m * vec3{a.x, a.y, b.z}, m * vec3{a.x, b.y, b.z}, 0xffffffff);
add_debug_line(list, world2proj, m * vec3{b.x, a.y, b.z}, m * vec3{b.x, b.y, b.z}, 0xffffffff);
}
void visualize_camera(ImDrawList* list, const mat4& world2proj, const mat4x3& xform, float aspect, uint32_t col, float thickness) {
const float axis_size = 0.025f;
const vec3* xforms = (const vec3*)&xform;
vec3 pos = xforms[3];
add_debug_line(list, world2proj, pos, pos + axis_size * xforms[0], 0xff4040ff, thickness);
add_debug_line(list, world2proj, pos, pos + axis_size * xforms[1], 0xff40ff40, thickness);
add_debug_line(list, world2proj, pos, pos + axis_size * xforms[2], 0xffff4040, thickness);
float xs = axis_size * aspect;
float ys = axis_size;
float zs = axis_size * 2.0f * aspect;
vec3 a = pos + xs * xforms[0] + ys * xforms[1] + zs * xforms[2];
vec3 b = pos - xs * xforms[0] + ys * xforms[1] + zs * xforms[2];
vec3 c = pos - xs * xforms[0] - ys * xforms[1] + zs * xforms[2];
vec3 d = pos + xs * xforms[0] - ys * xforms[1] + zs * xforms[2];
add_debug_line(list, world2proj, pos, a, col, thickness);
add_debug_line(list, world2proj, pos, b, col, thickness);
add_debug_line(list, world2proj, pos, c, col, thickness);
add_debug_line(list, world2proj, pos, d, col, thickness);
add_debug_line(list, world2proj, a, b, col, thickness);
add_debug_line(list, world2proj, b, c, col, thickness);
add_debug_line(list, world2proj, c, d, col, thickness);
add_debug_line(list, world2proj, d, a, col, thickness);
}
bool CameraPath::has_valid_timestamps() const {
float prev_timestamp = 0.0f;
for (size_t i = 0; i < keyframes.size(); ++i) {
if (!(keyframes[i].timestamp > prev_timestamp)) {
return false;
}
prev_timestamp = keyframes[i].timestamp;
}
return true;
}
void CameraPath::make_keyframe_timestamps_equidistant(const float duration_seconds) {
const float sanitized_duration = duration_seconds > 0.0f ? duration_seconds : default_duration_seconds;
for (size_t i = 0; i < keyframes.size(); ++i) {
keyframes[i].timestamp = sanitized_duration * (i + 1) / (float)keyframes.size();
}
}
void CameraPath::sanitize_keyframes() {
if (has_valid_timestamps()) {
return;
}
// Timestamps are invalid. Best effort is to equally space all frames. Default to 3 seconds duration.
make_keyframe_timestamps_equidistant(default_duration_seconds);
}
float CameraPath::duration_seconds() const {
if (keyframes.empty()) {
return 0.0f;
}
return keyframes.back().timestamp;
}
void CameraPath::set_duration_seconds(const float duration) {
const float old_duration = duration_seconds();
if (!(old_duration > 0.0f)) {
make_keyframe_timestamps_equidistant(duration);
return;
}
const float multiplier = duration / old_duration;
for (auto& kf : keyframes) {
kf.timestamp *= multiplier;
}
}
CameraPath::Pos CameraPath::get_pos(float playtime) {
if (keyframes.empty()) {
return {-1, 0.0f};
} else if (keyframes.size() == 1) {
return {0, playtime};
}
const float duration = loop ? keyframes.back().timestamp : keyframes[keyframes.size() - 2].timestamp;
playtime *= duration;
CameraKeyframe dummy;
dummy.timestamp = playtime;
// Binary search to obtain relevant keyframe in O(log(n_keyframes)) time
auto it = std::upper_bound(keyframes.begin(), keyframes.end(), dummy, [](const auto& a, const auto& b) {
return a.timestamp < b.timestamp;
});
int i = clamp((int)std::distance(keyframes.begin(), it), 0, (int)keyframes.size() - (loop ? 1 : 2));
float prev_timestamp = i == 0 ? 0.0f : keyframes[i - 1].timestamp;
return {
i,
(playtime - prev_timestamp) / (keyframes[i].timestamp - prev_timestamp),
};
}
bool CameraPath::imgui_viz(
ImDrawList* list,
mat4& view2proj,
mat4& world2proj,
mat4& world2view,
vec2 focal,
float aspect,
float znear,
float zfar
) {
bool changed = false;
// float flx = focal.x;
float fly = focal.y;
mat4 view2proj_guizmo = transpose(
mat4{
fly * 2.0f / aspect,
0.0f,
0.0f,
0.0f,
0.0f,
-fly * 2.0f,
0.0f,
0.0f,
0.0f,
0.0f,
(zfar + znear) / (zfar - znear),
-(2.0f * zfar * znear) / (zfar - znear),
0.0f,
0.0f,
1.0f,
0.0f,
}
);
if (!update_cam_from_path && !keyframes.empty()) {
auto p = get_pos(play_time);
int cur_cam_i = p.kfidx;
if (!loop) {
cur_cam_i += (int)round(p.t);
}
vec3 prevp;
for (int i = 0; i < keyframes.size(); i += max(min(keyframe_subsampling, (int)keyframes.size() - 1 - i), 1)) {
visualize_camera(list, world2proj, keyframes[i].m(), aspect, (i == cur_cam_i) ? 0xff80c0ff : 0x8080c0ff);
vec3 p = keyframes[i].T;
if (i && keyframe_subsampling == 1) {
add_debug_line(list, world2proj, prevp, p, 0xccffc040);
}
prevp = p;
}
ImGuiIO& io = ImGui::GetIO();
mat4 matrix = keyframes[cur_cam_i].m();
ImGuizmo::SetRect(0, 0, io.DisplaySize.x, io.DisplaySize.y);
if (ImGuizmo::Manipulate(
(const float*)&world2view,
(const float*)&view2proj_guizmo,
(ImGuizmo::OPERATION)m_gizmo_op,
(ImGuizmo::MODE)m_gizmo_mode,
(float*)&matrix,
NULL,
NULL
)) {
// Find overlapping keypoints...
int i0 = cur_cam_i;
while (i0 > 0 && keyframes[cur_cam_i].same_pos_as(keyframes[i0 - 1])) {
i0--;
}
int i1 = cur_cam_i;
while (i1 < keyframes.size() - 1 && keyframes[cur_cam_i].same_pos_as(keyframes[i1 + 1])) {
i1++;
}
vec3 tdiff = matrix[3].xyz() - keyframes[cur_cam_i].T;
mat3 rdiff = mat_log(mat3(matrix) * inverse(to_mat3(normalize(keyframes[cur_cam_i].R))));
for (int i = 0; i < keyframes.size(); ++i) {
float x = (get_playtime(i) - get_playtime(cur_cam_i)) / editing_kernel_radius;
float w = editing_kernel(x, editing_kernel_type);
keyframes[i].T += w * tdiff;
keyframes[i].R = quat(mat_exp(w * rdiff) * to_mat3(normalize(keyframes[i].R)));
}
// ...and ensure overlapping keypoints were edited exactly in tandem
for (int i = i0; i <= i1; ++i) {
keyframes[i].T = keyframes[cur_cam_i].T;
keyframes[i].R = keyframes[cur_cam_i].R;
}
changed = true;
}
visualize_camera(list, world2proj, eval_camera_path(play_time).m(), aspect, 0xff80ff80);
float dt = 0.001f;
float total_length = 0.0f;
for (float t = 0.0f;; t += dt) {
if (t > 1.0f) {
t = 1.0f;
}
vec3 p = eval_camera_path(t).T;
if (t) {
total_length += distance(prevp, p);
}
prevp = p;
if (t >= 1.0f) {
break;
}
}
dt = 0.001f / total_length;
static const uint32_t N_DASH_STEPS = 10;
uint32_t i = 0;
for (float t = 0.0f;; t += dt, ++i) {
if (t > 1.0f) {
t = 1.0f;
}
vec3 p = eval_camera_path(t).T;
if (t && (i / N_DASH_STEPS) % 2 == 0) {
float thickness = 1.0f;
if (editing_kernel_type != EEditingKernel::None) {
float x = (t + dt / 2.0f - get_playtime(cur_cam_i)) / editing_kernel_radius;
thickness += 4.0f * editing_kernel(x, editing_kernel_type);
}
add_debug_line(list, world2proj, prevp, p, 0xff80c0ff, thickness);
}
prevp = p;
if (t >= 1.0f) {
break;
}
}
}
return changed;
}
#endif // NGP_GUI
} // namespace ngp