Hugging Face's logo

camenduru
/
instant-ngp

Model card Files
xet
 Community
instant-ngp / src /camera_path.cu
camenduru's picture
camenduru
instant-ngp build
7873319
Raw History Blame Contribute Delete
14.8 kB
/*
* Copyright (c) 2020-2022, NVIDIA CORPORATION. All rights reserved.
*
* NVIDIA CORPORATION and its licensors retain all intellectual property
* and proprietary rights in and to this software, related documentation
* and any modifications thereto. Any use, reproduction, disclosure or
* distribution of this software and related documentation without an express
* license agreement from NVIDIA CORPORATION is strictly prohibited.
*/
/** @file camera_path.cpp
* @author Thomas Müller & Alex Evans, NVIDIA
*/
#include <neural-graphics-primitives/camera_path.h>
#include <neural-graphics-primitives/common.h>
#ifdef NGP_GUI
#include <imgui/imgui.h>
#include <imguizmo/ImGuizmo.h>
#endif
#include <json/json.hpp>
#include <fstream>
using namespace Eigen;
using namespace nlohmann;
NGP_NAMESPACE_BEGIN
CameraKeyframe lerp(const CameraKeyframe& p0, const CameraKeyframe& p1, float t, float t0, float t1) {
t = (t - t0) / (t1 - t0);
Eigen::Vector4f R1 = p1.R;
// take the short path
if (R1.dot(p0.R) < 0.f) {
R1=-R1;
}
return {
Eigen::Quaternionf(p0.R).slerp(t, Eigen::Quaternionf(R1)).coeffs(),
p0.T + (p1.T - p0.T) * t,
p0.slice + (p1.slice - p0.slice) * t,
p0.scale + (p1.scale - p0.scale) * t,
p0.fov + (p1.fov - p0.fov) * t,
p0.aperture_size + (p1.aperture_size - p0.aperture_size) * t,
// Note, the glow mode from the previous frame is used, since the modes cannot be interpolated
p0.glow_mode,
p0.glow_y_cutoff + (p1.glow_y_cutoff - p0.glow_y_cutoff) * t,
};
}
CameraKeyframe spline(float t, const CameraKeyframe& p0, const CameraKeyframe& p1, const CameraKeyframe& p2, const CameraKeyframe& p3) {
if (0) {
// catmull rom spline
CameraKeyframe q0 = lerp(p0, p1, t, -1.f, 0.f);
CameraKeyframe q1 = lerp(p1, p2, t, 0.f, 1.f);
CameraKeyframe q2 = lerp(p2, p3, t, 1.f, 2.f);
CameraKeyframe r0 = lerp(q0, q1, t, -1.f, 1.f);
CameraKeyframe r1 = lerp(q1, q2, t, 0.f, 2.f);
return lerp(r0, r1, t, 0.f, 1.f);
} else {
// cubic bspline
float tt=t*t;
float ttt=t*t*t;
float a = (1-t)*(1-t)*(1-t)*(1.f/6.f);
float b = (3.f*ttt-6.f*tt+4.f)*(1.f/6.f);
float c = (-3.f*ttt+3.f*tt+3.f*t+1.f)*(1.f/6.f);
float d = ttt*(1.f/6.f);
return p0 * a + p1 * b + p2 * c + p3 * d;
}
}
void to_json(json& j, const CameraKeyframe& p) {
j = json{{"R", p.R}, {"T", p.T}, {"slice", p.slice}, {"scale", p.scale}, {"fov", p.fov}, {"aperture_size", p.aperture_size}, {"glow_mode", p.glow_mode}, {"glow_y_cutoff", p.glow_y_cutoff}};
}
bool load_relative_to_first=false; // set to true when using a camera path that is aligned with the first training image, such that it is invariant to changes in the space of the training data
void from_json(bool is_first, const json& j, CameraKeyframe& p, const CameraKeyframe& first, const Eigen::Matrix<float, 3, 4>& ref) {
if (is_first && load_relative_to_first) {
p.from_m(ref);
} else {
p.R=Eigen::Vector4f(j["R"][0],j["R"][1],j["R"][2],j["R"][3]);
p.T=Eigen::Vector3f(j["T"][0],j["T"][1],j["T"][2]);
if (load_relative_to_first) {
Eigen::Matrix4f ref4 = Eigen::Matrix4f::Identity();
ref4.block<3, 4>(0, 0) = ref;
Eigen::Matrix4f first4 = Eigen::Matrix4f::Identity();
first4.block<3, 4>(0, 0) = first.m();
Eigen::Matrix4f p4 = Eigen::Matrix4f::Identity();
p4.block<3, 4>(0, 0) = p.m();
auto cur4 = ref4 * first4.inverse() * p4;
p.from_m(cur4.block<3, 4>(0, 0));
}
}
j.at("slice").get_to(p.slice);
j.at("scale").get_to(p.scale);
j.at("fov").get_to(p.fov);
if (j.contains("dof")) j.at("dof").get_to(p.aperture_size); else j.at("aperture_size").get_to(p.aperture_size);
if (j.contains("glow_mode")) j.at("glow_mode").get_to(p.glow_mode); else p.glow_mode = 0;
if (j.contains("glow_y_cutoff")) j.at("glow_y_cutoff").get_to(p.glow_y_cutoff); else p.glow_y_cutoff = 0.f;
}
void CameraPath::save(const std::string& filepath_string) {
json j = {
{"time", m_playtime},
{"path", m_keyframes}
};
std::ofstream f(filepath_string);
f << j;
}
void CameraPath::load(const std::string& filepath_string, const Eigen::Matrix<float, 3, 4> &first_xform) {
std::ifstream f(filepath_string);
if (!f) {
throw std::runtime_error{fmt::format("Camera path {} does not exist.", filepath_string)};
}
json j;
f >> j;
CameraKeyframe first;
m_keyframes.clear();
if (j.contains("time")) m_playtime=j["time"];
if (j.contains("path")) for (auto &el : j["path"]) {
CameraKeyframe p;
bool is_first = m_keyframes.empty();
from_json(is_first, el, p, first, first_xform);
if (is_first) {
first = p;
}
m_keyframes.push_back(p);
}
}
#ifdef NGP_GUI
int CameraPath::imgui(char path_filename_buf[128], float frame_milliseconds, Matrix<float, 3, 4>& camera, float slice_plane_z, float scale, float fov, float aperture_size, float bounding_radius, const Eigen::Matrix<float, 3, 4>& first_xform, int glow_mode, float glow_y_cutoff) {
int n=std::max(0,int(m_keyframes.size())-1);
int read= 0; // 1=smooth, 2=hard
if (!m_keyframes.empty()) {
if (ImGui::SliderFloat("camera path time", &m_playtime, 0.f, 1.f)) read=1;
ImGui::SliderFloat("auto play speed",&m_autoplayspeed, 0.f, 1.f);
if (m_autoplayspeed>0.f && m_playtime<1.f) {
m_playtime+=m_autoplayspeed*(frame_milliseconds/1000.f);
if (m_playtime>1.f) m_playtime=1.f;
read=1;
}
}
if (ImGui::Button("Add from cam")) {
int i=(int)ceil(m_playtime*(float)n+0.001f);
if (i>m_keyframes.size()) i=m_keyframes.size();
if (i<0) i=0;
m_keyframes.insert(m_keyframes.begin()+i, CameraKeyframe(camera, slice_plane_z, scale, fov, aperture_size, glow_mode, glow_y_cutoff));
m_update_cam_from_path = false;
int n=std::max(0,int(m_keyframes.size())-1);
m_playtime = n ? float(i)/float(n) : 1.f;
read = 2;
}
if (!m_keyframes.empty()) {
ImGui::SameLine();
if (ImGui::Button("split")) {
m_update_cam_from_path=false;
int i=(int)ceil(m_playtime*(float)n+0.001f);
if (i>m_keyframes.size()) i=(int)m_keyframes.size();
if (i<0) i=0;
m_keyframes.insert(m_keyframes.begin()+i, eval_camera_path(m_playtime));
m_playtime=float(i)/float(n+1);
read=2;
}
ImGui::SameLine();
int i=(int)round(m_playtime*(float)n);
if (ImGui::Button("|<")) { m_playtime=0.f; read=2; } ImGui::SameLine();
if (ImGui::Button("<")) { m_playtime=n?std::max(0.f,floorf((m_playtime-0.0001f)*(float)n)/(float)n):0.f; read=2;} ImGui::SameLine();
if (ImGui::Button(m_update_cam_from_path ? "STOP" : "READ")) { m_update_cam_from_path=!m_update_cam_from_path; read=2; } ImGui::SameLine();
if (ImGui::Button(">")) { m_playtime=n?std::min(1.f,ceilf((m_playtime+0.0001f)*(float)n)/(float)n):1.f; read=2;} ImGui::SameLine();
if (ImGui::Button(">|")) { m_playtime=1.f; read=2;} ImGui::SameLine();
if (ImGui::Button("Dup")) { m_update_cam_from_path=false; m_keyframes.insert(m_keyframes.begin()+i, m_keyframes[i]); m_playtime=i/float(n+1); read=2;} ImGui::SameLine();
if (ImGui::Button("Del")) { m_update_cam_from_path=false; m_keyframes.erase(m_keyframes.begin()+i); read=2;} ImGui::SameLine();
if (ImGui::Button("Set")) { m_keyframes[i]=CameraKeyframe(camera, slice_plane_z, scale, fov, aperture_size, glow_mode, glow_y_cutoff); read=2; if (n) m_playtime=i/float(n); }
if (ImGui::RadioButton("Translate", m_gizmo_op == ImGuizmo::TRANSLATE))
m_gizmo_op = ImGuizmo::TRANSLATE;
ImGui::SameLine();
if (ImGui::RadioButton("Rotate", m_gizmo_op == ImGuizmo::ROTATE))
m_gizmo_op = ImGuizmo::ROTATE;
ImGui::SameLine();
if (ImGui::RadioButton("Local", m_gizmo_mode == ImGuizmo::LOCAL))
m_gizmo_mode = ImGuizmo::LOCAL;
ImGui::SameLine();
if (ImGui::RadioButton("World", m_gizmo_mode == ImGuizmo::WORLD))
m_gizmo_mode = ImGuizmo::WORLD;
}
ImGui::InputText("##PathFile", path_filename_buf, 128);
ImGui::SameLine();
static std::string camera_path_load_error_string = "";
if (ImGui::Button("Load")) {
try {
load(path_filename_buf, first_xform);
} catch (std::exception& e) {
ImGui::OpenPopup("Camera path load error");
camera_path_load_error_string = std::string{"Failed to load camera path: "} + e.what();
}
}
if (ImGui::BeginPopupModal("Camera path load error", NULL, ImGuiWindowFlags_AlwaysAutoResize)) {
ImGui::Text("%s", camera_path_load_error_string.c_str());
if (ImGui::Button("OK", ImVec2(120, 0))) {
ImGui::CloseCurrentPopup();
}
ImGui::EndPopup();
}
if (!m_keyframes.empty()) {
ImGui::SameLine();
if (ImGui::Button("Save"))
save(path_filename_buf);
}
if (!m_keyframes.empty()) {
int i=(int)round(m_playtime*(float)n);
ImGui::Text("Current keyframe %d/%d:", i, n+1);
if (ImGui::SliderFloat("Field of view", &m_keyframes[i].fov, 0.0f, 120.0f)) read=2;
if (ImGui::SliderFloat("Aperture size", &m_keyframes[i].aperture_size, 0.0f, 0.1f)) read=2;
if (ImGui::SliderFloat("Slice Z", &m_keyframes[i].slice, -bounding_radius, bounding_radius)) read=2;
if (ImGui::SliderFloat("Scale", &m_keyframes[i].scale, 0.f,10.f)) read=2;
if (ImGui::SliderInt("Glow Mode", &m_keyframes[i].glow_mode, 0,16)) read=2;
if (ImGui::SliderFloat("Glow Y Cutoff", &m_keyframes[i].glow_y_cutoff, -2.f,3.f)) read=2;
}
return m_keyframes.empty() ? 0 : read;
}
bool DebugProject(const Matrix<float, 4, 4>&proj, Vector3f p, ImVec2& o) {
Vector4f ph; ph << p, 1.f;
Vector4f pa = proj * ph;
if (pa.w() <= 0.f) return false;
o.x = pa.x() / pa.w();
o.y = pa.y() / pa.w();
return true;
}
void add_debug_line(ImDrawList* list, const Matrix<float, 4, 4>& proj, Vector3f a, Vector3f b, uint32_t col, float thickness) {
ImVec2 aa, bb;
if (DebugProject(proj, a, aa) && DebugProject(proj, b, bb)) {
list->AddLine(aa, bb, col, thickness);
}
}
void visualize_unit_cube(ImDrawList* list, const Matrix<float, 4, 4>& world2proj, const Vector3f& a, const Vector3f& b, const Matrix3f& render_aabb_to_local) {
Eigen::Matrix3f m = render_aabb_to_local.transpose();
add_debug_line(list, world2proj, m * Vector3f{a.x(),a.y(),a.z()}, m * Vector3f{a.x(),a.y(),b.z()}, 0xffff4040); // Z
add_debug_line(list, world2proj, m * Vector3f{b.x(),a.y(),a.z()}, m * Vector3f{b.x(),a.y(),b.z()}, 0xffffffff);
add_debug_line(list, world2proj, m * Vector3f{a.x(),b.y(),a.z()}, m * Vector3f{a.x(),b.y(),b.z()}, 0xffffffff);
add_debug_line(list, world2proj, m * Vector3f{b.x(),b.y(),a.z()}, m * Vector3f{b.x(),b.y(),b.z()}, 0xffffffff);
add_debug_line(list, world2proj, m * Vector3f{a.x(),a.y(),a.z()}, m * Vector3f{b.x(),a.y(),a.z()}, 0xff4040ff); // X
add_debug_line(list, world2proj, m * Vector3f{a.x(),b.y(),a.z()}, m * Vector3f{b.x(),b.y(),a.z()}, 0xffffffff);
add_debug_line(list, world2proj, m * Vector3f{a.x(),a.y(),b.z()}, m * Vector3f{b.x(),a.y(),b.z()}, 0xffffffff);
add_debug_line(list, world2proj, m * Vector3f{a.x(),b.y(),b.z()}, m * Vector3f{b.x(),b.y(),b.z()}, 0xffffffff);
add_debug_line(list, world2proj, m * Vector3f{a.x(),a.y(),a.z()}, m * Vector3f{a.x(),b.y(),a.z()}, 0xff40ff40); // Y
add_debug_line(list, world2proj, m * Vector3f{b.x(),a.y(),a.z()}, m * Vector3f{b.x(),b.y(),a.z()}, 0xffffffff);
add_debug_line(list, world2proj, m * Vector3f{a.x(),a.y(),b.z()}, m * Vector3f{a.x(),b.y(),b.z()}, 0xffffffff);
add_debug_line(list, world2proj, m * Vector3f{b.x(),a.y(),b.z()}, m * Vector3f{b.x(),b.y(),b.z()}, 0xffffffff);
}
void visualize_nerf_camera(ImDrawList* list, const Matrix<float, 4, 4>& world2proj, const Eigen::Matrix<float, 3, 4>& xform, float aspect, uint32_t col, float thickness) {
const float axis_size = 0.025f;
const Vector3f *xforms = (const Vector3f*)&xform;
Vector3f pos = xforms[3];
add_debug_line(list, world2proj, pos, pos+axis_size*xforms[0], 0xff4040ff, thickness);
add_debug_line(list, world2proj, pos, pos+axis_size*xforms[1], 0xff40ff40, thickness);
add_debug_line(list, world2proj, pos, pos+axis_size*xforms[2], 0xffff4040, thickness);
float xs=axis_size*aspect;
float ys=axis_size;
float zs=axis_size*2.f*aspect;
Vector3f a=pos+xs*xforms[0]+ys*xforms[1]+zs*xforms[2];
Vector3f b=pos-xs*xforms[0]+ys*xforms[1]+zs*xforms[2];
Vector3f c=pos-xs*xforms[0]-ys*xforms[1]+zs*xforms[2];
Vector3f d=pos+xs*xforms[0]-ys*xforms[1]+zs*xforms[2];
add_debug_line(list, world2proj, pos, a, col, thickness);
add_debug_line(list, world2proj, pos, b, col, thickness);
add_debug_line(list, world2proj, pos, c, col, thickness);
add_debug_line(list, world2proj, pos, d, col, thickness);
add_debug_line(list, world2proj, a, b, col, thickness);
add_debug_line(list, world2proj, b, c, col, thickness);
add_debug_line(list, world2proj, c, d, col, thickness);
add_debug_line(list, world2proj, d, a, col, thickness);
}
bool CameraPath::imgui_viz(ImDrawList* list, Matrix<float, 4, 4> &view2proj, Matrix<float, 4, 4> &world2proj, Matrix<float, 4, 4> &world2view, Vector2f focal, float aspect) {
bool changed=false;
float flx = focal.x();
float fly = focal.y();
Matrix<float, 4, 4> view2proj_guizmo;
float zfar = 100.f;
float znear = 0.1f;
view2proj_guizmo <<
fly*2.f/aspect, 0, 0, 0,
0, -fly*2.f, 0, 0,
0, 0, (zfar+znear)/(zfar-znear), -(2.f*zfar*znear) / (zfar-znear),
0, 0, 1, 0;
if (!m_update_cam_from_path) {
ImDrawList* list = ImGui::GetForegroundDrawList();
int cur_cam_i=(int)round(m_playtime * (float)(m_keyframes.size()-1));
Eigen::Vector3f prevp;
for (int i=0;i<m_keyframes.size();++i) {
visualize_nerf_camera(list, world2proj, m_keyframes[i].m(), aspect, (i==cur_cam_i) ? 0xff80c0ff : 0x8080c0ff);
Eigen::Vector3f p=m_keyframes[i].T;
if (i) {
add_debug_line(list, world2proj, prevp, p, 0xccffc040);
}
prevp=p;
}
if (!m_keyframes.empty()) {
ImGuiIO& io = ImGui::GetIO();
Eigen::Matrix4f matrix=Eigen::Matrix4f::Identity();
matrix.block<3,4>(0,0) = m_keyframes[cur_cam_i].m();
ImGuizmo::SetRect(0, 0, io.DisplaySize.x, io.DisplaySize.y);
if (ImGuizmo::Manipulate((const float*)&world2view, (const float*)&view2proj_guizmo, (ImGuizmo::OPERATION)m_gizmo_op, (ImGuizmo::MODE)m_gizmo_mode, (float*)&matrix, NULL, NULL)) {
int i0=cur_cam_i; while (i0>0 && m_keyframes[cur_cam_i].SamePosAs(m_keyframes[i0-1])) i0--;
int i1=cur_cam_i; while (i1<m_keyframes.size()-1 && m_keyframes[cur_cam_i].SamePosAs(m_keyframes[i1+1])) i1++;
for (int i=i0;i<=i1;++i) {
m_keyframes[i].T=matrix.block<3,4>(0,0).col(3);
m_keyframes[i].R=Eigen::Quaternionf(matrix.block<3,3>(0,0)).coeffs();
}
changed=true;
}
visualize_nerf_camera(list, world2proj, eval_camera_path(m_playtime).m(), aspect, 0xff80ff80);
float dt = 0.05f / (float)m_keyframes.size();
Eigen::Vector3f prevp;
for (float t=0.f;;t+=dt) {
if (t>1.f) t=1.f;
Eigen::Vector3f p=eval_camera_path(t).T;
if (t) {
// draw a line
add_debug_line(list, world2proj, (prevp+p)*0.5f, p, 0xff80c0ff);
}
prevp=p;
if (t>=1.f) break;
}
}
}
return changed;
}
#endif //NGP_GUI
NGP_NAMESPACE_END