Instructions to use Viggle/Meridian with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use Viggle/Meridian with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("Viggle/Meridian", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
Download service/index.html from Viggle/Meridian: direct link, hf CLI and curl.
- Browser
- Download file 51.3 kB
-
https://huggingface.co/Viggle/Meridian/resolve/main/service/index.html
- Command line
-
hf download hf://Viggle/Meridian/service/index.html
-
curl -L -o index.html https://huggingface.co/Viggle/Meridian/resolve/main/service/index.html
51.3 kB
| <title>recam — camera moves for footage you already have</title> | |
| <style> | |
| :root{--bg:#0b0c0f;--pan:#14161c;--line:#262a35;--ink:#e8eaf0;--dim:#8b93a7;--acc:#7dd3fc;--ok:#4ade80;--bad:#f87171;--warn:#fbbf24} | |
| *{box-sizing:border-box} | |
| body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif} | |
| #rail{display:flex;gap:2px;padding:14px 20px 0;font-size:12px;letter-spacing:.04em} | |
| #rail div{padding:6px 12px;color:#4a5060;border-bottom:2px solid transparent;text-transform:uppercase} | |
| #rail div.on{color:var(--acc);border-bottom-color:var(--acc)} | |
| #rail div.done{color:var(--dim)} | |
| main{padding:22px 20px 60px;max-width:1240px;margin:0 auto} | |
| h1{font-size:22px;font-weight:600;margin:0 0 6px} | |
| p.sub{color:var(--dim);margin:0 0 22px;max-width:80ch} | |
| .pan{background:var(--pan);border:1px solid var(--line);border-radius:10px;padding:16px} | |
| button{background:#1e2230;color:var(--ink);border:1px solid var(--line);border-radius:7px;padding:8px 15px;font:inherit;cursor:pointer} | |
| button:hover{border-color:#3d4457} | |
| button.go{background:var(--acc);color:#08131a;border-color:var(--acc);font-weight:600} | |
| button.go:disabled{background:#232733;color:#4a5060;border-color:var(--line);cursor:not-allowed;font-weight:400} | |
| button.ghost{background:none} | |
| button.sm{padding:1px 7px;font-size:11px} | |
| #drop{border:1px dashed #333a4a;border-radius:12px;padding:44px;text-align:center;color:var(--dim);cursor:pointer} | |
| #drop.hot{border-color:var(--acc);color:var(--acc)} | |
| .chips{display:flex;flex-wrap:wrap;gap:7px;margin-top:14px} | |
| .chip{font-size:12px;padding:5px 11px;border:1px solid var(--line);border-radius:20px;cursor:pointer;color:var(--dim)} | |
| .chip:hover{color:var(--ink);border-color:#3d4457} | |
| label.f{display:block;font-size:11px;text-transform:uppercase;letter-spacing:.05em;color:var(--dim);margin:14px 0 5px} | |
| label.f b{color:var(--ink);font-variant-numeric:tabular-nums;font-weight:600;text-transform:none;letter-spacing:0} | |
| input[type=range]{width:100%;accent-color:var(--acc)} | |
| #tl{width:100%;height:38px;display:block;border-radius:6px;background:#0e1015;margin-top:6px} | |
| .kv{display:grid;grid-template-columns:auto 1fr;gap:4px 12px;font-size:13px;font-variant-numeric:tabular-nums} | |
| .kv i{color:var(--dim);font-style:normal} | |
| #studio{display:grid;grid-template-columns:1.2fr 1fr;gap:18px} | |
| #pv,#pk{background:#000;border-radius:8px;width:100%;display:block;image-rendering:pixelated} | |
| .tabs{display:flex;gap:2px;margin-bottom:8px;font-size:12px} | |
| .tabs span{padding:5px 11px;border-radius:6px;color:var(--dim);cursor:pointer} | |
| .tabs span.on{background:#1e2230;color:var(--ink)} | |
| .kv1{font-size:13px;color:var(--dim);margin-top:12px}.kv1 b{color:var(--ink);font-variant-numeric:tabular-nums;font-weight:600} | |
| .bar{height:5px;background:#22262f;border-radius:3px;margin-top:4px;position:relative;overflow:hidden} | |
| .bar i{position:absolute;inset:0 auto 0 0;background:var(--acc);border-radius:3px} | |
| .bar u{position:absolute;top:-2px;bottom:-2px;width:2px;background:var(--warn)} | |
| .gate{display:flex;gap:8px;align-items:baseline;font-size:13px;padding:5px 0} | |
| .gate em{font-style:normal;width:14px;color:var(--bad)}.gate em.w{color:var(--warn)} | |
| .fail{color:var(--bad)} | |
| .note{font-size:12px;color:var(--dim);border-left:2px solid var(--line);padding-left:11px;margin:14px 0} | |
| .stages{list-style:none;padding:0;margin:0;font-variant-numeric:tabular-nums} | |
| .stages li{padding:7px 0;color:#4a5060;display:flex;gap:10px} | |
| .stages li.on{color:var(--acc)}.stages li.ok{color:var(--dim)} | |
| .three{display:grid;grid-template-columns:repeat(3,1fr);gap:10px} | |
| .three video{width:100%;border-radius:8px;background:#000;display:block} | |
| .three p{font-size:11px;color:var(--dim);margin:6px 0 0;text-transform:uppercase;letter-spacing:.05em} | |
| .warnbox{background:#2a1d10;border:1px solid #543a17;color:var(--warn);border-radius:8px;padding:11px 14px;font-size:13px;margin:14px 0} | |
| .spin{color:var(--acc);font-size:13px} | |
| #view{width:100%;aspect-ratio:16/9;display:block;background:#0e1015;border-radius:6px;margin-top:8px;cursor:grab;touch-action:none} | |
| #view.pick{cursor:crosshair} | |
| #viewhint{font-size:11px;color:var(--dim);margin-top:4px;line-height:1.5} | |
| #viewhint b{color:var(--warn)} | |
| #keys{width:100%;border-collapse:collapse;font-size:12px;font-variant-numeric:tabular-nums;margin-top:8px} | |
| #keys td{padding:4px 6px;border-top:1px solid var(--line);vertical-align:middle} | |
| #keys tr.hd td{border-top:0;color:var(--dim);font-size:11px;text-transform:uppercase;letter-spacing:.05em} | |
| #keys tr.sel td{background:#1e2a3a} | |
| #keys tr.seg td{border-top:0;padding:0 6px 6px 40px;color:var(--dim);font-size:11px} | |
| #keys tr.seg td.warn{color:var(--warn)}#keys tr.seg td.bad{color:var(--bad)} | |
| #keys input[type=number]{width:58px;background:#1e2230;color:var(--ink);border:1px solid var(--line);border-radius:5px;padding:2px 4px;font:inherit} | |
| #keys img{width:128px;height:74px;object-fit:cover;background:#000;border-radius:4px;display:block;cursor:zoom-in;image-rendering:pixelated} | |
| #keys code{font-size:11px;color:var(--dim)} | |
| #keys tr[data-k]{cursor:pointer} | |
| .mode{font-size:12px;color:var(--dim);margin-top:10px}.mode b{color:var(--acc);font-weight:600} | |
| .tpl{display:flex;gap:8px;flex-wrap:wrap;align-items:center;font-size:12px;color:var(--dim)} | |
| #moves{margin-bottom:18px;gap:10px}#moves button{padding:10px 18px;font-size:14px}#moves .hint{margin-left:auto} | |
| .hide{display:none} | |
| a{color:var(--acc)} | |
| </style> | |
| <div id="rail"></div> | |
| <main> | |
| <section id="s1"> | |
| <h1>Give it a camera it never had</h1> | |
| <p class="sub">Upload a clip. Its geometry is reconstructed at once; you then place a few camera keys in 3D | |
| — where the camera is, what it looks at, which moment of the clip it shows — and the model renders | |
| the view that was never shot.</p> | |
| <div class="pan"> | |
| <div id="drop">Drop a video here, or click to choose<br><span style="font-size:12px">mp4 / mov / webm · a few seconds is plenty</span></div> | |
| <input type="file" id="file" accept="video/*" class="hide"> | |
| <div style="margin-top:18px;font-size:12px;color:var(--dim)">or start from a sample plate</div> | |
| <div class="chips" id="samples"></div> | |
| </div> | |
| </section> | |
| <section id="s2" class="hide"> | |
| <h1>The studio</h1> | |
| <p class="sub">The left pane is exactly what the model will be conditioned on, at the resolution it will be | |
| conditioned at. Each key is one camera: a position, a point it looks at, the source frame it shows and the | |
| output frame it lands on. Between keys the camera glides; the same source frame at two keys is a freeze.</p> | |
| <div class="pan tpl" id="moves"> | |
| <span>start from a move:</span> | |
| <button data-tpl="orbit" title="swing 30° around the subject">orbit</button> | |
| <button data-tpl="push" title="close on the subject">push in</button> | |
| <button data-tpl="slide" title="track sideways, subject held centred">slide</button> | |
| <button data-tpl="crane" title="lift and lean in, subject held centred">crane</button> | |
| <button id="tplbt" title="real time up to the preview frame, then a frozen 35° orbit, then the clip resumes">freeze + orbit</button> | |
| <button id="tplsrc" title="the clip's own camera, as keys you can edit">the clip's own camera</button> | |
| <span class="hint">each replaces the keys below · Ctrl+Z undoes</span> | |
| </div> | |
| <div id="studio"> | |
| <div class="pan"> | |
| <div class="tabs" id="tabs"> | |
| <span class="on" data-k="truth">what the model sees</span> | |
| <span data-k="holes">where the pixels are missing</span> | |
| </div> | |
| <video id="pv" autoplay loop muted playsinline></video> | |
| <img id="pk" class="hide"> | |
| <label class="f">preview frame <b id="kfv">0</b> <button class="ghost sm" id="kfplay">▶</button> | |
| <span id="pknote" style="float:right;text-transform:none;letter-spacing:0"></span></label> | |
| <input type="range" id="kf" min="0" value="0"> | |
| <div class="note" id="pvnote"></div> | |
| </div> | |
| <div class="pan"> | |
| <label class="f">the window <b id="spv"></b> <span id="p2" class="spin" style="text-transform:none;letter-spacing:0"></span></label> | |
| <canvas id="tl"></canvas> | |
| <div style="font-size:11px;color:var(--dim);margin-top:4px">the strip is the whole clip: shaded = the frames reconstructed, white ticks = your keys, red = a hard cut</div> | |
| <div id="sbox" class="hide"> | |
| <label class="f">window starts at frame <b id="sv"></b> · moving it reconstructs again and resets the keys</label> | |
| <input type="range" id="sr" min="0" value="0"> | |
| </div> | |
| <label class="f">take length</label> | |
| <div class="chips" id="lens" style="margin-top:0"></div> | |
| <div class="f" style="display:flex;justify-content:space-between;align-items:center;margin-top:8px"><span id="vwmode"></span><button class="ghost sm hide" id="ovw">back to the overview</button></div> | |
| <canvas id="view"></canvas> | |
| <div id="viewhint"></div> | |
| </div> | |
| </div> | |
| <div class="pan" style="margin-top:18px"> | |
| <div class="tpl"> | |
| <span>fine-tune the keys:</span> | |
| <button id="addkey">+ key at the preview frame</button> | |
| <button class="ghost" id="clearkeys">clear the keys</button> | |
| </div> | |
| <table id="keys"></table> | |
| <div class="mode" id="modev"></div> | |
| <div id="gauges"></div> | |
| <div id="gates"></div> | |
| <div style="margin-top:16px;display:flex;gap:9px;align-items:center"> | |
| <button class="go" id="go2">Render this take</button> | |
| <button class="ghost" id="back2">← another clip</button> | |
| <span id="p4" class="spin"></span> | |
| </div> | |
| </div> | |
| </section> | |
| <section id="s3" class="hide"> | |
| <h1>Rendering</h1> | |
| <p class="sub">Three forwards of the 4-step student at the 480 class. The warp is already done; what happens now is | |
| the model filling in every pixel the original camera never saw.</p> | |
| <div class="pan" style="max-width:520px"> | |
| <ul class="stages" id="st5"></ul> | |
| <div class="bar" style="height:4px;margin-top:10px"><i id="pb" style="width:0"></i></div> | |
| <div id="e5"></div> | |
| </div> | |
| </section> | |
| <section id="s4" class="hide"> | |
| <h1>The take</h1> | |
| <div class="three"> | |
| <div><video id="v6a" autoplay loop muted playsinline></video><p>the original frame, cropped</p></div> | |
| <div><video id="v6b" autoplay loop muted playsinline style="image-rendering:pixelated"></video><p>the warp — holes and all</p></div> | |
| <div><video id="v6c" autoplay loop muted playsinline></video><p>the render</p></div> | |
| </div> | |
| <div style="display:flex;gap:9px;align-items:center;margin-top:10px"> | |
| <button class="ghost sm" id="v6play">play / pause</button><input type="range" id="tk" min="0" value="0" style="flex:1;margin:0"> | |
| </div> | |
| <div class="note" style="margin-top:16px" id="n6"></div> | |
| <div style="margin-top:16px;display:flex;gap:9px"> | |
| <a id="dl" download><button class="go">Download the take</button></a> | |
| <a id="dlg" download><button class="ghost">Download the 3-up comparison</button></a> | |
| <button class="ghost" id="again">Another move, same clip</button> | |
| <button class="ghost" id="new">New clip</button> | |
| </div> | |
| <div class="kv" style="margin-top:18px" id="k6"></div> | |
| </section> | |
| </main> | |
| <script> | |
| const RAIL = ["clip", "studio", "render", "the take"]; | |
| const FPS = 24, WIN = 124; // WIN: the most frames one reconstruction takes (joint attention); longer clips get a trim slider | |
| // the four templates are the parametric family on the server, sampled into four keys the user can then edit | |
| const TEMPLATES = { | |
| orbit: {yaw: -30, dolly: 1, truck: 0, boom: 0, aim: false}, | |
| push: {yaw: 0, dolly: 0.72, truck: 0, boom: 0, aim: false}, | |
| slide: {yaw: 0, dolly: 1, truck: 0.35, boom: 0, aim: true}, | |
| crane: {yaw: 0, dolly: 1.30, truck: 0, boom: 0.45, aim: true}, | |
| }; | |
| const st = {clip: null, n: 0, cuts: [], frames: 73, start: 0, seed: 1234, prep: null, g: null, tab: "truth", | |
| keys: [], sel: -1, fit: false, pick: -1, hist: [], last: null}; // keys: {pos, look, src, t, ease, focal} as in recam/path.py; hist: undo steps | |
| const $ = id => document.getElementById(id); | |
| const post = (u, b) => fetch(u, {method: "POST", headers: {"content-type": "application/json"}, body: JSON.stringify(b)}) | |
| .then(r => r.ok ? r.json() : r.text().then(t => { fail(`${u} ${r.status}: ${t.slice(0, 300)}`); throw new Error(t); })); | |
| function fail(t) { // a 5xx must be visible, not a silently unfulfilled promise | |
| const e = $("err") || Object.assign(document.body.appendChild(document.createElement("div")), {id: "err", className: "warnbox"}); | |
| e.style.cssText = "position:fixed;left:16px;right:16px;top:12px;z-index:9;cursor:pointer"; e.textContent = t; e.onclick = () => e.remove(); | |
| } | |
| // the window: up to WIN frames from `start`, never across a hard cut (the geometry is one joint pass over the span) | |
| const spanEnd = () => { const c = st.cuts.find(c => c > st.start); return Math.min(st.start + WIN - 1, st.n - 1, c == null ? 1e9 : c - 1); }; | |
| const base = () => ({clip: st.clip, start: st.start, span_end: spanEnd(), frames: st.frames, pivot: [0.5, 0.5], pivot_frame: st.start, seed: st.seed}); | |
| const pathPayload = () => ({...base(), path: st.keys.map(k => ({pos: k.pos, look: k.look, src: k.src, t: k.t, ease: k.ease, focal: k.focal || 1}))}); | |
| // a key's lens: `focal` multiplies the source frame's own focal length (path.py interpolates it linearly between keys); the table shows it as a horizontal fov | |
| const lens = k => st.prep.src_poses[Math.min(Math.max(k.src - st.start, 0), st.prep.src_poses.length - 1)].k; | |
| const fovOf = k => 2 * Math.atan(0.5 / (lens(k)[0] * (k.focal || 1))) * 180 / Math.PI; | |
| const focalOf = (k, fov) => +(0.5 / (Math.tan(Math.min(Math.max(fov, 10), 120) * Math.PI / 360) * lens(k)[0])).toFixed(4); | |
| const focalAt = t => { const K = st.keys, i = K.findIndex(k => k.t > t); if (i <= 0) return (i === 0 ? K[0] : K[K.length - 1])?.focal || 1; const a = K[i - 1], b = K[i]; return +((a.focal || 1) + ((b.focal || 1) - (a.focal || 1)) * (t - a.t) / (b.t - a.t)).toFixed(4); }; | |
| const sec = f => (f / FPS).toFixed(2); | |
| const piv = () => st.prep.piv; // the scene centre: the centre-of-picture depth at frame `start`, also the unit of every coordinate | |
| let screen = 0; | |
| function show(i) { | |
| screen = i; | |
| [1, 2, 3, 4].forEach(k => $("s" + k).classList.toggle("hide", k !== i + 1)); | |
| $("rail").innerHTML = RAIL.map((t, k) => `<div class="${k === i ? "on" : k < i ? "done" : ""}">${k + 1}. ${t}</div>`).join(""); | |
| window.scrollTo(0, 0); | |
| } | |
| // ---------- S1: the clip ---------------------------------------------------- | |
| fetch("/samples").then(r => r.json()).then(ss => $("samples").innerHTML = | |
| ss.map(s => `<span class="chip" data-s="${s}">${s.replace(/^sp_|\.mp4$/g, "").replace(/_/g, " ")}</span>`).join("")); | |
| $("samples").onclick = e => { const el = e.target; if (el.dataset.s) { el.textContent = "loading…"; post("/sample", {name: el.dataset.s}).then(landed).catch(() => el.textContent = "failed — try again"); } }; | |
| $("drop").onclick = () => $("file").click(); | |
| $("file").onchange = e => e.target.files[0] && send(e.target.files[0]); | |
| $("drop").ondragover = e => { e.preventDefault(); $("drop").classList.add("hot"); }; | |
| $("drop").ondragleave = () => $("drop").classList.remove("hot"); | |
| $("drop").ondrop = e => { e.preventDefault(); $("drop").classList.remove("hot"); send(e.dataTransfer.files[0]); }; | |
| function send(f) { | |
| $("drop").textContent = "normalising …"; | |
| const fd = new FormData(); fd.append("file", f); | |
| fetch("/upload", {method: "POST", body: fd}).then(r => r.ok ? r.json() : r.text().then(t => Promise.reject(`${r.status}: ${t.slice(0, 200)}`))) | |
| .then(landed).catch(t => $("drop").textContent = `upload failed (${t}) — drop another clip, or the same one again`); | |
| } | |
| function landed(c) { | |
| if (!c.lengths.length) { $("drop").textContent = `too short: ${c.frames} frames, the model needs 73 (3 s) — drop another clip`; return; } | |
| Object.assign(st, {clip: c.clip, n: c.frames, cuts: c.cuts, w: c.w, h: c.h, start: 0, g: null, sel: -1}); | |
| const wl = spanEnd() + 1; // default take length: the chip closest to the window, so a 5 s clip is a 5 s take, not its first 3 s | |
| st.frames = c.lengths.reduce((a, L) => Math.abs(L - wl) < Math.abs(a - wl) ? L : a); | |
| $("lens").innerHTML = c.lengths.map(L => `<span class="chip" data-l="${L}">${L} frames · ${(L / 24).toFixed(1)}s</span>`).join(""); | |
| $("sbox").classList.toggle("hide", st.n <= WIN && !st.cuts.length); | |
| $("sr").max = Math.max(0, st.n - 24); $("sr").value = 0; | |
| show(1); // before drawing: canvases are 0 wide while hidden | |
| reconstruct(); | |
| } | |
| // ---------- S2: the studio -------------------------------------------------- | |
| $("sr").oninput = e => { st.start = +e.target.value; winsync(); }; | |
| $("sr").onchange = () => reconstruct(); | |
| $("lens").onclick = e => { | |
| if (!e.target.dataset.l) return; | |
| const L = +e.target.dataset.l, old = st.frames, dflt = JSON.stringify(st.keys) === JSON.stringify(clipKeys()); | |
| st.frames = L; | |
| if (dflt) st.keys = clipKeys(); // the untouched default pair follows the new length to the clip's end | |
| else st.keys.forEach(k => k.t = Math.round(k.t * (L - 1) / (old - 1))); // edited keys stay where they were, proportionally | |
| winsync(); edit(); | |
| }; | |
| function winsync() { | |
| $("sv").textContent = st.start; | |
| $("spv").textContent = `frames ${st.start}–${spanEnd()} of ${st.n}`; | |
| $("kfv").textContent = $("kf").value; | |
| [...$("lens").children].forEach(c => c.style.borderColor = +c.dataset.l === st.frames ? "var(--acc)" : ""); | |
| drawTl(); | |
| } | |
| function drawTl() { | |
| const c = $("tl"), x = c.getContext("2d"); | |
| c.width = c.clientWidth * 2; c.height = 76; | |
| const W = c.width, px = i => i / Math.max(1, st.n - 1) * (W - 4) + 2; | |
| x.clearRect(0, 0, W, 76); | |
| x.fillStyle = "#171a21"; x.fillRect(2, 22, W - 4, 32); | |
| x.fillStyle = "#2b3a4a"; x.fillRect(px(st.start), 22, px(spanEnd()) - px(st.start), 32); | |
| x.fillStyle = "#fff"; st.keys.forEach(k => x.fillRect(px(k.src) - 1, 26, 2, 24)); | |
| x.fillStyle = "#f87171"; st.cuts.forEach(i => x.fillRect(px(i) - 1.5, 16, 3, 44)); | |
| x.fillStyle = "#5a6274"; x.font = "18px sans-serif"; | |
| x.fillText("0", 2, 74); x.fillText(String(st.n - 1), W - 40, 74); | |
| } | |
| addEventListener("resize", () => { if (screen === 1) { drawTl(); drawView(); } }); | |
| let pReq = 0; | |
| function reconstruct() { // the whole window in one VGGT pass, the moment the clip lands; the keys start as the clip's own camera | |
| const my = ++pReq; | |
| dirty(); still(null); $("p4").textContent = ""; | |
| st.prep = null; st.g = null; st.keys = []; st.sel = -1; st.pick = -1; st.hist = []; st.last = null; vw.clouds = {}; vw.cloud = null; vw.cframe = -1; vw.thru = false; | |
| for (const v of TH.values()) if (typeof v === "string") URL.revokeObjectURL(v); | |
| TH.clear(); | |
| winsync(); keysync(); | |
| $("p2").textContent = "reconstructing the geometry … (about 2 s)"; | |
| post("/prepare", base()).then(r => { | |
| if (my !== pReq) return; | |
| st.prep = r; $("p2").textContent = `${r.ms} ms`; | |
| $("view").style.aspectRatio = `${r.canvas[0]}/${r.canvas[1]}`; // the editor has the render's frame, so looking through a key shows the model's picture | |
| st.keys = clipKeys(); st.sel = -1; | |
| frameView(); edit(); | |
| }).catch(() => $("p2").textContent = "reconstruction failed — see the message at the top"); | |
| } | |
| $("tabs").onclick = e => { if (e.target.dataset.k) { st.tab = e.target.dataset.k; still(null); paint(); } }; | |
| $("back2").onclick = () => { show(0); }; | |
| let wReq = 0; // the path revision: every edit bumps it, and only a warp of the current revision may unlock Render | |
| function dirty() { wReq++; $("go2").disabled = true; if (st.g) $("p4").textContent = "edited — not warped yet"; } | |
| function edit() { // every change to the keys or the take length: stale the old warp, keep an undo step, leave any still, warp again | |
| const cur = JSON.stringify({keys: st.keys, frames: st.frames}); | |
| if (st.last && st.last !== cur) st.hist.push(st.last); | |
| st.last = cur; | |
| dirty(); still(null); keysync(); rewarp(); | |
| } | |
| function rewarp() { // the whole path, every time the keys change; a path that does not fit would only 500 on the server, keysync() says why | |
| if (!st.prep || !st.fit) return; | |
| const my = wReq; | |
| $("p4").textContent = "warping …"; | |
| post("/warp", pathPayload()).then(g => { | |
| if (my !== wReq) return; | |
| st.g = g; $("p4").textContent = `ready · ${g.ms} ms`; | |
| $("kf").max = g.frames - 1; | |
| paint(); gauges(); drawView(); | |
| }).catch(() => { $("p4").innerHTML = '<span style="color:var(--bad)">warp failed — see the message at the top</span>'; }); | |
| } | |
| // ---------- keys -------------------------------------------------------------- | |
| const srcCam = f => st.prep.src_poses[Math.min(Math.max(f - st.start, 0), st.prep.src_poses.length - 1)]; // the source camera at source frame f | |
| const srcKey = (f, t) => { const c = srcCam(f); return {pos: c.pos.slice(), look: c.look.slice(), src: f, t, ease: false}; }; | |
| const clipKeys = () => [srcKey(st.start, 0), srcKey(Math.min(st.start + st.frames - 1, spanEnd()), st.frames - 1)]; // the clip's own camera, in real time | |
| function orbitOf(p, deg) { // p swung around the vertical axis through the scene centre | |
| const c = piv(), a = deg * Math.PI / 180, dx = p[0] - c[0], dz = p[2] - c[2]; | |
| return [c[0] + dx * Math.cos(a) + dz * Math.sin(a), p[1], c[2] - dx * Math.sin(a) + dz * Math.cos(a)]; | |
| } | |
| function setKeys(K) { st.keys = K; st.sel = -1; edit(); } | |
| $("addkey").onclick = () => { // a key at the preview frame, holding whatever the path does there (so adding one changes nothing until you move it) | |
| if (!st.prep) return; | |
| let t = +$("kf").value; const K = st.keys; | |
| const at = K.findIndex(k => k.t === t); | |
| if (at >= 0) { // clicking a key parks the preview on it, so "add" there would be a no-op: add halfway to the next key instead (the previous one at the end) | |
| const nb = K[at + 1] || K[at - 1]; | |
| t = nb ? Math.round((t + nb.t) / 2) : t ? 0 : st.frames - 1; | |
| if (K.some(k => k.t === t)) { $("modev").innerHTML = `<span style="color:var(--bad)">no room for a key next to key ${at + 1} — move the preview frame</span>`; return; } | |
| } | |
| let k; | |
| if (st.g && st.fit && st.g.frames === st.frames) { const c = st.g.cams[t]; k = {pos: c.pos.slice(), look: c.look.slice(), src: st.g.tmap[t], t, ease: false, focal: focalAt(t)}; } | |
| else k = srcKey(Math.min(st.start + t, spanEnd()), t); | |
| K.push(k); K.sort((a, b) => a.t - b.t); | |
| st.sel = K.indexOf(k); seek(t); edit(); | |
| }; | |
| $("moves").onclick = e => { if (e.target.dataset.tpl) template(e.target.dataset.tpl); }; | |
| function template(name) { // the parametric move on the server, sampled at four instants into four keys | |
| if (!st.prep) return; | |
| const p = TEMPLATES[name], tf = Math.min(st.frames, spanEnd() - st.start + 1); // the family plays the clip in real time, so it cannot outrun the window | |
| dirty(); const my = wReq; | |
| $("p4").textContent = "warping the template …"; | |
| post("/warp", {...base(), frames: tf, shape: "sweep_ease", freeze_frame: st.start, freeze_n: 0, | |
| yaw: p.yaw, dolly: p.dolly, truck: p.truck, boom: p.boom, aim: p.aim, lite: true}).then(g => { | |
| if (my !== wReq) return; // the window changed, or the keys were edited, while the template was in flight | |
| setKeys([0, 1 / 3, 2 / 3, 1].map(u => { | |
| const i = Math.round(u * (tf - 1)), c = g.cams[i]; | |
| return {pos: c.pos.slice(), look: c.look.slice(), src: g.tmap[i], t: Math.round(u * (st.frames - 1)), ease: false}; | |
| })); | |
| frameView(); drawView(); | |
| }).catch(() => { $("p4").innerHTML = '<span style="color:var(--bad)">the template failed — see the message at the top</span>'; }); | |
| } | |
| $("tplbt").onclick = () => { // real time up to the preview frame's source frame, a 1 s frozen orbit of 35 deg, then real time again | |
| if (!st.prep) return; | |
| const hi = spanEnd(), s = Math.min(Math.max(st.g ? st.g.tmap[+$("kf").value] : st.start, st.start), st.start + st.frames - 26, hi); | |
| const t1 = s - st.start, t2 = t1 + 23, K = []; | |
| if (t1 > 0) K.push(srcKey(st.start, 0)); | |
| const h = srcKey(s, t1); h.ease = true; K.push(h); | |
| const o = srcKey(s, t2); o.pos = orbitOf(o.pos, 35); o.look = piv().slice(); K.push(o); | |
| K.push(srcKey(Math.min(hi, s + st.frames - 1 - t2), st.frames - 1)); | |
| setKeys(K); frameView(); drawView(); | |
| }; | |
| $("tplsrc").onclick = () => { if (st.prep) setKeys(clipKeys()); }; | |
| $("clearkeys").onclick = () => setKeys([]); | |
| $("keys").onchange = e => { // change, not input: keysync() re-renders the table, which would steal the focus mid-typing | |
| const k = st.keys[+e.target.dataset.i], f = e.target.dataset.f; | |
| if (!k) return; | |
| const v = Math.round(+e.target.value); | |
| if (f === "src") k.src = Math.min(Math.max(v, st.start), spanEnd()); | |
| else if (f === "t") k.t = Math.min(Math.max(v, 1), st.frames - 2); | |
| else if (f === "fov") k.focal = focalOf(k, +e.target.value); | |
| e.target.blur(); | |
| edit(); | |
| }; | |
| $("keys").onclick = e => { | |
| const d = e.target.dataset; | |
| if (d.del != null) { st.keys.splice(+d.del, 1); st.sel = -1; edit(); return; } | |
| if (d.pick != null) { st.pick = +d.pick; st.sel = st.pick; keysync(); return; } | |
| if (d.ctr != null) { st.keys[+d.ctr].look = piv().slice(); edit(); return; } | |
| if (e.target.tagName === "IMG") { st.sel = +e.target.closest("tr").dataset.k; vw.thru = true; still(st.sel); keysync(); return; } | |
| if (e.target.tagName === "INPUT" || e.target.tagName === "LABEL") return; | |
| const tr = e.target.closest("tr[data-k]"); | |
| if (tr) { st.sel = +tr.dataset.k; vw.thru = true; seek(st.keys[st.sel].t); keysync(); } | |
| }; | |
| $("ovw").onclick = () => { vw.thru = false; keysync(); }; | |
| function keysync() { | |
| const K = st.keys, lo = st.start, hi = spanEnd(), n = st.frames; | |
| K.forEach((k, i) => { if (i === 0) k.t = 0; if (i === K.length - 1) k.t = n - 1; }); // the first and last key frame the take | |
| const order = K.every((k, i) => i === 0 || k.t > K[i - 1].t), fwd = K.every((k, i) => i === 0 || k.src >= K[i - 1].src); | |
| const ok = K.length >= 2 && order && fwd; | |
| st.fit = ok; | |
| let rows = ""; | |
| K.forEach((k, i) => { | |
| const th = JSON.stringify([k.src, k.pos, k.look, k.focal || 1]), u = TH.get(th); | |
| const when = i === 0 ? `0 <small>(the start)</small>` : i === K.length - 1 ? `${n - 1} <small>(the end)</small>` | |
| : `<input type="number" data-i="${i}" data-f="t" value="${k.t}" min="1" max="${n - 2}" step="1">`; | |
| rows += `<tr data-k="${i}" class="${i === st.sel ? "sel" : ""}"><td>${i + 1}</td> | |
| <td><img data-th='${th}' ${typeof u === "string" ? `src="${u}"` : ""} title="what the model is given at this key — click to see it large"></td> | |
| <td><input type="number" data-i="${i}" data-f="src" value="${k.src}" min="${lo}" max="${hi}" step="1"> <small>${sec(k.src)} s</small></td> | |
| <td>${when} <small>${sec(k.t)} s</small></td> | |
| <td><button class="ghost sm" data-pick="${i}" title="then click a point of the scene in the 3D view">${st.pick === i ? "click the scene …" : "pick in 3D"}</button> | |
| <button class="ghost sm" data-ctr="${i}" title="the centre of the picture at the window's first frame">centre</button></td> | |
| <td><input type="number" data-i="${i}" data-f="fov" value="${fovOf(k).toFixed(0)}" min="10" max="120" step="1" title="horizontal field of view; ${k.focal && k.focal !== 1 ? `${k.focal.toFixed(2)}× the clip's own lens` : "the clip's own lens"}"> <small>fov°</small></td> | |
| <td><button class="ghost sm" data-del="${i}">✕</button></td></tr>`; | |
| if (i < K.length - 1 && order) { | |
| const nf = K[i + 1].t - k.t, ds = K[i + 1].src - k.src, v = ds / nf; | |
| const [cls, txt] = ds < 0 ? ["bad", "clip time runs backwards — a later key must show a later (or the same) frame"] | |
| : v === 0 ? ["", `frozen on frame ${k.src} for ${sec(nf)} s (bullet time)`] | |
| : Math.abs(v - 1) < 0.05 ? ["", `real time for ${sec(nf)} s`] // a take a few frames longer than the clip holds its last frame: still real time | |
| : ["warn", `${sec(nf)} s, the clip plays at ${v.toFixed(2)}× — the model was trained on real time, holds and reverses only`]; | |
| rows += `<tr class="seg"><td colspan="7" class="${cls}">↓ ${txt}</td></tr>`; | |
| } | |
| }); | |
| $("keys").innerHTML = rows && `<tr class="hd"><td></td><td>what the model sees</td><td>clip frame it shows</td><td>output frame it lands on</td><td>aims at</td><td>lens</td><td></td></tr>` + rows; | |
| $("modev").innerHTML = !st.prep ? "" : K.length < 2 ? `<span style="color:var(--bad)">a path needs two or more keys — add one at the preview frame, or start from a move above</span>` | |
| : !order ? `<span style="color:var(--bad)">the keys' output frames must increase from key 1 to the last — fix the one that is out of order</span>` | |
| : !fwd ? `<span style="color:var(--bad)">clip time runs backwards between two keys — a later key must show a later (or the same) frame</span>` | |
| : `<b>${K.length} keys</b>, a ${sec(n - 1)} s take from ${spanEnd() - lo + 1} reconstructed frames`; | |
| if (!ok) $("go2").disabled = true; | |
| $("view").classList.toggle("pick", st.pick >= 0); | |
| const T = thru(); | |
| $("vwmode").textContent = !st.prep ? "" : T ? `through key ${st.sel + 1}'s camera, clip frame ${K[st.sel].src} with that frame's own lens: the picture the model is given there` | |
| : "overview: every key, the source camera and the path"; | |
| $("ovw").classList.toggle("hide", !T); | |
| $("viewhint").innerHTML = st.pick >= 0 ? `<b>click a point of the scene to aim key ${st.pick + 1} at it</b> · Esc cancels` | |
| : T ? "drag: aim the camera · shift+drag: move it sideways / up · wheel: forward / back · double-click a point: aim at it" | |
| : "drag: turn the view · shift+drag: slide it · wheel: zoom · drag a key (white): move it · drag its ring: aim it · click a key's row to look through it · double-click a point: aim the selected key at it"; | |
| if (st.prep) drawTl(); | |
| thumbs(); loadCloud(); drawView(); | |
| } | |
| addEventListener("keydown", e => { | |
| if (e.key === "Escape" && st.pick >= 0) { st.pick = -1; keysync(); } | |
| if ((e.ctrlKey || e.metaKey) && e.key === "z" && screen === 1 && st.hist.length && e.target.tagName !== "INPUT") { // undo: back to the keys before the last edit | |
| e.preventDefault(); | |
| const h = JSON.parse(st.hist.pop()); st.last = null; | |
| st.keys = h.keys; st.frames = h.frames; st.sel = -1; | |
| winsync(); edit(); | |
| } | |
| }); | |
| // the per-key still: the source frame warped to the key's camera (/warp1), cached by (src, pos, look) | |
| const TH = new Map(); | |
| function thumbs() { | |
| if (vw.drag || !st.prep) return; | |
| const my = pReq; // a still from an earlier window must not land in this window's cache | |
| $("keys").querySelectorAll("img[data-th]").forEach(img => { | |
| const th = img.dataset.th; | |
| if (TH.has(th)) return; | |
| const [src, pos, look, focal] = JSON.parse(th); | |
| TH.set(th, fetch("/warp1", {method: "POST", headers: {"content-type": "application/json"}, body: JSON.stringify({...base(), src, pos, look, focal})}) | |
| .then(r => r.ok ? r.blob() : r.text().then(t => Promise.reject(fail(`/warp1 ${r.status}: ${t.slice(0, 200)}`)))) | |
| .then(b => { | |
| if (my !== pReq) return; | |
| const u = URL.createObjectURL(b); | |
| TH.set(th, u); | |
| $("keys").querySelectorAll("img[data-th]").forEach(im => { if (im.dataset.th === th) im.src = u; }); | |
| if (TH.size > 80) { const [k0, v0] = TH.entries().next().value; if (typeof v0 === "string") URL.revokeObjectURL(v0); TH.delete(k0); } | |
| })); | |
| }); | |
| } | |
| function still(i) { // i: show key i's still in the left pane instead of the video; null: back to the video | |
| const k = st.keys[i], u = k && TH.get(JSON.stringify([k.src, k.pos, k.look, k.focal || 1])); | |
| const on = typeof u === "string"; | |
| if (on) $("pk").src = u; | |
| $("pk").classList.toggle("hide", !on); $("pv").classList.toggle("hide", on); | |
| $("pknote").textContent = on ? `key ${i + 1}: clip frame ${k.src} from its camera — the still the model is given at output frame ${k.t}` : ""; | |
| } | |
| function seek(i) { $("pv").pause(); $("pv").currentTime = (i + 0.5) / FPS; $("kf").value = i; $("kfv").textContent = i; } | |
| $("kf").oninput = e => { still(null); seek(+e.target.value); drawView(); }; | |
| $("kfplay").onclick = () => { still(null); $("pv").play(); }; | |
| $("pv").ontimeupdate = () => { if (!$("pv").paused) { $("kf").value = Math.floor($("pv").currentTime * FPS); $("kfv").textContent = $("kf").value; drawView(); } }; | |
| // ---------- the 3D editor: a software-projected point cloud, keys you can drag and aim ---------- | |
| // Everything is in the path's frame (see recam/path.py): x right, y down, z forward, units of the scene-centre depth. | |
| const vw = {az: 0.6, el: 0.45, dist: 2.6, ctr: null, thru: false, cloud: null, cframe: -1, clouds: {}, drag: null, zbuf: null, ibuf: null, img: null}; | |
| const thru = () => vw.thru && st.sel >= 0 && !!st.keys[st.sel]; // the viewer is the selected key's camera | |
| function loadCloud() { // the selected key's source frame (or the window's first): the scene at that instant, one frame's cloud | |
| if (!st.prep) return; | |
| const f = st.sel >= 0 && st.keys[st.sel] ? st.keys[st.sel].src : st.start; | |
| if (vw.cframe === f) return; | |
| vw.cframe = f; vw.cloud = vw.clouds[f] || null; | |
| if (vw.cloud) return; | |
| const my = pReq; | |
| post("/cloud", {...base(), frame: f, stride: 2}) | |
| .then(c => { if (my !== pReq) return; vw.clouds[f] = {pts: new Float32Array(c.pts), rgb: new Uint8Array(c.rgb), n: c.n}; if (vw.cframe === f) { vw.cloud = vw.clouds[f]; drawView(); } }); | |
| } | |
| function frameView() { // pull the viewer back just far enough that the source camera and every key fit | |
| const c = piv(), pts = [...st.prep.src_poses.map(p => p.pos), ...st.keys.map(k => k.pos)]; | |
| vw.dist = Math.max(1.5, 1.9 * Math.max(...pts.map(p => Math.hypot(p[0] - c[0], p[1] - c[1], p[2] - c[2])))); | |
| vw.ctr = c.slice(); | |
| } | |
| function viewCam() { // the viewer's camera and the pixel lens it projects with (fx, fy, cx, cy): through a key it IS that key's camera with the | |
| // source frame's own lens, i.e. what /warp1 renders; otherwise it orbits vw.ctr. Returns eye and the (right, down, forward) basis. | |
| const W = $("view").width, H = $("view").height; | |
| const nrm = v => { const n = Math.hypot(...v) || 1; return v.map(x => x / n); }; | |
| const cross = (a, b) => [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]]; | |
| let eye, f, fx, fy, cx, cy; | |
| if (thru()) { | |
| const k = st.keys[st.sel], L = st.prep.src_poses[k.src - st.start].k, m = k.focal || 1; | |
| eye = k.pos; f = nrm([k.look[0] - k.pos[0], k.look[1] - k.pos[1], k.look[2] - k.pos[2]]); | |
| fx = L[0] * W * m; fy = L[1] * H * m; cx = L[2] * W; cy = L[3] * H; | |
| } else { | |
| const ctr = vw.ctr, [ca, sa, ce, se] = [Math.cos(vw.az), Math.sin(vw.az), Math.cos(vw.el), Math.sin(vw.el)]; | |
| eye = [ctr[0] + vw.dist * sa * ce, ctr[1] - vw.dist * se, ctr[2] - vw.dist * ca * ce]; | |
| f = nrm([ctr[0] - eye[0], ctr[1] - eye[1], ctr[2] - eye[2]]); | |
| fx = fy = H * 0.9; cx = W / 2; cy = H / 2; | |
| } | |
| let r = cross(f, [0, -1, 0]); | |
| r = Math.hypot(...r) < 1e-6 ? [1, 0, 0] : nrm(r); // straight up or down: keep x as right, like path.py | |
| return {eye, r, d: cross(f, r), f, fx, fy, cx, cy}; | |
| } | |
| function project(c, X) { // -> [sx, sy, depth] or null when behind the viewer | |
| const q = [X[0] - c.eye[0], X[1] - c.eye[1], X[2] - c.eye[2]]; | |
| const z = q[0] * c.f[0] + q[1] * c.f[1] + q[2] * c.f[2]; | |
| if (z < 0.05) return null; | |
| return [c.cx + c.fx * (q[0] * c.r[0] + q[1] * c.r[1] + q[2] * c.r[2]) / z, c.cy + c.fy * (q[0] * c.d[0] + q[1] * c.d[1] + q[2] * c.d[2]) / z, z]; | |
| } | |
| function drawView() { | |
| const cv = $("view"), g = st.g; | |
| if (!st.prep || !cv.clientWidth) return; | |
| const W = cv.clientWidth * 2, H = cv.clientHeight * 2; | |
| if (cv.width !== W || cv.height !== H) { cv.width = W; cv.height = H; vw.img = null; } | |
| const x = cv.getContext("2d"), c = viewCam(), T = thru(), sp = T ? 3 : 1; // through a key the points are spread over the whole frame: fatter splats | |
| const P = X => project(c, X); | |
| // 1. the cloud, splatted 3x3 with a z-buffer; ibuf remembers which point owns each pixel (for aiming at a point) | |
| if (!vw.img) { vw.img = x.createImageData(W, H); vw.zbuf = new Float32Array(W * H); vw.ibuf = new Int32Array(W * H); } | |
| const px = vw.img.data; px.fill(0); vw.zbuf.fill(1e9); vw.ibuf.fill(-1); | |
| for (let i = 3; i < px.length; i += 4) px[i] = 255; | |
| if (vw.cloud) { | |
| const {pts, rgb, n} = vw.cloud; | |
| for (let i = 0; i < n; i++) { | |
| const s = P([pts[3 * i], pts[3 * i + 1], pts[3 * i + 2]]); | |
| if (!s) continue; | |
| const sx = Math.round(s[0]), sy = Math.round(s[1]); | |
| for (let dy = -sp; dy <= sp; dy++) for (let dx = -sp; dx <= sp; dx++) { | |
| const X = sx + dx, Y = sy + dy; | |
| if (X < 0 || Y < 0 || X >= W || Y >= H) continue; | |
| const o = Y * W + X; | |
| if (s[2] >= vw.zbuf[o]) continue; | |
| vw.zbuf[o] = s[2]; vw.ibuf[o] = i; | |
| px[4 * o] = rgb[3 * i]; px[4 * o + 1] = rgb[3 * i + 1]; px[4 * o + 2] = rgb[3 * i + 2]; | |
| } | |
| } | |
| } | |
| x.putImageData(vw.img, 0, 0); | |
| x.fillStyle = "#666"; x.font = "22px sans-serif"; | |
| x.fillText(vw.cloud ? `${T ? `key ${st.sel + 1} sees` : "scene at"} clip frame ${vw.cframe}` : "loading the point cloud …", 20, 40); | |
| // 2. the vector layer: source camera grey, the path blue with a tick every 12 frames, scene centre yellow, keys white | |
| const line = (pts, col, w) => { x.strokeStyle = col; x.lineWidth = w; x.beginPath(); let up = true; | |
| pts.forEach(p => { const s = P(p); if (!s) { up = true; return; } up ? x.moveTo(s[0], s[1]) : x.lineTo(s[0], s[1]); up = false; }); x.stroke(); }; | |
| const dot = (p, col, r, hollow) => { const s = P(p); if (!s) return; x.beginPath(); x.arc(s[0], s[1], r, 0, 7); | |
| if (hollow) { x.strokeStyle = col; x.lineWidth = 3; x.stroke(); } else { x.fillStyle = col; x.fill(); } return s; }; | |
| const srcs = st.prep.src_poses.map(p => p.pos); | |
| line(srcs, "#6b7280", 2); | |
| const s0 = P(srcs[0]); if (s0) { x.fillStyle = "#9ca3af"; x.font = "18px sans-serif"; x.fillText("source camera", s0[0] + 10, s0[1] + 24); } | |
| if (g && st.fit) { | |
| const cams = g.cams.map(k => k.pos); | |
| line(cams, "#7dd3fc", 3); | |
| cams.forEach((p, i) => { if (i % 12 === 0) dot(p, "#7dd3fc", 3); }); | |
| const fi = Math.min(+$("kf").value, cams.length - 1); | |
| dot(cams[fi], "#7dd3fc", 8); line([cams[fi], g.cams[fi].look], "#7dd3fc", 1.5); | |
| } | |
| dot(piv(), "#fbbf24", 8); | |
| st.keys.forEach((k, i) => { | |
| const col = i === st.sel ? "#fb923c" : "#fff"; | |
| dot(k.look, col, 7, true); | |
| if (T && i === st.sel) return; // the eye itself: only its aim ring is drawn | |
| line([k.pos, k.look], col, i === st.sel ? 2 : 1); | |
| frustum(x, P, k.pos, k.look, col); | |
| const s = dot(k.pos, col, 9); | |
| if (s) { x.fillStyle = col; x.font = "bold 20px sans-serif"; x.fillText(i + 1, s[0] + 12, s[1] - 8); } | |
| }); | |
| } | |
| function frustum(x, P, pos, look, col) { // a little wire pyramid showing where the key's camera points | |
| const f = [look[0] - pos[0], look[1] - pos[1], look[2] - pos[2]], n = Math.hypot(...f) || 1; | |
| const fz = f.map(v => v / n * 0.22); | |
| const r0 = [fz[2], 0, -fz[0]], rn = Math.hypot(...r0) || 1, r = r0.map(v => v / rn * 0.13); | |
| const d0 = [fz[1] * r[2] - fz[2] * r[1], fz[2] * r[0] - fz[0] * r[2], fz[0] * r[1] - fz[1] * r[0]], dn = Math.hypot(...d0) || 1, d = d0.map(v => v / dn * 0.08); | |
| const corner = (a, b) => [pos[0] + fz[0] + a * r[0] + b * d[0], pos[1] + fz[1] + a * r[1] + b * d[1], pos[2] + fz[2] + a * r[2] + b * d[2]]; | |
| const cs = [corner(1, 1), corner(-1, 1), corner(-1, -1), corner(1, -1)].map(P), o = P(pos); | |
| if (!o || cs.some(s => !s)) return; | |
| x.strokeStyle = col; x.lineWidth = 1.5; x.beginPath(); | |
| cs.forEach((s, i) => { x.moveTo(o[0], o[1]); x.lineTo(s[0], s[1]); x.lineTo(cs[(i + 1) % 4][0], cs[(i + 1) % 4][1]); }); | |
| x.stroke(); | |
| } | |
| const mouse = e => { const R = $("view").getBoundingClientRect(); return [(e.clientX - R.left) * 2, (e.clientY - R.top) * 2]; }; | |
| function cloudPoint(mx, my) { // the cloud point under the pointer (within 8 px), as [x, y, z], or null | |
| if (!vw.cloud) return null; | |
| const cv = $("view"); mx = Math.round(mx); my = Math.round(my); | |
| let best = -1, bd = 1e9; | |
| for (let dy = -8; dy <= 8; dy++) for (let dx = -8; dx <= 8; dx++) { | |
| const X = mx + dx, Y = my + dy; | |
| if (X < 0 || Y < 0 || X >= cv.width || Y >= cv.height) continue; | |
| const i = vw.ibuf[Y * cv.width + X]; | |
| if (i >= 0 && dx * dx + dy * dy < bd) { bd = dx * dx + dy * dy; best = i; } | |
| } | |
| if (best < 0) return null; | |
| const p = vw.cloud.pts; | |
| return [p[3 * best], p[3 * best + 1], p[3 * best + 2]]; | |
| } | |
| $("view").onpointerdown = e => { // overview: a key handle under the pointer starts a drag, anything else turns (shift: slides) the view. | |
| // Through a key: a drag aims that camera, shift+drag moves it -- the scene follows the mouse, the camera goes the other way | |
| if (!st.prep) return; | |
| const cv = $("view"), [mx, my] = mouse(e), c = viewCam(), T = thru(); | |
| let hit = null; | |
| if (st.pick < 0 && !T) st.keys.forEach((k, i) => { | |
| for (const [f, p] of [["look", k.look], ["pos", k.pos]]) { // a key body beats a look ring: the rings crowd the centre of the view | |
| if (f === "look" && hit && hit.f === "pos") continue; | |
| const s = project(c, p); | |
| if (s && Math.hypot(s[0] - mx, s[1] - my) < (f === "pos" ? 22 : 14)) hit = {i, f, z: s[2]}; | |
| } | |
| }); | |
| const zb = vw.zbuf && vw.zbuf[Math.round(my) * cv.width + Math.round(mx)], z = zb < 1e9 ? zb : T ? 1 : vw.dist; // the depth under the pointer: a slide moves the scene there by exactly the mouse motion | |
| vw.drag = hit ? {...hit, c, x: mx, y: my, moved: false, edit: true} : {f: e.shiftKey ? "pan" : T ? "aim" : "orbit", c, z, x: mx, y: my, moved: false, edit: T}; | |
| if (hit) { st.sel = hit.i; keysync(); } | |
| cv.setPointerCapture(e.pointerId); | |
| }; | |
| $("view").onpointermove = e => { | |
| const D = vw.drag; | |
| if (!D) return; | |
| const [mx, my] = mouse(e), dx = mx - D.x, dy = my - D.y; | |
| D.x = mx; D.y = my; D.moved = D.moved || Math.abs(dx) + Math.abs(dy) > 0; | |
| if (D.f === "orbit") { vw.az += dx * 0.005; vw.el = Math.max(-1.45, Math.min(1.45, vw.el + dy * 0.005)); drawView(); return; } | |
| const c = D.edit ? viewCam() : D.c, k = st.keys[st.sel]; // through a key the camera itself moves, so its basis is re-read every step | |
| const step = (z, j) => (dx * z / c.fx) * c.r[j] + (dy * z / c.fy) * c.d[j]; // mouse motion as a world offset at depth z | |
| if (D.f === "pan" && !D.edit) { for (let j = 0; j < 3; j++) vw.ctr[j] -= step(D.z, j); drawView(); return; } | |
| dirty(); // from here on the drag edits a key: the old warp and its green light are already wrong | |
| if (D.f === "pan") { for (let j = 0; j < 3; j++) { k.pos[j] = +(k.pos[j] - step(D.z, j)).toFixed(4); k.look[j] = +(k.look[j] - step(D.z, j)).toFixed(4); } drawView(); return; } | |
| if (D.f === "aim") { const z = Math.hypot(k.look[0] - k.pos[0], k.look[1] - k.pos[1], k.look[2] - k.pos[2]); for (let j = 0; j < 3; j++) k.look[j] = +(k.look[j] - step(z, j)).toFixed(4); drawView(); return; } | |
| const p = D.f === "pos" ? st.keys[D.i].pos : st.keys[D.i].look; // a handle: move in the view plane, at the handle's depth | |
| for (let j = 0; j < 3; j++) p[j] = +(p[j] + step(D.z, j)).toFixed(4); | |
| drawView(); | |
| }; | |
| $("view").onpointerup = e => { | |
| const D = vw.drag; vw.drag = null; | |
| if (!D) return; | |
| if (!D.moved && st.pick >= 0 && st.keys[st.pick]) { // pick mode: a click aims the key at the point under the pointer | |
| const p = cloudPoint(...mouse(e)); | |
| if (!p) return; | |
| st.keys[st.pick].look = p; st.pick = -1; | |
| edit(); return; | |
| } | |
| if (D.edit && D.moved) edit(); | |
| }; | |
| $("view").onwheel = e => { | |
| e.preventDefault(); | |
| if (!thru()) { vw.dist *= Math.exp(e.deltaY * 0.0015); vw.dist = Math.max(0.3, Math.min(30, vw.dist)); drawView(); return; } | |
| const k = st.keys[st.sel], c = viewCam(), s = -e.deltaY * 0.001; // dolly the key's camera along its own axis; the path re-warps once the wheel rests | |
| for (let j = 0; j < 3; j++) { k.pos[j] = +(k.pos[j] + s * c.f[j]).toFixed(4); k.look[j] = +(k.look[j] + s * c.f[j]).toFixed(4); } | |
| dirty(); drawView(); | |
| clearTimeout(vw.wt); vw.wt = setTimeout(edit, 400); | |
| }; | |
| $("view").ondblclick = e => { // aim the selected key at the cloud point under the pointer | |
| const k = st.keys[st.sel], p = cloudPoint(...mouse(e)); | |
| if (!k || !p) return; | |
| k.look = p; | |
| edit(); | |
| }; | |
| function paint() { | |
| if (!st.g) return; | |
| [...$("tabs").children].forEach(s => s.classList.toggle("on", s.dataset.k === st.tab)); | |
| const v = $("pv"), src = st.g[st.tab]; | |
| if (v.dataset.src !== src) { v.dataset.src = src; v.src = src; } | |
| const cc = st.g.cond_canvas, cv = st.g.canvas; | |
| $("pvnote").innerHTML = st.tab === "truth" | |
| ? `Shown at ${cc[0]}×${cc[1]}, pixel for pixel, because that is the resolution the model is conditioned at | |
| — not the ${cv[0]}×${cv[1]} it outputs. Judging a move on a crisp full-canvas warp is judging something | |
| the model never receives.` | |
| : `Magenta is disocclusion: no source pixel exists behind it at any frame. ${(100 * (1 - st.g.coverage)).toFixed(0)}% | |
| of this take is magenta and the model will invent all of it. That fraction is deliberately <em>not</em> a gate | |
| — clean takes at 54% exist, and where the holes fall matters far more than how many.`; | |
| } | |
| function gauges() { | |
| // `moved` (server, per frame) is translation only; a key that merely turns away from its source frame is a move too | |
| const fwd = (p, l) => l.map((x, i) => x - p[i]), ang = (a, b) => Math.acos(Math.min(1, a.reduce((s, x, i) => s + x * b[i], 0) / (Math.hypot(...a) * Math.hypot(...b) + 1e-9))) * 180 / Math.PI; | |
| const turned = Math.max(0, ...st.keys.map(k => { const s = st.prep.src_poses[k.src - st.start]; return ang(fwd(k.pos, k.look), fwd(s.pos, s.look)); })); | |
| const zoomed = Math.max(0, ...st.keys.map(k => Math.abs((k.focal || 1) - 1))); | |
| const g = st.g; | |
| $("gauges").innerHTML = `<div class="kv1">clearance ahead <b>${g.ahead.toFixed(3)}</b> · parallax <b>${(100 * g.moved).toFixed(1)}%</b> of the scene-centre depth | |
| · turned <b>${turned.toFixed(1)}°</b>${zoomed > 0.01 ? ` · zoomed <b>${(100 * zoomed).toFixed(0)}%</b>` : ""} | |
| · invented pixels <b>${(100 * (1 - g.coverage)).toFixed(0)}%</b> <small>(not a gate)</small></div>`; | |
| const gates = [ | |
| [g.ahead >= -0.1, "the camera stays outside the scene", `clearance ${g.ahead.toFixed(3)} is below the −0.1 line — the subject would pass through the lens. This is the only gauge that catches a fly-through; near, behind and collide are all blind to it.`], | |
| [g.moved > 0.004 || turned > 0.5 || zoomed > 0.01, "the camera actually moves", "every key sits on its source camera, aims where it aimed and keeps its lens: the render would just reproduce the source. Drag a key (or shift+drag / wheel while looking through it), aim it elsewhere, or change its fov."], | |
| ]; | |
| const bad = gates.filter(([ok]) => !ok); // a gate that passes says nothing worth a line | |
| $("gates").innerHTML = bad.map(([, t, why]) => `<div class="gate fail"><em>✗</em><span>${t}<br>${why}</span></div>`).join("") | |
| + (zoomed > 0.01 && g.moved <= 0.004 ? `<div class="gate"><em class="w">!</em><span>a zoom with no parallax tends to come back as the source — add a dolly</span></div>` : ""); | |
| $("go2").disabled = !st.fit || bad.length > 0; | |
| } | |
| // ---------- S3 / S4: render, take ------------------------------------------- | |
| const STAGES = [["warp", "warping the trajectory"], ["encode", "encoding the conditioning"], | |
| ["forward", "denoising — 3 forwards"], ["decode", "decoding to pixels"], ["write", "writing the take"], ["done", "done"]]; | |
| $("go2").onclick = () => { // locked at once: a second click must not start a second render | |
| $("go2").disabled = true; $("p4").textContent = "submitting …"; | |
| post("/render", pathPayload()).then(r => { st.job = r.job; show(2); $("e5").innerHTML = ""; poll(); }) | |
| .catch(() => { $("go2").disabled = false; $("p4").textContent = ""; }); | |
| }; | |
| const backToStudio = () => { show(1); winsync(); keysync(); if (st.g) { gauges(); $("p4").textContent = "ready"; } }; // the keys and their warp are still there | |
| function poll(miss = 0) { | |
| fetch("/job/" + st.job).then(r => r.ok ? r.json() : Promise.reject(r.status)).then(j => { | |
| const cur = STAGES.findIndex(([k]) => (j.stage || "").startsWith(k)); | |
| $("st5").innerHTML = STAGES.map(([k, t], i) => | |
| `<li class="${i === cur ? "on" : i < cur ? "ok" : ""}">${i < cur ? "✓" : i === cur ? "▸" : "·"} | |
| <span>${i === cur && j.stage.startsWith("forward") ? j.stage : t}</span></li>`).join(""); | |
| $("pb").style.width = (j.pct || 0) + "%"; | |
| if (j.error) { $("e5").innerHTML = `<div class="warnbox">${j.error}</div><button class="ghost" id="back3">← back to the studio, keys kept</button>`; return; } | |
| if (j.done) return take(j); | |
| setTimeout(poll, 900); | |
| }).catch(e => { | |
| if (miss < 5) return setTimeout(() => poll(miss + 1), 2000); // the page times out while the GPU is busy: keep asking for ten seconds before giving up | |
| $("e5").innerHTML = `<div class="warnbox">lost contact with the job (${e}) — the render itself keeps running on the GPU</div> | |
| <button id="repoll">ask again</button> <button class="ghost" id="back3">← back to the studio, keys kept</button>`; | |
| }); | |
| } | |
| $("e5").onclick = e => { if (e.target.id === "repoll") { $("e5").innerHTML = ""; poll(); } else if (e.target.id === "back3") backToStudio(); }; | |
| function take(j) { | |
| show(3); | |
| $("v6a").src = `/take/${st.job}/source.mp4`; | |
| $("v6b").src = `/take/${st.job}/render.mp4`; | |
| $("v6c").src = `/take/${st.job}/out.mp4`; | |
| $("dl").href = `/take/${st.job}/out.mp4`; $("dlg").href = `/take/${st.job}/grid.mp4`; | |
| const g = j.gauges || {}; | |
| $("tk").max = (g.frames || st.frames) - 1; $("tk").value = 0; | |
| $("n6").textContent = `${(100 * (1 - (g.coverage || 0))).toFixed(0)}% of the middle pane has no source pixel behind it: the model invented that region. ` | |
| + "Consistent from frame to frame is not the same as right — nothing here checks the invented part against the truth."; | |
| $("k6").innerHTML = Object.entries({ | |
| "frames": g.frames, "scene-centre depth": (g.zm || 0).toFixed(2), | |
| "clearance ahead": (g.ahead || 0).toFixed(3), | |
| "parallax": (100 * (g.moved || 0)).toFixed(1) + "% of the scene-centre depth", | |
| "invented": (100 * (1 - (g.coverage || 0))).toFixed(0) + "%", | |
| "wall clock": j.t + "s", "seed": st.seed, "camera": `keyframe path, ${st.keys.length} keys`, | |
| }).map(([k, v]) => `<i>${k}</i><span>${v}</span>`).join(""); | |
| } | |
| const V3 = ["v6a", "v6b", "v6c"].map($); // the render is the clock; the other two panes and the scrubber follow it | |
| $("v6play").onclick = () => V3[2].paused ? V3.forEach(v => v.play()) : V3.forEach(v => v.pause()); | |
| V3[2].ontimeupdate = () => { | |
| V3.slice(0, 2).forEach(v => { if (Math.abs(v.currentTime - V3[2].currentTime) > 0.08) v.currentTime = V3[2].currentTime; }); | |
| if (!V3[2].paused) $("tk").value = Math.floor(V3[2].currentTime * FPS); | |
| }; | |
| $("tk").oninput = e => V3.forEach(v => { v.pause(); v.currentTime = (+e.target.value + 0.5) / FPS; }); | |
| $("again").onclick = backToStudio; | |
| $("new").onclick = () => location.reload(); | |
| show(0); | |
| </script> | |