chore: deploy private lightloom build
Browse files- .raymarch_impl.txt +210 -0
- .research_synth.txt +376 -0
- app.py +25 -5
- docs/DEEP_RESEARCH_REPORT.md +199 -0
- frontend/js/stage-scroll.js +164 -99
- scripts/deploy_private_space.py +6 -1
- src/lightloom/audio_in/asr.py +11 -3
- src/lightloom/depth/depth.py +7 -4
.raymarch_impl.txt
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// ============================================================================
|
| 2 |
+
// MERGED PASTE-READY PATCH for frontend/js/stage-scroll.js
|
| 3 |
+
// Base = Design 1 (trivial VERT + fragment relief raymarch + OVERSCAN=1.0 +
|
| 4 |
+
// CPU median-pinned focus + scroll-velocity eyepoint), with the heightfield
|
| 5 |
+
// FIXED to full [0,1] range (Design 3's mapping) so parallax is not halved.
|
| 6 |
+
// Replace ONLY the marked regions; everything else (recycle, featherDraw,
|
| 7 |
+
// smoothDepthToStrip, addSection, queue, onFocus, reset, setReducedMotion,
|
| 8 |
+
// pendingAhead, debug, pixelAt, resize, fitMeshToFrustum body, canvas/texture
|
| 9 |
+
// setup) stays byte-for-byte.
|
| 10 |
+
// GLSL target: three ShaderMaterial = GLSL ES 1.00 (no glslVersion set, and the
|
| 11 |
+
// vendored three deprecates-but-supports WebGL1) -> loop bounds MUST be compile-
|
| 12 |
+
// time constants. We bake them via template literals at module-eval. Verified.
|
| 13 |
+
// ============================================================================
|
| 14 |
+
|
| 15 |
+
// ---- (A) CONSTANTS: replace the LEAN_* block (current lines ~37-45) ----------
|
| 16 |
+
// Keep STRIP_H/STRIP_W (29), CAM_FOV/CAM_DIST (33-34), DEPTH_GAIN (35),
|
| 17 |
+
// DEPTH_GAMMA (36). Change OVERSCAN, drop MESH_SEG_*, add raymarch consts.
|
| 18 |
+
const OVERSCAN = 1.0; // CHANGED 1.22 -> 1.0: plane fills frustum exactly so vUv maps 1:1
|
| 19 |
+
const MESH_SEG = 1; // CHANGED: pure full-screen quad; relief is per-fragment now
|
| 20 |
+
// --- relief / parallax-occlusion raymarch (DepthFlow recipe) ---
|
| 21 |
+
const PARALLAX = 0.05; // eyepoint horizontal budget = 5% of view width (4-6% range)
|
| 22 |
+
const MARCH_LIN = 20; // coarse linear forward steps (first-hit bracket)
|
| 23 |
+
const MARCH_BIN = 7; // binary-refine steps inside the bracketed interval
|
| 24 |
+
const VEL_TAU = 0.18; // s — exponential smoothing time-const for scroll velocity
|
| 25 |
+
const VEL_REF = 220.0; // px/s mapped to full +/-1 parallax (scroll ~110px/s => ~0.5 budget)
|
| 26 |
+
const AERIAL_DESAT = 0.45; // max desaturation at the far plane
|
| 27 |
+
const AERIAL_HAZE = 0.38; // max lerp toward the haze tint at the far plane
|
| 28 |
+
const AERIAL_BLUR = 0.9; // texel-scale of the far-plane 4-tap micro-blur
|
| 29 |
+
const HAZE_TINT = [0.62, 0.68, 0.78]; // cool sky/haze tint (per-world settable via uniform)
|
| 30 |
+
|
| 31 |
+
// ---- (B) VERT: replace the whole VERT template (current lines ~58-99) --------
|
| 32 |
+
// Trivial pass-through quad. No depth fetch, no displacement, no VTF needs.
|
| 33 |
+
const VERT = `
|
| 34 |
+
precision highp float;
|
| 35 |
+
varying vec2 vUv;
|
| 36 |
+
void main(){
|
| 37 |
+
vUv = uv;
|
| 38 |
+
gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
|
| 39 |
+
}`;
|
| 40 |
+
|
| 41 |
+
// ---- (C) FRAG: replace the whole FRAG template (current lines ~100-121) ------
|
| 42 |
+
const FRAG = `
|
| 43 |
+
precision highp float;
|
| 44 |
+
varying vec2 vUv;
|
| 45 |
+
uniform sampler2D uImg; // continuous painted strip
|
| 46 |
+
uniform sampler2D uDep; // continuous depth strip, global [0,1], 0=far 1=near
|
| 47 |
+
uniform float uU0; // window left edge, normalized on the strip
|
| 48 |
+
uniform float uViewU; // window width, normalized
|
| 49 |
+
uniform float uTexel; // 1.0 / STRIP_W
|
| 50 |
+
uniform float uTexelV; // 1.0 / STRIP_H
|
| 51 |
+
uniform float uFocus; // CPU-pinned view-centre median depth (aerial pivot + swim guard)
|
| 52 |
+
uniform float uGain; // depth contrast gain
|
| 53 |
+
uniform float uGamma; // depth gamma (<1 lifts mids)
|
| 54 |
+
uniform float uHParallax; // signed eyepoint offset in [-1,1] from smoothed scroll velocity
|
| 55 |
+
uniform float uParallax; // budget (fraction of view width)
|
| 56 |
+
uniform float uAerialDesat;
|
| 57 |
+
uniform float uAerialHaze;
|
| 58 |
+
uniform float uAerialBlur;
|
| 59 |
+
uniform vec3 uHazeTint;
|
| 60 |
+
|
| 61 |
+
const int LIN = ${MARCH_LIN};
|
| 62 |
+
const int BIN = ${MARCH_BIN};
|
| 63 |
+
|
| 64 |
+
// window-uv (vUv) -> strip-uv, clamped into the painted canvas
|
| 65 |
+
vec2 toStrip(vec2 w){
|
| 66 |
+
return vec2(clamp(uU0 + w.x * uViewU, 0.0, 1.0), clamp(w.y, 0.001, 0.999));
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
// HEIGHTFIELD height in FULL [0,1] (FIX vs Design 1's [0,0.5] relief): contrast-stretched
|
| 70 |
+
// depth used directly as the surface height so the ray can intersect anywhere along [1->0].
|
| 71 |
+
// 0 = far (deep), 1 = near (tall). Pivoting/aerial use uFocus separately.
|
| 72 |
+
float heightAt(vec2 w){
|
| 73 |
+
float d0 = texture2D(uDep, toStrip(w)).r;
|
| 74 |
+
float d = pow(clamp(d0, 0.0, 1.0), uGamma);
|
| 75 |
+
return clamp(uFocus + (d - uFocus) * uGain, 0.0, 1.0);
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
vec3 toSRGB(vec3 c){
|
| 79 |
+
c = max(c, vec3(0.0));
|
| 80 |
+
vec3 lo = c * 12.92; vec3 hi = 1.055 * pow(c, vec3(1.0/2.4)) - 0.055;
|
| 81 |
+
return mix(lo, hi, step(vec3(0.0031308), c));
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
void main(){
|
| 85 |
+
// --- view ray (parallax-occlusion): eyepoint shifts horizontally by P across the ray.
|
| 86 |
+
// ray param t in [0,1]; ray height descends 1 -> 0; surface = heightAt(baseUv - P*t).
|
| 87 |
+
// First t where rayH < surfH is the FIRST hit => occlusion-correct (no smear). ----------
|
| 88 |
+
float P = uHParallax * uParallax; // signed, |P| <= uParallax
|
| 89 |
+
vec2 baseUv = vUv;
|
| 90 |
+
float prevT = 0.0;
|
| 91 |
+
bool hit = false;
|
| 92 |
+
float hitT = 1.0;
|
| 93 |
+
// coarse linear forward search for the FIRST below-surface crossing
|
| 94 |
+
for (int i = 1; i <= LIN; i++){
|
| 95 |
+
float t = float(i) / float(LIN);
|
| 96 |
+
float rayH = 1.0 - t; // ray descends from 1 -> 0
|
| 97 |
+
float surfH = heightAt(baseUv - vec2(P * t, 0.0));
|
| 98 |
+
if (rayH < surfH){ hitT = t; hit = true; break; }
|
| 99 |
+
prevT = t;
|
| 100 |
+
}
|
| 101 |
+
// binary refine WITHIN [prevT, hitT] (keeps the FIRST crossing; sub-texel)
|
| 102 |
+
float tA = prevT, tB = hit ? hitT : 1.0;
|
| 103 |
+
for (int j = 0; j < BIN; j++){
|
| 104 |
+
float tm = 0.5 * (tA + tB);
|
| 105 |
+
float rayH = 1.0 - tm;
|
| 106 |
+
float surfH = heightAt(baseUv - vec2(P * tm, 0.0));
|
| 107 |
+
if (rayH < surfH) tB = tm; else tA = tm;
|
| 108 |
+
}
|
| 109 |
+
float tHit = 0.5 * (tA + tB);
|
| 110 |
+
vec2 hitUv = baseUv - vec2(P * tHit, 0.0);
|
| 111 |
+
|
| 112 |
+
// --- sample the painted strip at the first hit (single fetch, occlusion-correct) -------
|
| 113 |
+
vec2 sStrip = toStrip(hitUv);
|
| 114 |
+
vec3 c = texture2D(uImg, sStrip).rgb;
|
| 115 |
+
|
| 116 |
+
// --- AERIAL PERSPECTIVE: cue from the hit's height (0 far -> 1 near) -------------------
|
| 117 |
+
float dHit = texture2D(uDep, sStrip).r;
|
| 118 |
+
dHit = pow(clamp(dHit, 0.0, 1.0), uGamma);
|
| 119 |
+
dHit = clamp(uFocus + (dHit - uFocus) * uGain, 0.0, 1.0);
|
| 120 |
+
float farness = 1.0 - dHit; // 1 at far, 0 at near
|
| 121 |
+
// far-only 4-tap micro-blur (also softens the few-px disocclusion smear at cliffs)
|
| 122 |
+
if (uAerialBlur > 0.0 && farness > 0.01){
|
| 123 |
+
float r = uAerialBlur * farness;
|
| 124 |
+
vec3 b = texture2D(uImg, sStrip + vec2( uTexel*r, 0.0)).rgb
|
| 125 |
+
+ texture2D(uImg, sStrip + vec2(-uTexel*r, 0.0)).rgb
|
| 126 |
+
+ texture2D(uImg, sStrip + vec2(0.0, uTexelV*r)).rgb
|
| 127 |
+
+ texture2D(uImg, sStrip + vec2(0.0, -uTexelV*r)).rgb;
|
| 128 |
+
c = mix(c, b * 0.25, farness * 0.6);
|
| 129 |
+
}
|
| 130 |
+
float luma = dot(max(c, vec3(0.0)), vec3(0.299, 0.587, 0.114));
|
| 131 |
+
c = mix(c, vec3(luma), uAerialDesat * farness); // desaturate toward far
|
| 132 |
+
c = mix(c, uHazeTint, uAerialHaze * farness * farness); // haze concentrated at far
|
| 133 |
+
|
| 134 |
+
// --- linear -> sRGB (unchanged), near-bright shade (replaces old vShade), vignette -----
|
| 135 |
+
c = toSRGB(c);
|
| 136 |
+
c *= mix(0.92, 1.05, dHit);
|
| 137 |
+
float vy = smoothstep(0.0, 0.12, vUv.y) * smoothstep(1.0, 0.88, vUv.y);
|
| 138 |
+
c *= mix(0.82, 1.0, vy);
|
| 139 |
+
gl_FragColor = vec4(c, 1.0);
|
| 140 |
+
}`;
|
| 141 |
+
|
| 142 |
+
// ---- (D) init(): replace the canVTF gate + material uniforms + mesh -----------
|
| 143 |
+
// (current lines ~142-158). DELETE the `const canVTF = ...` line entirely.
|
| 144 |
+
// depTex.generateMipmaps=false / wrapT=ClampToEdge lines STAY (correct).
|
| 145 |
+
mat = new THREE.ShaderMaterial({
|
| 146 |
+
uniforms: {
|
| 147 |
+
uImg: { value: imgTex }, uDep: { value: depTex },
|
| 148 |
+
uU0: { value: 0 }, uViewU: { value: 0.25 },
|
| 149 |
+
uTexel: { value: 1 / STRIP_W }, uTexelV: { value: 1 / STRIP_H },
|
| 150 |
+
uFocus: { value: 0.5 }, uGain: { value: DEPTH_GAIN }, uGamma: { value: DEPTH_GAMMA },
|
| 151 |
+
uHParallax: { value: 0 }, // driven by smoothed scroll velocity in loop()
|
| 152 |
+
uParallax: { value: PARALLAX },
|
| 153 |
+
uAerialDesat: { value: AERIAL_DESAT },
|
| 154 |
+
uAerialHaze: { value: AERIAL_HAZE },
|
| 155 |
+
uAerialBlur: { value: AERIAL_BLUR },
|
| 156 |
+
uHazeTint: { value: new THREE.Vector3(HAZE_TINT[0], HAZE_TINT[1], HAZE_TINT[2]) },
|
| 157 |
+
},
|
| 158 |
+
vertexShader: VERT, fragmentShader: FRAG,
|
| 159 |
+
depthWrite: false, depthTest: false,
|
| 160 |
+
});
|
| 161 |
+
mesh = new THREE.Mesh(new THREE.PlaneGeometry(2, 2, MESH_SEG, MESH_SEG), mat); // pure quad
|
| 162 |
+
scene.add(mesh);
|
| 163 |
+
fitMeshToFrustum(); // UNCHANGED body; with OVERSCAN=1.0 it now fits exactly -> vUv 1:1
|
| 164 |
+
|
| 165 |
+
// ---- (E) MODULE STATE: add near `let scrollX = 0;` (current line ~49) ---------
|
| 166 |
+
let prevScrollX = 0; // scrollX last frame, for d(scrollX)/dt
|
| 167 |
+
let velSmooth = 0; // smoothed px/s scroll velocity
|
| 168 |
+
let _focusCache = 0.5, _focusFrame = 0;
|
| 169 |
+
|
| 170 |
+
// ---- (F) FOCUS PROBE helper: add near reset()/loop() --------------------------
|
| 171 |
+
// Reads the depth canvas centre column (CPU, cheap, throttled 1-in-6).
|
| 172 |
+
// Pins the focus/aerial plane to the view-centre MEDIAN-ish depth (no swim).
|
| 173 |
+
function focusDepthTarget() {
|
| 174 |
+
if ((_focusFrame++ % 6) !== 0) return _focusCache; // throttle off the hot path
|
| 175 |
+
try {
|
| 176 |
+
const viewU = Math.min(1, (aspect * STRIP_H) / STRIP_W);
|
| 177 |
+
const cx = Math.round(scrollX + viewU * STRIP_W * 0.5); // centre column on the strip (px)
|
| 178 |
+
if (cx < 0 || cx >= STRIP_W) return _focusCache;
|
| 179 |
+
const col = depCtx.getImageData(cx, Math.round(STRIP_H * 0.25), 1, Math.round(STRIP_H * 0.5)).data;
|
| 180 |
+
let s = 0, n = 0;
|
| 181 |
+
for (let i = 0; i < col.length; i += 4) { s += col[i]; n++; }
|
| 182 |
+
_focusCache = n ? (s / n) / 255 : 0.5;
|
| 183 |
+
} catch (_) { /* tainted/blocked -> keep last */ }
|
| 184 |
+
return _focusCache;
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
// ---- (G) loop(): replace the lean block (current lines ~344-349) ---------------
|
| 188 |
+
// Everything else in loop() (viewU, maxScroll, speed=110, uU0, focus/onFocus,
|
| 189 |
+
// render) stays byte-for-byte. `rm` and `dt` already exist above.
|
| 190 |
+
// --- scroll-velocity-driven eyepoint (NOT time, NOT cumulative position) ---
|
| 191 |
+
const instVel = dt > 1e-4 ? (scrollX - prevScrollX) / dt : 0; // px/s this frame
|
| 192 |
+
prevScrollX = scrollX;
|
| 193 |
+
const aV = 1 - Math.exp(-dt / VEL_TAU); // one-pole smoothing
|
| 194 |
+
velSmooth += (instVel - velSmooth) * aV;
|
| 195 |
+
// reject the recycle() jump and reduced-motion -> ease the eyepoint to a still flat image
|
| 196 |
+
const clampedVel = Math.max(-2 * VEL_REF, Math.min(2 * VEL_REF, velSmooth));
|
| 197 |
+
const targetVel = rm ? 0 : clampedVel;
|
| 198 |
+
const hTarget = Math.max(-1, Math.min(1, targetVel / VEL_REF)); // signed, bounded [-1,1]
|
| 199 |
+
mat.uniforms.uHParallax.value += (hTarget - mat.uniforms.uHParallax.value) * Math.min(1, dt * 6);
|
| 200 |
+
// pin the FOCUS/aerial plane to the view-centre median depth (slow ease, throttled probe)
|
| 201 |
+
mat.uniforms.uFocus.value += (focusDepthTarget() - mat.uniforms.uFocus.value) * Math.min(1, dt * 1.5);
|
| 202 |
+
|
| 203 |
+
// ---- (H) reset(): zero the new state (current lines ~358-364) ------------------
|
| 204 |
+
// inside reset(), alongside `writeX = 0; scrollX = 0;` add:
|
| 205 |
+
prevScrollX = 0; velSmooth = 0; _focusCache = 0.5; _focusFrame = 0;
|
| 206 |
+
if (mat) { mat.uniforms.uHParallax.value = 0; mat.uniforms.uFocus.value = 0.5; }
|
| 207 |
+
|
| 208 |
+
// ---- (I) recycle(): keep velocity consistent (current lines ~319-327) ----------
|
| 209 |
+
// inside recycle(), after `... scrollX = Math.max(0, scrollX - shift); ...` add:
|
| 210 |
+
prevScrollX = Math.max(0, prevScrollX - shift); // so next-frame velocity isn't a spike
|
.research_synth.txt
ADDED
|
@@ -0,0 +1,376 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
==========================================================================================
|
| 2 |
+
THEME: Depth, 3D & Branded Innovation
|
| 3 |
+
VERDICT: Stop tuning the lean — it is flat by construction (verified: stage-scroll.js:96 is a center-anchored zoom with zero parallax where the eye looks and no scroll term). Ship ONE depth path: a fragment-shader steep-parallax raymarch (DepthFlow recipe) over the single continuous strip, driven by scroll velocity, on globally-normalized depth. Brand the innovation as "the painter paints its own hidden layer": a SECOND FLUX.2-klein pass that inpaints the disocclusion backfill, turning a generic 3D-photo into a FLUX-native streaming Layered Depth Strip. BUT none of this matters until the Day-0 blocker is fixed — the live painter import (Flux2KleinInpaintPipeline) does not exist in the pinned diffusers 0.36.0; the Voice-Scroll cannot currently run as written.
|
| 4 |
+
|
| 5 |
+
TOP RECOMMENDATIONS:
|
| 6 |
+
[high/M] FIRST, UNBLOCK THE PAINTER. Verified against the installed env: diffusers 0.36.0 exports only Flux2Pipeline (no Flux2KleinInpaintPipeline/Flux2KleinKVPipeline; imported at scroll.py:73,78, klein.py:50,53, panorama.py:30,32), and Flux2Pipeline.__call__ has NO mask_image and NO strength. Rewrite outpaint_right on the real Flux2Pipeline image= reference path with manual latent compositing for the right-edge mask, OR pin the exact diffusers commit that defines the inpaint class and record it in requirements.txt with a VERIFY citation. Run load_fast_pipeline()+one outpaint_right() in the Space and paste the image before building anything else.
|
| 7 |
+
WHY: Every depth/frontend/demo deliverable sits on top of a painter that cannot be constructed on the pinned environment. This is a true Day-0 blocker; until it is green the live demo does not exist. It also protects the honesty axis — a stale import in the very files that claim source-verification is the worst thing a BFL judge can find.
|
| 8 |
+
[high/M] Replace renderer #3 with a fragment-shader steep-parallax (DepthFlow-style) raymarch over the SINGLE continuous 8192px strip+depth texture: per output pixel, march the view ray into the heightfield (16-24 coarse forward steps + 6-8 binary refine), take the FIRST surface hit, sample the image there. Pin the FAR plane (steady/focus at the view-center median depth) and drive the eyepoint offset O.x from scroll VELOCITY (never a time/Lissajous term), bounded to ~4-6% of view width, eased to ~0 at rest.
|
| 9 |
+
WHY: This is the ONLY candidate that is simultaneously tear-free (first-hit occlusion, not forward UV smear), a TRUE viewpoint change (so it reads as 3D at frame center, fixing 'too flat'), seam-free (one continuous strip, no per-quad geometry — retires the renderer-#2 seam shear), and proven real-time (DepthFlow ships 8K@50fps). It retires all three rejected renderers at once with no per-frame network.
|
| 10 |
+
[high/S] Kill the double per-strip depth normalization (verified: depth.py:17-30 min-max+invert, then app.py:893-895 and app.py:1188-1190 re-stretch each strip again). Return raw disparity and normalize ONCE against a per-session running [lo,hi] (EMA of 2/98 percentiles), persisted in live state across GPU-worker recycles. Add a unit test that two adjacent strips of the same flat scene produce identical normalized depth.
|
| 11 |
+
WHY: Independent per-strip scaling makes 'near' mean something different every section, so the browser affine seam-match (140px feather only) cannot fix a whole-section DC/contrast difference — this is the structural root of the renderer-#2 seam shear AND a contributor to 'flat'. One global scale is the single highest-leverage fix for both complaints and regression-locks the thing that broke before.
|
| 12 |
+
[high/L] Ship the branded innovation: a STREAMING LAYERED DEPTH STRIP. Per appended strip, run a SECOND klein pass masked to a soft-disocclusion map (SLIDE: S=tanh of the depth-gradient discontinuity) to inpaint the hidden background ONCE at generate-time; store it as a BG layer the raymarch/composite reveals as content sweeps. Brand it 'the painter paints what it hides.' Pair with the verified-negative adaLN result as a matched negative+positive contribution.
|
| 13 |
+
WHY: This is the defensible, FLUX-centric novelty that out-impresses a Voice-Scroll clone: the depth effect is generated BY the sponsor's model (klein doing double duty: outpaint forward + inpaint behind), amortized into a FLUX call you already pay for, and is the only single-image-depth architecture that survives fork-per-call ZeroGPU + an 8192px streaming canvas. It gives true accretion-deletion — the depth cue the lean structurally lacks.
|
| 14 |
+
[high/S] Add a depth-driven AERIAL-PERSPECTIVE pass (desaturate + lerp toward a per-world sky tint + tiny blur as depth->far) in the existing fragment shader, plus a disocclusion stretch-fill clamp (at 4-6% parallax the smear is a few px and reads as soft blur — no neural inpaint needed for the live fallback). This stacks under the raymarch and is incapable of tearing.
|
| 15 |
+
WHY: Aerial perspective is a primary monocular depth cue the project currently lacks (vShade is only a 0.90-1.06 brightness nudge). Coarse DepthAnything is well-suited to atmospheric cues (tolerant of low contrast) and badly suited to hard parallax (exposes speckle). This is the cheapest visible-depth win and a clean fallback that makes the raymarch land even before the klein-backfill exists.
|
| 16 |
+
[medium/M] Add interim ASR captions and couple scroll/parallax to live voice energy. Switch asr-worker.js to emit partial hypotheses (greyed interim that hardens on final-commit, anchored under the scroll focus), and drive scroll speed + parallax amplitude from a smoothed RMS/pitch envelope (already in-browser) with inertia. Tie word-settle to the next outpaint emit so a judge SEES the word become world.
|
| 17 |
+
WHY: Two cheap, GPU-free wins the user explicitly asked for. Interim captions are table-stakes 2026 fluidity (their absence reads as unfinished); voice-coupling is the highest aliveness-per-effort change (2025 IMX research: time-evolving eased reactivity feels alive, instantaneous does not). Together they make the voice->world causation unmistakable, rescuing the format from the 'screensaver' read.
|
| 18 |
+
[medium/S] Do NOT pursue live per-strip 3DGS, full MPI (32-64 planes), or a 360/3DGS fly-through for THIS submission. Confine splats to ONE pre-baked Spark.js fly-through of the FINISHED spoken world as the demo finale, and frame Lightloom explicitly AGAINST World Labs Marble (cloud, minutes, one-shot, static) — Lightloom is live, local, continuous, voice-streamed on one ZeroGPU slot.
|
| 19 |
+
WHY: World Labs Marble + World API (Jan 2026) + Spark 2.0 already commoditized 'single image -> explorable 3DGS in browser', so a naive splat pivot reads as a downgraded Marble clone and competes where 5.83B local loses. Live per-strip splatting also blows the 120s/concurrency-1 budget and re-introduces seam alignment of independent Gaussian clouds. Pre-bake the wow; own the live-local-voice axis.
|
| 20 |
+
|
| 21 |
+
BOLD MOVES:
|
| 22 |
+
* The Streaming Layered Depth Strip: make klein paint its own hidden background layer (a second masked pass over the SLIDE soft-disocclusion map) so depth is generated BY FLUX, not bolted on. 'The painter paints what it hides' — a publishable, weight-free, ZeroGPU-survivable system contribution no Voice-Scroll clone can have.
|
| 23 |
+
* FLUX.2 4D position-IDs as a streaming spatial-stitch primitive: subclass the pipeline to override _prepare_image_ids so the previous strip's W-coordinates sit immediately LEFT of the new target (instead of resetting to W=0), making RoPE encode true horizontal adjacency. Verified in the installed source (pipeline_flux2.py _prepare_image_ids resets H/W to 0; references stamped at T=10,20,30). This is the positive mirror of the verified-negative adaLN result — born-consistent seams at generation time, defensible line-by-line against BFL's own code.
|
| 24 |
+
* Publish the matched pair as the research artifact: VERIFIED NEGATIVE (klein adaLN is a scalar broadcast -> per-column LoRA streaming is impossible) + VERIFIED POSITIVE (klein inpaint CAN synthesize disocclusion backfill in real time, and W-offset reference IDs enable neighbor-stitch). A clean negative+positive about THEIR model, with a 30-line repro, is exactly BFL's 'inspectable open research' register.
|
| 25 |
+
* Depth-from-the-painter (Modal stretch): train a tiny depth/normal head or rank-16 LoRA on klein's frozen VAE latents so the strip emits aligned depth as a byproduct of the same forward pass — scale-consistent across strips by construction (shared latent space => no seam fit), self-distilled from Depth Pro targets. Turns 'we use DepthAnything' into 'our painter is geometry-aware' (cite JointDiT 2505.00482).
|
| 26 |
+
* One pre-baked Spark.js fly-through finale: after a session, lift the finished 8192px strip+depth into per-pixel Gaussians and let the user walk INTO their spoken world — the single most shareable demo artifact — while keeping the live loop on the safe raymarch path.
|
| 27 |
+
|
| 28 |
+
RISKS:
|
| 29 |
+
- DAY-0 BLOCKER (verified): the live painter import does not resolve on diffusers 0.36.0 (Flux2KleinInpaintPipeline/KVPipeline absent at 6 call sites; Flux2Pipeline.__call__ has no mask_image/strength). The deployed 'random-cover-1' build may be running 6 pre-rendered covers, NOT live generation — confirm what actually paints live in the Space before trusting any 'working' claim.
|
| 30 |
+
- The klein-backfill (Layered Depth Strip) is L-effort and adds a second klein pass per strip to a 120s/concurrency-1 budget already mostly consumed by FLUX+depth. Ship the raymarch + aerial-perspective + stretch-fill FALLBACK first so a clean depth lands even if the backfill pass is too slow; gate the backfill behind a flag.
|
| 31 |
+
- Honesty exposure: klein.py's docstring cites a non-existent file/class (pipeline_flux2_klein_kv.py:611-638, Flux2KleinKVPipeline). A stale source citation in the file that claims source-verification directly undermines the honesty angle BFL judges reward. Re-verify and cite real lines (pipeline_flux2.py _prepare_image_ids L407-454) before any judge reads the code.
|
| 32 |
+
- Depth is luminance-biased on stylized art (bright skies read as NEAR with invert=True) and the invert flag may be double-flipping for the pinned -hf revision. Verify invert against the model card and clamp the sky band, or the raymarch will parallax the wrong things — a worse failure than flatness.
|
| 33 |
+
- smoothDepthToStrip runs ~1M synchronous main-thread array ops per section at the live rate — a recurring multi-ms stall that will jank the scroll precisely when the user speaks. Move it to an OffscreenCanvas worker or a GPU prepass before adding the raymarch's per-frame uniform work.
|
| 34 |
+
- World Labs Marble/World API/Spark 2.0 already own 'image -> explorable 3DGS in browser.' Any 3D framing that doesn't foreground LIVE + LOCAL + VOICE + CONTINUOUS + one-tiny-GPU-slot reads as a derivative Marble clone — pre-empt the comparison explicitly or lose it by default.
|
| 35 |
+
|
| 36 |
+
REPORT SECTION:
|
| 37 |
+
# Depth, 3D & Branded Innovation
|
| 38 |
+
|
| 39 |
+
## The verdict, stated without comfort
|
| 40 |
+
|
| 41 |
+
The depth problem is not a tuning problem and never was. It is a **representation error**, and I verified the mechanism in the shipped code: renderer #3's vertex stage is `p.x = position.x * (1.0 + W)` (`stage-scroll.js:96`) — a horizontal **zoom about screen-center**, not a viewpoint change. The differential displacement between a near and a far vertex is `position.x · (W_near − W_far)`, which is **identically zero at `position.x ≈ 0`** — the center of frame, exactly where the eye tracks content as it scrolls. There is **no scroll term and no time term** in the displacement, so two columns at the same screen-x never separate by more than a sub-pixel fraction through the visible window. Raising `LEAN_X` cannot fix this; it only shears the periphery harder. "Too subtle / nearly flat" is the correct and inevitable read of this formula.
|
| 42 |
+
|
| 43 |
+
Three independent perceptual analyses converge on the same root cause from the other direction: the lean produces only **expansion-compression** parallax, but the dominant depth cue at any real depth separation is **accretion-deletion** — near content progressively covering and uncovering far content (Ono/Yoonessi & Baker, JOV 2013). Because the strip is sampled at the *undisplaced* uv (`FRAG`, line 108-110), no pixel is ever covered or uncovered. **The single most powerful depth cue is structurally absent.** That is the whole story.
|
| 44 |
+
|
| 45 |
+
## Before any of this matters: the painter is broken on the pinned environment
|
| 46 |
+
|
| 47 |
+
I ran the imports against the installed stack. **`diffusers 0.36.0` exports only `Flux2Pipeline`** — `Flux2KleinInpaintPipeline` and `Flux2KleinKVPipeline` do not exist, yet they are imported at six sites (`scroll.py:73,78`, `klein.py:50,53`, `panorama.py:30,32`). Worse, `Flux2Pipeline.__call__` has **no `mask_image` and no `strength` parameter** (verified signature). The masked right-edge outpaint that *is* the Voice-Scroll cannot run as written. The deployed "random-cover-1" build is plausibly serving the 6 pre-rendered covers, not live generation. **This is the Day-0 blocker; fix it before touching depth.** Either pin the exact diffusers commit that defines the inpaint class (and cite it in `requirements.txt`), or rewrite `outpaint_right` on the real `Flux2Pipeline.image=` path with manual latent compositing — then run one `outpaint_right()` in the Space and paste the image. Everything below assumes a painter that actually paints.
|
| 48 |
+
|
| 49 |
+
## The definitive depth path (commit to one, retire the other three)
|
| 50 |
+
|
| 51 |
+
The specialists split between four candidates — DepthFlow raymarch, SLIDE/LDI 2-layer, quantized multiplane bands, and "depth-as-atmosphere." They are not equally correct for *this* system. The decisive constraints are: **one continuous 8192px strip** (not a hero image), **fork-per-call ZeroGPU** (heavy work must amortize into a FLUX call already paid for), and **three already-rejected failure modes** (tear, vibration, seam-shear). Under those constraints:
|
| 52 |
+
|
| 53 |
+
- **Full MPI (32-64 planes) is a trap** — VRAM/bandwidth-prohibitive for a live 8192px canvas; SinMPI needs per-scene optimization. Discard.
|
| 54 |
+
- **Quantized depth bands** are better than the lean but still warp per-band geometry and re-expose the seam at band boundaries under strong parallax.
|
| 55 |
+
- **The raymarch is the only formulation that retires all three rejections at once.**
|
| 56 |
+
|
| 57 |
+
**Ship a fragment-shader steep-parallax (DepthFlow-recipe) raymarch over the single continuous strip.** Per output pixel: march the view ray into the depth heightfield — ~16-24 coarse forward steps until the ray penetrates the surface, then 6-8 binary-refine steps — and shade the **first surface hit**. This is geometry-correct occlusion (foreground covers background → no tear), it is a **true viewpoint change** (reads as 3D at frame center, fixing flatness), and because it samples the **one continuous strip texture** rather than per-quad meshes, the renderer-#2 seam shear cannot occur (the march samples across the seam transparently). DepthFlow ships this at 8K@50fps on an RTX 3060, so the budget is not in question. Drive the eyepoint offset from **scroll velocity** (never a clock), pin the **far plane** (steady/focus at the view-center median depth), bound to **~4-6% of view width**, and ease to ~0 at rest — this is the one-line reconciliation of all three failures: real per-pixel parallax (fixes #1), no oscillation (fixes #2's nausea), not flat (fixes #3).
|
| 58 |
+
|
| 59 |
+
Two prerequisites make the march *bite* instead of degenerating back to flat, and both are confirmed bugs:
|
| 60 |
+
|
| 61 |
+
1. **Kill the double normalization.** `depth.py:17-30` min-max-normalizes, then `app.py:893-895` and `app.py:1188-1190` re-stretch each strip *again* per-strip. So "near" means something different every section, and the browser affine match (140px feather only) cannot undo a whole-section DC/contrast difference. Return **raw disparity** and normalize **once** against a per-session running `[lo,hi]` (EMA of 2/98 percentiles), persisted across GPU-worker recycles. Add a regression test: two adjacent flat-scene strips must normalize identically. This is the structural root of the seam shear and a contributor to flatness.
|
| 62 |
+
2. **Add a fallback fill + atmosphere.** At 4-6% parallax the disocclusion smear is a few pixels — clamp the march to the last valid background sample (DepthFlow's free edge-stretch); you do **not** need a neural inpaint for the live fallback. Stack a **depth-driven aerial-perspective** pass (desaturate + lerp toward a per-world sky tint + tiny blur as depth→far) in the existing fragment shader. The project currently has *zero* aerial cue (`vShade` is a 0.90-1.06 brightness nudge); coarse DepthAnything is well-suited to atmospheric cues and badly suited to hard parallax, so this is the cheapest visible-depth win and makes the renderer land even before the branded layer exists.
|
| 63 |
+
|
| 64 |
+
Also non-negotiable: depth is **luminance-biased** on painterly output (bright skies read as NEAR under `invert=True`, and the invert flag may double-flip on the pinned `-hf` revision). Verify `invert` against the model card and clamp the sky band, or the raymarch will parallax the wrong things — a worse failure than flatness. And move `smoothDepthToStrip` off the main thread (it is ~1M synchronous ops per section, a recurring scroll-jank stall) before adding per-frame march uniforms.
|
| 65 |
+
|
| 66 |
+
## The branded innovation: the painter paints what it hides
|
| 67 |
+
|
| 68 |
+
A raymarched 3D-photo, however clean, is a **commodity** (DepthFlow, Immersity, LeiaPix). To impress the people who wrote FLUX, the depth must be **generated by their model**. The distinctive, defensible contribution is a **Streaming Layered Depth Strip**:
|
| 69 |
+
|
| 70 |
+
For each appended strip, run a **second klein pass** masked to a SLIDE-style **soft-disocclusion map** (`S = tanh(γ · depth-gradient discontinuity)`) to inpaint the **hidden background layer** — *once*, at generate-time, amortized into the FLUX call you already pay for. Store it as a BG layer the raymarch reveals as content sweeps the frame. Branded honestly: **"the painter paints what it hides."** This is novel in exactly the ways BFL judges recognize: (1) the depth/3D effect is tied to the FLUX painter itself — klein does double duty (outpaint forward + inpaint behind) — not a generic depth-wobble bolted on; (2) it is the *only* single-image-depth architecture that survives fork-per-call ZeroGPU + a continuously-streaming 8192px canvas; (3) it gives true **accretion-deletion**, the cue the lean structurally lacks. Ship the raymarch + aerial-perspective + stretch-fill **fallback first**; gate the klein backfill behind a flag so a clean depth lands even if the second pass is too slow for the 120s budget.
|
| 71 |
+
|
| 72 |
+
Pair this with the **FLUX.2 4D position-ID stitch** — a second, weight-free positive result. Verified in the installed source (`pipeline_flux2.py`, `_prepare_image_ids` L407-454): every reference resets its H/W grid to 0. Override it so the previous strip's W-coordinates sit **immediately left** of the new target, making RoPE encode true horizontal adjacency — born-consistent seams at *generation* time, not patched in the shader. This is the **positive mirror of the verified-negative adaLN result**, defensible line-by-line against BFL's own code.
|
| 73 |
+
|
| 74 |
+
**The research artifact writes itself as a matched pair:** the VERIFIED NEGATIVE (klein's adaLN is a scalar broadcast → per-column LoRA streaming is provably impossible) plus the VERIFIED POSITIVE (klein inpaint *can* synthesize disocclusion backfill in real time; W-offset reference IDs *do* enable neighbor-stitch). A clean negative+positive about *their* model, with a 30-line repro, is precisely BFL's "inspectable open research" register. **Caveat:** `klein.py`'s current docstring cites a non-existent file (`pipeline_flux2_klein_kv.py:611-638`) and class — fix that stale citation before a judge reads the file, or the honesty angle collapses.
|
| 75 |
+
|
| 76 |
+
## What to refuse
|
| 77 |
+
|
| 78 |
+
Do **not** pursue live per-strip 3DGS, full MPI, or a live 360/splat fly-through for this submission. World Labs **Marble + World API (Jan 2026) + Spark 2.0** already own "image → explorable 3DGS in browser"; a naive splat pivot reads as a downgraded Marble clone and competes exactly where 5.83B-local loses, while blowing the 120s/concurrency-1 budget on per-strip Gaussian-cloud seam alignment. Confine splats to **one pre-baked Spark.js fly-through of the finished spoken world** as the demo finale — the single most shareable artifact — and frame Lightloom explicitly **against** Marble: cloud/minutes/one-shot/static versus **live/local/continuous/voice-streamed on one ZeroGPU slot.** That intersection is genuinely unclaimed, and it, not the 3D fidelity, is the moat.
|
| 79 |
+
==========================================================================================
|
| 80 |
+
THEME: Gradio: confirmation + amazing uses
|
| 81 |
+
VERDICT: CONFIRMED — Lightloom's Gradio use is not merely valid, it is the single strongest possible reading of the rule: a reference-grade gradio.Server app (the custom-frontend backend Gradio shipped April 2026), with the @gradio/client transport load-bearing for ZeroGPU quota attribution. BUT the win is invisible to judges and one silent HF SDK bump from total failure. Two hard moves make it undeniable AND a wow: (1) PIN sdk_version + boot-assert gr.Server (disqualifier-class fix), (2) PUBLISH the WebGL voice-scroll renderer as the first-of-its-kind custom component to PyPI + the Gradio Gallery. Ship interim captions from the already-built (currently dead) client-side path. Do NOT adopt FastRTC for the core loop.
|
| 82 |
+
|
| 83 |
+
TOP RECOMMENDATIONS:
|
| 84 |
+
[high/S] PIN the Gradio version in README front-matter: add `sdk_version: <the exact gr.Server-providing build the live Space runs today>` (read it off the deployed /health, do NOT guess), keep gradio UNPINNED in requirements.txt, and add a boot-time `assert hasattr(gr, 'Server'), 'Lightloom needs gradio>=6.6 with gr.Server'` plus `gradio.__version__` in /health.
|
| 85 |
+
WHY: Verified locally: gradio 6.5.1 has NO gr.Server (hasattr==False) and the whole app is `app = gr.Server(...)` at app.py:28. README has `sdk: gradio` with NO sdk_version, so the Space silently follows HF's default. The 6.6+ band (where gr.Server lives) has shipped breaking changes (#12921, #12995). A silent HF bump below 6.6 kills the app at import, on stage, with no local repro. This is the only disqualifier-class fragility in the entire Gradio layer.
|
| 86 |
+
[high/L] SINGLE MOST IMPRESSIVE MOVE: extract the WebGL voice-scroll/depth-lean renderer into a publishable custom Gradio component via `gradio cc create` (3D template), wrap stage-scroll.js + the depth-lean shader as the Svelte frontend, give it a clean Python API (value=list of {image_url, depth_url, caption}; events on_scroll/on_section), and `gradio cc publish` to PyPI + the public Gradio Custom Components Gallery as e.g. `gradio-voicescroll`.
|
| 87 |
+
WHY: The Gallery currently has nothing resembling a live, voice-driven, depth-displaced painterly-panorama renderer — this would be the FIRST of its kind, a verifiable, pip-installable, citable artifact in Gradio's own registry. It converts an unshippable bespoke demo into a permanent branded thing others can `pip install` (the contributor-graph/citation closure the user cares about), proves the WebGL is real not a video, and is exactly the 'real innovation, not a clone' signal BFL rewards. It also directly serves the depth innovation: package renderer (3) + RGB-guided depth sharpening as THE branded effect.
|
| 88 |
+
[high/S] Wire interim on-screen captions from the ALREADY-BUILT client-side path that is currently 100% dead code: pass `onInterim` into recorder.js from controller.js _initRecorder, feed _segTo16k() output to the existing asr-worker.js (transformers.js WebGPU Whisper), and render the growing partial into a NON-destructive #transcript span; replace with the authoritative server transcript when /scroll_live returns. Route NONE of this through the GPU.
|
| 89 |
+
WHY: Verified: controller.js:153-178 wires onSegment/onTranscript but NOT onInterim, so recorder.js's interim hook (fired every 1100ms with 16k audio) never runs. asr.js/asr-worker.js exist but controller.js never imports LL.asr into the caption path. The user's #1 wanted feature ('words as you speak') literally does not function today — captions only appear after a full ZeroGPU round-trip. This delivers it with zero new infra, zero GPU cost, zero ZeroGPU/WebRTC risk.
|
| 90 |
+
[high/S] Make the Gradio backbone LEGIBLE to a video-only judge: in the About overlay + one HUD line + the demo video, state plainly 'Built on gr.Server (Gradio Server mode, shipped April 2026) — custom three.js/WebGL frontend on Gradio's backend: live SSE streaming + queue + per-call ZeroGPU quota via @gradio/client'. Cite the field guide's own words ('gr.Server is your friend — go well beyond the stock components').
|
| 91 |
+
WHY: The Off-Brand badge ($1,500) rewards 'best custom UI pushing past the default Gradio look' and the field guide literally points to gr.Server. The win is satisfied in code (verified: gr.Server + 7 @app.api endpoints, @gradio/client transport with the documented quota-attribution comment at api.js:3-5) but INVISIBLE to a judge who only watches the video. Naming the exact blessed pattern is the cheapest, highest-leverage way to claim the badge and prove the dependency is real, not decorative.
|
| 92 |
+
[medium/M] Expose the painter as a Gradio MCP server too: stack @app.mcp.tool() over the existing scroll_live/transcribe @app.api endpoints (one-line decorators, additive, independent of the @app.api path). Add one demo line: 'an agent paints a continuous world by speaking.'
|
| 93 |
+
WHY: @app.mcp.tool() is a gr.Server-EXCLUSIVE surface — it makes the Gradio dependency doubly undeniable, opens the 'best agentic app' category (this hackathon is heavily MCP-framed), and reframes Lightloom as agent-drivable real-time generative infrastructure, all from already-written functions. Keep it 'undocumented'/private if you don't want it in the public API surface. Low risk because it is purely additive.
|
| 94 |
+
[medium/S] Do NOT adopt FastRTC/WebRTC for the core voice->paint loop; document it as a 'considered and rejected' verified-negative in VERIFY_LOG. Keep the @app.api + @gradio/client transport. If revisited, use FastRTC for mic-transport+VAD ONLY, prototype-gated on a throwaway Space.
|
| 95 |
+
WHY: Three independent findings converge: FastRTC's ReplyOnPause is pause-triggered, not partial-during-speech (so it would NOT beat the existing client-side onInterim loop); WebRTC needs ONE long-lived peer connection which fights ZeroGPU's fork-per-call/kill-on-release model (no published pattern of a FastRTC handler driving @spaces.GPU diffusion); and Spaces WebRTC requires a Cloudflare TURN dependency that can fail live behind NATs. The latency math: WebRTC only shaves audio-upload (~100-160KB), a rounding error against klein+depth GPU time. A documented rejection strengthens the honesty narrative judges weight.
|
| 96 |
+
|
| 97 |
+
BOLD MOVES:
|
| 98 |
+
* Publish `gradio-voicescroll` — the FIRST live, voice-driven, depth-displaced painterly-panorama custom component in the public Gradio Custom Components Gallery (pip install + Gallery listing + PyPI). A permanent, citable, clickable artifact that a Voice-Scroll clone structurally cannot produce, and the cleanest path to closure+publication+branding all at once.
|
| 99 |
+
* Bundle the BRANDED depth effect INTO that component as its signature: ship renderer (3)'s continuous depth-lean + RGB-guided joint-bilateral depth sharpening (guided filter, arXiv 2201.06823) + 2-3 alpha-banded LDI/CPSL layers so parallax reveals inpainted background instead of tearing — turning 'too flat' and 'tears at edges' into one named effect ('Relief Loom') that IS the component's reason to exist.
|
| 100 |
+
* Dual-surface the same backend: ONE gr.Server app that is simultaneously (a) a custom-WebGL Off-Brand showcase app and (b) an MCP server (@app.mcp.tool over the live endpoints) — 'an agent paints a continuous world by speaking' — hitting both the Off-Brand badge and the agentic-app category from code that already exists.
|
| 101 |
+
* Frame Lightloom as 'Gradio as a real-time generative-media ENGINE, not a form': a latency-bound, frame-streaming FLUX.2-klein outpaint loop over SSE on ONE ZeroGPU slot with concurrency-1 serialization and per-call quota via @gradio/client — the exact workload most entrants will NOT attempt on gr.Server (they'll reach for gr.Blocks + an Image component). Pair with the verified @gradio/client-vs-raw-fetch quota-attribution finding as a genuine systems craft signal for a FLUX-authoring judge.
|
| 102 |
+
* Add a one-paragraph reproducibility hook in the README: a `gradio_client` snippet (`client.submit('/scroll_live', ...)`) so judges can drive the live, ZeroGPU-attributed pipeline programmatically — proving the endpoints are real queued API surfaces, not a faked demo.
|
| 103 |
+
|
| 104 |
+
RISKS:
|
| 105 |
+
- DISQUALIFIER-CLASS: gradio is unpinned (README has `sdk: gradio`, NO sdk_version) and gr.Server does NOT exist in 6.5.1 (verified: hasattr==False). A silent HF default bump below 6.6 — or a 6.6+ breaking change (#12921/#12995 are in exactly that band) — kills the app at `app = gr.Server()` import, on stage, with no local repro. Until sdk_version is pinned, every other Gradio win is moot.
|
| 106 |
+
- INVISIBILITY: the Off-Brand win is satisfied in code but a video-only judge cannot see that the SSE transport, queue, and ZeroGPU quota ARE Gradio. README mentions gradio.Server ONCE in passing; the in-app HUD/About never surface it. The badge is left on the table unless the backbone is named in the video.
|
| 107 |
+
- INTERIM CAPTIONS ARE DEAD CODE (verified): controller.js never passes onInterim to recorder.js and never imports LL.asr into the caption path, so the user's explicitly-requested #1 feature does not function — 'transcript shown instantly' is currently FALSE; captions wait for a full GPU round-trip.
|
| 108 |
+
- FastRTC is a seductive trap: it would NOT improve interim captions (ReplyOnPause is pause-triggered), it fights ZeroGPU fork-per-call, and it adds a Cloudflare-TURN failure mode in a live demo. Adopting it wholesale risks breaking the verified quota-attribution path for a transport gain that is a rounding error.
|
| 109 |
+
- PUBLICATION SCOPE: `gradio cc publish` makes the component (and its embedded shaders/renderer) PUBLIC — collides with the active privacy-mode constraint (dev Space personal, nothing public yet). The component publish must be a deliberate, timed flip, NOT an accidental leak; sanitize the private LoRA repo id and any /health BOM before going public (note: /health currently leaks the full model ledger + the un-published LoRA repo id regardless of privacy mode).
|
| 110 |
+
- HONESTY/LEDGER: the live scroll_live path fuses an un-ledgered style LoRA (Efradeca/lightloom-style-lora, absent from MODEL_REFS) — if you advertise the param ledger as the honesty artifact while a fused LoRA on the LIVE path is omitted, a sharp judge can call it. Register it (even at ~0B net) with a pinned revision before leaning on the honesty pillar.
|
| 111 |
+
|
| 112 |
+
REPORT SECTION:
|
| 113 |
+
# Gradio: Confirmation + Amazing Uses
|
| 114 |
+
|
| 115 |
+
## Verdict: confirmed, strongest-possible reading — but invisible and one bump from death
|
| 116 |
+
|
| 117 |
+
Lightloom does not merely satisfy the Gradio requirement; it uses **the single newest and most differentiated Gradio surface that exists**. `app.py:28` instantiates `app = gr.Server(...)` and declares seven live GPU endpoints as `@app.api(name=..., concurrency_id="gpu", concurrency_limit=1, stream_every=0.05)` generators — `transcribe`, `panorama`, `world`, `scroll`, `scroll_live`, `recital`, plus the bench-gated routes — each streamed as SSE and consumed in the browser through `@gradio/client` (`Client.connect(window.location.origin)` + `client.submit(...)`). This is **exactly** the pattern the Build Small field guide blesses ("gr.Server is your friend — go well beyond the stock components and make it yours") and the target of the dedicated **Off-Brand badge ($1,500)**. `gradio.Server` was published April 1 2026; at judging it is ~2 months old, and most entrants will ship stock `gr.Blocks`. The honest framing is not "do more Gradio" — it is **"we are the reference-grade `gradio.Server` app."**
|
| 118 |
+
|
| 119 |
+
The dependency is also genuinely **load-bearing**, not cosmetic. `frontend/js/api.js:3-5,39-40` reaches the GPU pipeline *only* through `@gradio/client`, with a verified in-code comment: raw `fetch()` against the endpoints **breaks ZeroGPU quota attribution** because the client injects the iframe/session auth headers `fetch` cannot. That is a non-obvious systems finding already proven in your code — surface it.
|
| 120 |
+
|
| 121 |
+
## The one hard truth: it is unpinned and unreproducible
|
| 122 |
+
|
| 123 |
+
I verified locally: **`gradio 6.5.1` has no `gr.Server` (`hasattr(gradio,'Server') == False`).** The entire app boots only if HF injects a 6.6+ build. README front-matter has `sdk: gradio` with **no `sdk_version`** (confirmed), so the Space silently follows HF's default. The 6.6+ band — exactly where `gr.Server` lives — has shipped breaking changes (issues #12921, #12995). A silent HF default bump, or a 6.x API drift, kills the app at `app = gr.Server()` **at import, on stage, with no way to reproduce locally on 6.5.1.** This is the only disqualifier-class fragility in the Gradio layer and it is trivially fixable.
|
| 124 |
+
|
| 125 |
+
**Fix (do first):** read the gr.Server-providing version off the *deployed* `/health`, pin it as `sdk_version: <x.y.z>` in README front-matter (keep gradio **unpinned** in `requirements.txt` — pinning there genuinely breaks the HF build), add a boot `assert hasattr(gr,'Server')` with an actionable message, surface `gradio.__version__` in `/health`, and add a 10-line CI import-smoke test against the pinned version.
|
| 126 |
+
|
| 127 |
+
## The amazing move: publish the renderer as a custom component
|
| 128 |
+
|
| 129 |
+
The highest-impact creative move is to **extract the WebGL voice-scroll/depth-lean renderer into a publishable custom Gradio component** (`gradio cc create` → wrap `stage-scroll.js` + the depth-lean shader as the Svelte frontend → clean Python API `value=[{image_url, depth_url, caption}]`, events `on_scroll`/`on_section` → `gradio cc publish`). The Custom Components Gallery currently contains **nothing** resembling a live, voice-driven, depth-displaced painterly panorama — `gradio-voicescroll` would be the **first of its kind**, a pip-installable, registry-listed, **citable** artifact. This is the strongest available "real innovation, not a clone" signal for BFL, it converts an unshippable bespoke demo into a permanent branded thing (the contributor-graph/citation closure you want), and it proves the WebGL is real. Bundle the **branded depth effect into the component as its signature** (renderer (3)'s continuous lean + RGB-guided joint-bilateral sharpening + 2–3 alpha-banded LDI/CPSL layers) so "too flat" and "tears at edges" become one named effect that *is* the component's reason to exist.
|
| 130 |
+
|
| 131 |
+
## The user's #1 feature is dead code — ship it from what exists
|
| 132 |
+
|
| 133 |
+
Interim "words-as-you-speak" captions **do not function today.** `controller.js:153-178` (`_initRecorder`) wires `onSegment` and `onTranscript` but **never passes `onInterim`**, so `recorder.js`'s interim hook (fired every ~1100ms with 16k-resampled in-progress audio, recorder.js:125-129) is orphaned. `asr.js`/`asr-worker.js` (transformers.js WebGPU Whisper) exist but are never imported into the caption path. So "transcript shown instantly?" is **No** — captions wait for a full ZeroGPU round-trip. Wire `onInterim` → `_segTo16k()` → the existing in-browser worker → a **non-destructive** `#transcript` span, reconciled against the server transcript on return. Zero new infra, zero GPU cost, zero ZeroGPU risk. This is the real answer — **not FastRTC.**
|
| 134 |
+
|
| 135 |
+
## Reject FastRTC for the core loop (verified-negative)
|
| 136 |
+
|
| 137 |
+
Three findings converge against FastRTC as the headline: its `ReplyOnPause` primitive is **pause-triggered, not partial-during-speech** (it would not beat your existing client-side interim loop); WebRTC needs one **long-lived peer connection**, which fights ZeroGPU's **fork-per-call / kill-on-release** model (no published pattern of a FastRTC handler driving `@spaces.GPU` diffusion); and Spaces WebRTC requires a **Cloudflare TURN** dependency that can fail behind NATs live. The latency math seals it: WebRTC only shaves audio upload (~100–160 KB), a rounding error against klein+depth GPU time. Document this as a "considered and rejected" entry in `VERIFY_LOG` — a crisp negative on a trendy library reinforces the honesty pillar judges weight.
|
| 138 |
+
|
| 139 |
+
## Make it doubly undeniable + legible
|
| 140 |
+
|
| 141 |
+
- **Name the backbone in the video and About overlay:** "Built on `gr.Server` (Gradio Server mode, shipped April 2026) — custom three.js frontend on Gradio's backend: live SSE + queue + per-call ZeroGPU quota via `@gradio/client`." The win is in the code but invisible to a video-only judge; one line claims the Off-Brand badge.
|
| 142 |
+
- **Dual-surface the same backend as an MCP server** (`@app.mcp.tool()` stacked over `scroll_live`/`transcribe` — a `gr.Server`-exclusive surface): "an agent paints a continuous world by speaking." Additive, low-risk, opens the agentic-app category.
|
| 143 |
+
- **Add a `gradio_client` repro snippet** to the README so judges can call the live, quota-attributed pipeline programmatically — proving the endpoints are real.
|
| 144 |
+
|
| 145 |
+
**Sequencing:** (1) pin `sdk_version` + boot-assert [S, blocking]; (2) wire interim captions [S]; (3) name the gr.Server backbone in video/About [S]; (4) register the live-path style LoRA in the ledger before leaning on the honesty pillar [S]; then (5) build + publish `gradio-voicescroll` as the headline innovation [L], deliberately timed against privacy mode.
|
| 146 |
+
==========================================================================================
|
| 147 |
+
THEME: Sponsors, Modal & Fine-tuning
|
| 148 |
+
VERDICT: Today the sponsor story is thin-to-false: the HOST sponsor (Cohere) runs ZERO live weights, the OpenAI-track claim is a lie (zero Codex-attributed commits), and Modal does one throwaway LoRA that isn't even pinned to the runtime it loads into. With ~48h left, the winning play is NOT more features — it is (1) make Cohere load-bearing on every beat via the obsolete-excuse-killing transformers 5.12 path, (2) fine-tune Klein to PAINT its own depth (joint RGB+depth LoRA) as the headline BFL innovation that attacks depth at the source, and (3) turn Modal from a LoRA toy into a warm GPU microservice that does the one thing ZeroGPU physically cannot. Everything else is a distraction.
|
| 149 |
+
|
| 150 |
+
TOP RECOMMENDATIONS:
|
| 151 |
+
[high/M] Make Cohere load-bearing on EVERY beat: bump transformers to 5.12.x, load CohereLabs/cohere-transcribe-03-2026 via the NATIVE CohereAsrForConditionalGeneration (NO trust_remote_code), and replace whisper-large-v3-turbo as the runtime ASR. Add Tiny Aya (lazy, GPU bf16, non-English beats only) as the translator Cohere Transcribe lacks. This IS Cohere's own recommended Transcribe->Tiny Aya pipeline.
|
| 152 |
+
WHY: On a Cohere-HOSTED hackathon, shipping zero live Cohere weights while the README advertises 'Cohere: Transcribe (+Aya)' is both the weakest sponsor story possible AND a truthfulness violation on a scored axis. The 'incompatible/emits garbage' excuse in asr.py:1-8 is obsolete: CohereAsr is native in transformers v5.12 (the garbage came from the OLD remote-code path on the 5.11 pin). One local verification (transcribe an EN and an ES wav) decides it; if it loads clean, the host sponsor becomes load-bearing on every spoken phrase AND ASR quality jumps (5.42% vs 7.44% WER, #1 Open ASR leaderboard).
|
| 153 |
+
[high/L] PRIMARY fine-tune: train a JOINT RGB+DEPTH 'paint-the-depth' LoRA on FLUX.2-klein-base-4B (rank-16) whose target is a [RGB | sharp-depth] 2-up tile, labeling the depth half with Depth Pro / DAv2-Large (the SHARP teacher), via the diffusers img2img dreambooth path. The painter EMITS depth aligned to its own brushwork and seam-consistent strip-to-strip; crop the depth half at inference.
|
| 154 |
+
WHY: This attacks the #1 unsolved problem (flat/coarse depth + strip-seam shear) at the SOURCE and is the single most BFL-defensible result: 'we made the FLUX authors' own step-distilled Klein co-generate geometry.' It removes the separate DepthAnything model from the runtime, is grounded in 2024-26 joint-RGBD work (JointNet/JointDiT/GeoVideo/IDC-Net), and reuses the EXACT Modal self-distill+fuse pipeline already proven. Cheap (~1h L40S/run, ~$2). Main risk is the 2-up tiling and validating depth survives 4-step distillation — hence the SHIP-FIRST fallback below.
|
| 155 |
+
[high/M] SHIP-FIRST depth fix (no training, lands inside 48h): layered-depth + Klein disocclusion inpaint. Threshold DepthAnything into 2-3 layers at depth discontinuities, compute the disocclusion sliver the near layer uncovers at max lean, run ONE extra Klein inpaint per strip to hallucinate the hidden background, and render 2-3 alpha-composited parallax planes in stage-scroll.js.
|
| 156 |
+
WHY: The REAL root cause of all three rejected renderers is not depth contrast — it is that monocular depth gives no occluded content, so any real parallax tears, vibrates, or must be neutered to flat (renderer #3, what shipped). The classic LDI/3D-Photo-Inpainting/SLIDE pipeline solves this by pre-inpainting the hidden background ONCE. You already own the inpainter (Klein is your right-edge outpainter). Zero new model, zero training risk, converts the 'too subtle' lean into bold HOLE-FREE parallax. Highest impact-per-effort; do this before the joint-RGBD LoRA.
|
| 157 |
+
[high/L] Turn Modal from a LoRA toy into a warm 'Deep Loom' GPU microservice (A100-80/L40S, @modal.asgi_app + min_containers=1 + CPU memory snapshot) that takes a finished strip PNG and returns BOTH a sharp Depth-Pro/MoGe depth map AND a feed-forward 3D Gaussian splat (.ply), streamed async into the browser via World Labs Spark (THREE.js). The live ZeroGPU path keeps painting; geometry upgrades as splats arrive.
|
| 158 |
+
WHY: This is the architecturally HONEST Modal play: run exactly what ZeroGPU physically cannot (per-strip 3DGS needs fp32/persistent state/>120s — impossible on a bf16/120s/fork-per-call slot). It directly fixes depth, is on-brand (FLUX painter feeds a 3D loom), and is distinctive vs any Voice-Scroll clone. Budget is a non-issue: $200 = ~80 A100-h; feed-forward image->splat is ~3s/strip (Flash3D/Splatter Image, 2025). Pursue ONLY after the no-train LDI fix ships; this is the wow-ceiling stretch, not the safe floor.
|
| 159 |
+
[medium/S] Fix the Modal LoRA harness reproducibility + integration BEFORE retraining anything: pin diffusers AND the fetched trainer script to the runtime's exact commit (784fa62...), pin transformers/peft/accelerate exactly, make validate/stylegrid/covers fuse at the PRODUCTION lora_scale (read LIGHTLOOM_STYLE_LORA_SCALE) and add one render at the REAL live strip geometry (~384x512 partial-mask outpaint), and assert the pushed adapter basename.
|
| 160 |
+
WHY: The adapter is trained against unpinned diffusers HEAD (launch.py:26) but fused into a pinned runtime (requirements.txt:11 @784fa62) — the artifact is non-reproducible and may not compose. Worse, the 'proof' grids fuse at scale 1.0 while the lora.py comment claims 0.75 and production reads an env var, so the validation exercises a config production never uses. Any new fine-tune inherits this rot; fix the harness first or the joint-RGBD LoRA lands unreproducible too.
|
| 161 |
+
[high/M] Adopt MiniCPM-V 4.6 (1.3B, transformers-5.11-native, AutoModelForImageTextToText, +0.3B) as a CADENCED 'Seeing Director': at world-start it reads the cover strip to lock palette/style, and every Nth beat (or on detected drift) it inspects the last painted strip and steers continuity from pixels — closing the open-loop 'always lighthouses' drift with perception, not more prompt heuristics.
|
| 162 |
+
WHY: Same-vendor (OpenBMB) upgrade that makes the Director genuinely load-bearing and gives a branded line BFL respects: 'the language model that writes the next FLUX shot does so by VISUALLY INSPECTING the previous FLUX strip.' It also props the OpenBMB award and quietly corrects a honesty hole: README sells 'Director picks the style' but app.py:839 picks RANDOM by default. Run it on a cadence to protect the 120s cap; measure latency in a G-gate before committing to the live path.
|
| 163 |
+
[medium/S] Do NOT chase: a MiniCPM Director fine-tune (100% JSON already, pure regression risk), a standalone depth-refinement or disocclusion-fill net (second GPU call inside the 120s fork; disocclusion-fill solves the REJECTED renderers' tear), Cohere Embed/Rerank or any Command model (API-only / 7-112B / blow the budget+Off-the-Grid), GPU memory snapshots (alpha, don't speed weight load, break under torch.compile), Modal Sandboxes for dataset gen (use Functions+.map()), and a nightly Cron (low-wow, burns credit).
|
| 164 |
+
WHY: Each looks advanced but is a dead end with <48h. Naming them as evaluated-and-rejected (esp. MiniCPM-o 4.5 omni: right capability, wrong 9B/transformers-5.x-fragile footprint) is itself a credibility flex on the honesty axis and pre-empts 'why didn't you use X?' from judges. Concentrate the budget on depth + Cohere + the OpenAI-track unblock.
|
| 165 |
+
|
| 166 |
+
BOLD MOVES:
|
| 167 |
+
* 'Lightloom paints in depth' — fine-tune Klein to co-generate a painterly depth channel in the SAME 4-step pass, so the painter emits geometry by construction aligned to its brushwork and seam-consistent. Brandable, novel, and it deletes the separate depth model from the runtime. This is the headline BFL result.
|
| 168 |
+
* Two-tier honest architecture: keep the LIVE on-device painter on ONE ZeroGPU slot (the 'Build Small' story) and offload the ONE thing ZeroGPU physically cannot do — real per-strip 3DGS — to a warm Modal 'Deep Loom' endpoint that streams sharp depth + .ply splats into a World Labs Spark (THREE.js) fly-through. The two-tier split is JUSTIFIED by a measured impossibility, not gratuitous.
|
| 169 |
+
* Run the host sponsor's OWN reference pipeline verbatim and say so: Cohere Transcribe (#1 Open ASR, beating Whisper) -> Tiny Aya translate -> MiniCPM Director -> FLUX.2-klein painter, fully on-device, and openly note you corrected an obsolete compatibility claim by reading the transformers v5.12 source instead of guessing.
|
| 170 |
+
* Brand a citable artifact on a NAMED published gap: 'Painterly Layer-Depth' — DepthAnything/DepthPro are provably TRAINED to ignore painterly media (Illustrator's Depth, arXiv:2511.17454, Nov 2025). Self-label from your own pipeline and train a tiny layer-index model tuned to YOUR 22 styles. A real, defensible Field Notes paper, not a generic 3D-photo wiggle.
|
| 171 |
+
* Pair a VERIFIED NEGATIVE (Klein adaLN scalar-broadcast forbids per-column LoRA streaming; FLUX reference conditioning is appearance- not view-consistent — no camera term) with a VERIFIED POSITIVE (here is the measured Modal cost/latency to do real 3D off the slot). That rigor-pairing is exactly what the FLUX authors reward over a clone.
|
| 172 |
+
|
| 173 |
+
RISKS:
|
| 174 |
+
- TIME WALL: submission is June 15; it is June 13. The joint-RGBD LoRA and the Modal splat service are both L-effort — they can blow the window. The no-train layered-depth + Klein-inpaint fix (M) is the only depth move guaranteed to land; the LoRA and splat service must be explicitly sequenced AFTER it as stretch, not the floor.
|
| 175 |
+
- OpenAI-track claim is currently FALSE and a stated disqualifier: README:83 says 'built with Codex' but 15/15 commits are Efrain-authored, no Codex trailer, no public remote. Either re-author the uncommitted tree via Codex on a fresh public repo (Codex makes the commits — Claude-authored commits do NOT count) with a one-line trailer carve-out, or DELETE the claim. Shipping it as-is risks the honesty score that gates the whole entry. This is a blocking human decision (the no-trailer hook conflicts with the only attribution Codex emits).
|
| 176 |
+
- Cohere Transcribe being transcribe-only BREAKS the multilingual->English path unless Tiny Aya ships as the translator — so the two Cohere models are coupled; landing only Transcribe regresses non-English demos. Tiny Aya is CC-BY-NC-4.0: must be declared loudly (non-commercial research demo under Cohere AUP) or the NC license sinks the honesty score.
|
| 177 |
+
- Joint-RGBD depth may not survive 4-step distillation, and the model could trade RGB fidelity for the depth half. Validate depth quality through distillation on a tiny set BEFORE committing the live path; keep the RefControl-style depth-conditioning LoRA (proven on Klein via thedeoxen/refcontrol) as the offline fallback.
|
| 178 |
+
- Modal harness is NOT reproducible (unpinned diffusers HEAD vs pinned runtime 784fa62) and validates the WRONG fuse scale — any new fine-tune inherits this and may not compose into the live pipeline. Fix the harness before retraining or the headline LoRA ships broken.
|
| 179 |
+
- Model-id integrity: the audit flags openbmb/MiniCPM5-1B and FLUX.2-klein-4B SHAs as needing live-Hub resolution. If any repo_id/SHA in preflight.json doesn't resolve, the 'verifiable at /health' ledger is fabricated verification — fatal with OpenBMB/BFL on the panel. Re-run preflight against the live Hub before submission.
|
| 180 |
+
- A warm Modal endpoint (min_containers=1) idles credit; only hold it warm during the recorded demo and use scaledown_window. ZeroGPU daily-quota exhaustion mid-demo remains a live failure mode the Modal HQ-fallback tier can cover — but present it honestly as a 'cinema render' fallback, not as cheating the on-device claim.
|
| 181 |
+
|
| 182 |
+
REPORT SECTION:
|
| 183 |
+
## Sponsors, Modal & Fine-tuning
|
| 184 |
+
|
| 185 |
+
### The hard truth first
|
| 186 |
+
|
| 187 |
+
Three sponsor surfaces are not just thin — they are currently *false*, and on an honesty-judged, Cohere-hosted event with the FLUX authors on the panel, false is worse than absent:
|
| 188 |
+
|
| 189 |
+
1. **Cohere (HOST sponsor) contributes ZERO live weights.** `asr.py:1-8` ships an obsolete excuse ("Cohere Transcribe is incompatible / emits garbage"); the runtime is Whisper-turbo and Tiny Aya is `runtime=False`. The README still advertises "Cohere: Transcribe (+Aya)". That is the weakest possible host-sponsor story *and* a truthfulness violation.
|
| 190 |
+
2. **The OpenAI-track claim is a lie.** `README:83` says "built with Codex"; 15/15 commits are Efrain-authored with no Codex trailer and no public remote. The project's own R7 names this the *only* disqualifier.
|
| 191 |
+
3. **Modal is a throwaway.** It trains one rank-16 LoRA against *unpinned* diffusers HEAD (`launch.py:26`) while the runtime pins `@784fa62` — the artifact is non-reproducible and may not even compose into the pipeline that fuses it.
|
| 192 |
+
|
| 193 |
+
Do not paper over these. Fix or delete each. The rest of this section is how to convert them from liabilities into the strongest part of the entry.
|
| 194 |
+
|
| 195 |
+
### Cohere — make the host sponsor load-bearing on every beat (M, do first)
|
| 196 |
+
|
| 197 |
+
The "incompatible" excuse is obsolete. `CohereAsrForConditionalGeneration` is **native** in transformers ≥5.4 (the garbage came from the old `trust_remote_code` path on the 5.11 pin). The move:
|
| 198 |
+
|
| 199 |
+
- Bump to **transformers 5.12.x**, load `cohere-transcribe-03-2026` via the built-in class (no remote code), and **replace Whisper as the runtime ASR**. One local test — transcribe one EN and one ES wav with `language=` set — decides the whole integration. It is a strict quality upgrade (5.42% vs 7.44% WER, #1 Open ASR leaderboard, ~3× throughput), same 16 kHz pipeline, same bf16 slot.
|
| 200 |
+
- Cohere Transcribe is **transcribe-only**, which *breaks* the non-English→English path Whisper's `task='translate'` did. That gap is exactly why **Tiny Aya** (3.35B, lazy-loaded on GPU, non-English beats only) becomes the translator — turning *both* Cohere models load-bearing and implementing **Cohere's own recommended Transcribe→Tiny Aya pipeline verbatim**. Declare Tiny Aya's **CC-BY-NC-4.0** loudly (non-commercial research demo under Cohere's AUP), keep both as local weights (Off-the-Grid preserved), and never touch the hosted API.
|
| 201 |
+
|
| 202 |
+
The honest hook for judges: "we run Cohere's own reference pipeline on-device, and we corrected an obsolete compatibility claim by reading the transformers v5.12 source instead of guessing."
|
| 203 |
+
|
| 204 |
+
### Best model to fine-tune — paint the depth (L, headline; with an M floor)
|
| 205 |
+
|
| 206 |
+
Depth is the #1 unsolved problem, and the root cause is **not contrast — it is missing occluded content**. All three rejected renderers failed for one reason: monocular depth gives a single front surface, so real parallax tears (#1), vibrates (#2), or must be neutered to flat (#3, what shipped). Two layers of fix, sequenced by risk:
|
| 207 |
+
|
| 208 |
+
- **Floor (no training, M, ship this first):** layered-depth + Klein disocclusion inpaint. Threshold DepthAnything into 2-3 layers at discontinuities, compute the disocclusion sliver the near layer uncovers at max lean, run **one extra Klein inpaint per strip** to hallucinate the hidden background, and render 2-3 alpha-composited parallax planes. You already own the inpainter; this is the classic LDI/3D-Photo-Inpainting/SLIDE pipeline executed live, and it converts the "too subtle" lean into bold, **hole-free** parallax with zero training risk.
|
| 209 |
+
- **Headline (L, the BFL flex):** a **joint RGB+depth "paint-the-depth" LoRA** on `klein-base-4B` (rank-16) whose target is a `[RGB | sharp-depth]` 2-up tile, with the depth half labeled offline by **Depth Pro / DAv2-Large** (the sharp teacher; DepthAnything-V2-*Small* is explicitly the soft, low-contrast variant). The painter then **emits depth aligned to its own brushwork and seam-consistent strip-to-strip** — killing the strip-seam shear at the source and removing the separate depth model from the runtime. This is grounded in 2024-26 joint-RGBD work (JointNet, JointDiT, GeoVideo, IDC-Net), reuses the exact Modal self-distill+fuse pipeline, costs ~$2/run, and is the most defensible "we made the FLUX authors' own distilled model do something new" result available. **Validate that depth survives 4-step distillation on a tiny set before wiring it live**; if it doesn't hold, fall back to the RefControl-style depth-*conditioning* LoRA (proven on Klein via `thedeoxen/refcontrol`) used offline to bake sharp depth into the cover variants.
|
| 210 |
+
|
| 211 |
+
Do **not** fine-tune the MiniCPM Director (100% JSON already — pure regression risk) or build a standalone depth-refiner/disocclusion-fill net (second GPU call inside the 120s fork; disocclusion-fill only matters for the renderers you rejected).
|
| 212 |
+
|
| 213 |
+
### Best additional Modal use — run what ZeroGPU physically cannot (L, stretch)
|
| 214 |
+
|
| 215 |
+
Budget is a non-issue ($200 ≈ 80 A100-h; the LoRA proved the account works). The honest, on-brand play is **not** to duplicate the live painter — it is to offload the *one* thing ZeroGPU cannot do. ZeroGPU is a bf16-only, 120s, fork-per-call slot; per-strip 3DGS needs fp32/persistent state/>120s. So stand up a warm **"Deep Loom"** microservice (`@modal.asgi_app`, `min_containers=1`, CPU memory snapshot, A100-80/L40S) that takes a finished strip PNG and returns **both** a sharp Depth-Pro/MoGe depth map **and** a feed-forward 3D Gaussian splat (`.ply`, ~3s/strip per Flash3D/Splatter Image 2025), streamed asynchronously into a **World Labs Spark** (THREE.js, already your renderer stack) fly-through. The live path keeps painting; geometry upgrades as splats arrive. The two-tier split is *justified by a measured impossibility*, which is itself the systems contribution: pair the verified negative (Klein adaLN scalar-broadcast forbids per-column LoRA streaming; FLUX reference conditioning is appearance- not view-consistent) with a verified positive (here is the Modal cost/latency to do real 3D off the slot). Use CPU snapshots, not GPU snapshots (alpha, don't speed weight loading, break under `torch.compile`), and Functions+`.map()` for any dataset fan-out, not Sandboxes. **Sequence this strictly after the no-train depth fix** — it is the wow ceiling, not the floor, and it can blow the 48h window.
|
| 216 |
+
|
| 217 |
+
### Before retraining anything — fix the harness (S)
|
| 218 |
+
|
| 219 |
+
Pin diffusers **and** the fetched trainer script to the runtime commit `784fa62`, pin transformers/peft/accelerate exactly, make `validate/stylegrid/covers` fuse at the **production** `lora_scale` (the grids currently fuse at 1.0 while the comment claims 0.75 and production reads an env var — they prove nothing about the shipped config), add one render at the real **~384×512 partial-mask** strip geometry, and assert the pushed adapter basename. Any new fine-tune inherits this rot otherwise.
|
| 220 |
+
|
| 221 |
+
### One same-vendor multiplier (M)
|
| 222 |
+
|
| 223 |
+
Adopt **MiniCPM-V 4.6** (1.3B, transformers-5.11-native, +0.3B) as a **cadenced "Seeing Director"**: at world-start it reads the cover strip to lock palette/style; every Nth beat it inspects the last painted strip and steers continuity from pixels — closing the open-loop drift with perception, not more prompt heuristics. Branded line: "the model that writes the next FLUX shot does so by visually inspecting the previous FLUX strip." It also props the OpenBMB award and corrects the honesty hole that the README sells Director-picked style while `app.py:839` picks **random** by default. Measure latency in a G-gate first.
|
| 224 |
+
|
| 225 |
+
### Sequencing under the 48h wall
|
| 226 |
+
|
| 227 |
+
1. Cohere Transcribe verification + swap (decides the host-sponsor story) — **today**.
|
| 228 |
+
2. No-train layered-depth + Klein-inpaint (the depth floor that *will* land).
|
| 229 |
+
3. Resolve the OpenAI-track decision (re-author via Codex on a fresh public repo, or delete the claim) — blocking, human.
|
| 230 |
+
4. Harness pins + Tiny Aya translator + MiniCPM-V cadence.
|
| 231 |
+
5. **Stretch only:** joint-RGBD LoRA, then the Modal Deep Loom splat service.
|
| 232 |
+
==========================================================================================
|
| 233 |
+
THEME: Real-time UX, captions & fluidity
|
| 234 |
+
VERDICT: WEAK as shipped — but the fix is mostly deletion + wiring, not a build. The headline feature the user asked for (words as you speak) is 100% dead code, and the live path is architecturally serialized so captions trail the voice by 5–15s. Two non-negotiable moves: (1) fire the client caption ONCE per closed VAD clause on Moonshine-tiny, killing the O(n²) bug by construction; (2) split words and paint into two clocks so the transcript never queues behind the painter. Do both and the demo goes from "captions lag, world jerks" to "caption leads the paint, world flows."
|
| 235 |
+
|
| 236 |
+
TOP RECOMMENDATIONS:
|
| 237 |
+
[high/S] Wire the interim caption on CLOSED VAD clauses only: delete the 1.1s growing-buffer tick (recorder.js:125-131), call onInterim(this._segTo16k()) ONCE inside _cutLiveSegment (recorder.js:168) on the final merged clause, and in controller.js _initRecorder add onInterim -> render to #transcript in a non-destructive 'is-interim' span. Import asr.js/asr-worker.js (absent from index.html), call LL.asr.init() in _prewarm, and let the existing server 'transcript' event (controller.js:272) overwrite the span in place.
|
| 238 |
+
WHY: This IS the user's #1 wanted feature and it is fully built but never imported. One transcribe per clause (never over a growing buffer) makes the O(n^2) bug structurally impossible — the exact symptom that got client ASR reverted. Zero server round-trip, zero GPU quota. Highest impact-per-line in the report.
|
| 239 |
+
[high/S] Swap the browser model from onnx-community/whisper-base.en (asr-worker.js:11) to onnx-community/moonshine-tiny-ONNX (same pipeline('automatic-speech-recognition', id) call, one string + dtype/device test). Keep WebGPU->WASM fallback; run it on device:'wasm' so a ~1s ASR forward never steals GPU frame-time from the three.js scroll.
|
| 240 |
+
WHY: Whisper ALWAYS pads to a 30s mel window, so every short clause pays a fixed forward-pass tax — the heaviness that made whisper-base feel laggy. Moonshine is linear in actual audio (no 30s pad), ~34-50ms vs 277ms, lower WER, and its provisional->finalized streaming encoder gives stable interims natively. A defensible 2024-2026 SOTA choice (transformers.js author-endorsed) judges recognize.
|
| 241 |
+
[high/L] Decouple words from paint into two clocks. Lane A: each closed clause hits ASR-only and appends pending-NNN.txt to the on-disk session queue (~200ms). Lane B: ONE long-lived self-clocked @spaces.GPU painter generator per mic session that pops pending lines, paints one strip, yields, and re-grants on the duration cap (disk-carried continuity already makes it stateless across forks). DELETE the serial _drainLive await-loop (controller.js:251-265) and stop hardcoding concurrency_limit=1 now that ZeroGPU runs handlers concurrently by default.
|
| 242 |
+
WHY: Today the transcript inherits the painter's latency: a clause spoken while painting is invisible until the single GPU generator frees (5-15s behind voice). Splitting lanes drops caption latency to ~one in-flight strip (~0.8s), ~0 with the client caption in front. This is the load-bearing fix, fully specified with line numbers in the project's own PROJECT_EVALUATION.md but NOT yet implemented in app.py/controller.js.
|
| 243 |
+
[medium/S] Decouple depth from the visible strip. Drop the '!s.depth -> return' gate in controller.js _tryAdd (line 325); render the section flat the instant it lands and hot-swap depth on the later 'depth' event (the renderer already accepts depthUrl:null). Server-side, overlap estimate_depth(strip k) with paint(strip k+1) on a second CUDA stream — DepthAnything-Small (25M) hides almost entirely behind the 4B klein forward.
|
| 244 |
+
WHY: Right now a <100ms depth call gates BOTH the visible paint AND the next strip's start inside one serial generator (app.py:1187-1192 then loop). Showing the strip flat-then-popping depth advances the frontier ~100-200ms sooner per strip and lets the painter keep its clock. Low risk, pure gate removal + stream overlap.
|
| 245 |
+
[high/M] Eliminate per-section frontend hitching: cache the static bundle and stop full-strip texture re-uploads. Add 'Cache-Control: public, max-age=31536000, immutable' + ETag to the bare FileResponse routes (app.py:100-127) with content-hashed filenames, and switch stage-scroll.js from CanvasTexture+needsUpdate (full 8192x512 RGBA re-upload, ~33.6MB/section x2) to gl.texSubImage2D over only the ~384px written band (~0.8MB). Move the main-thread depth gaussian/affine off the rAF loop (Web Worker / OffscreenCanvas, or precompute server-side).
|
| 246 |
+
WHY: The most VISIBLE flaw in the live experience is the scroll jank every time a strip lands — full-strip texture re-upload plus synchronous getImageData/putImageData pixel loops block requestAnimationFrame exactly when a section arrives. Polish is a judged axis; this turns 'janks when a strip lands' into genuinely continuous. The cache headers are a free repeat-load win for judges who reload the Space.
|
| 247 |
+
[medium/S] Make prewarm cheap and pre-emptible, and make quota terminal. Add a warm_only branch to the painter that constructs pipes and returns WITHOUT painting; track the prewarm job (controller.js:59-66 fires it fire-and-forget on a 1200ms timer) and abort it the instant real audio arrives so the first phrase isn't queued behind a full warm on the single slot. On quota_exceeded during live, stop the recorder, clear _phraseQ, and route to the showcase (the copy already promises it) instead of silently re-firing into an exhausted quota.
|
| 248 |
+
WHY: Prewarm currently can serialize the user's first real phrase behind a full model-loading warm on concurrency-1, AND a continuous talker outruns the GPU forever with an unbounded _phraseQ and no backpressure. Both directly attack the 'instant first phrase' goal. Bound the queue (drop-oldest, continuity favors recency) and make quota a terminal state.
|
| 249 |
+
|
| 250 |
+
BOLD MOVES:
|
| 251 |
+
* WET-INK CAPTION bound to a two-clock pipeline: interim words appear instantly as luminous 'wet ink' (client Moonshine, ~34ms, 100% on-device, no cloud) writing onto the LEADING RIGHT EDGE of the unrolling scroll — the exact place the next strip will be painted — then 'dry' to crisp when server whisper-turbo finalizes. Each finalized clause is spatially ANCHORED to the strip of world it generated and drifts left with the scroll. This makes speech->world causality physically VISIBLE and turns the standard interim->final ASR pattern into a branded, hard-to-clone interaction.
|
| 252 |
+
* Ship the 'two clocks' as the honest systems contribution, not just plumbing. Surface a dual-track HUD ('you said / I'm painting') where the caption clock and the paint frontier are visibly independent, with an indicator that words are NEVER queued behind paint. Write it up as 'control-decoupled live latent painting on a single stateless serverless GPU fork' — the StreamDiT / Rolling Forcing / StreamDiffusionV2 control-decoupling principle ported to a NEW regime (real-time painterly horizontal outpaint under fork-per-call, 120s cap, concurrency). Back it with a measured latency table (646ms/strip x M, ASR ~200ms interleaved, time-to-first-caption ~0) and pair it with the already-verified NEGATIVE result (klein adaLN scalar-broadcast => no per-column LoRA streaming). That reads as rigorous engineering, exactly what BFL rewards over a Voice-Scroll clone.
|
| 253 |
+
* Stabilized, animated captions per Google's 123-user live-caption study: diff interim tokens into add/insert/correct, only repaint a prior token when semantically different (similarity < 0.85), fade new tokens in (translateY+opacity ~250ms). Naive full-string swap (current #transcript behavior) flickers and looks broken; stabilized captions beat raw on 5 of 6 UX measures (p<0.01). This is what separates 'words materialize fluidly' from 'subtitle bar flickers'.
|
| 254 |
+
* Make the world DEVELOP out of depth instead of a 3D-photo: a screen-space depth-scan reveal (Codrops 2025) sweeps just ahead of the newest strip — near-depth pixels resolve/sharpen first, far-depth lingers in a faint procedural halftone, synced to voice cadence. It is screen-space so it physically CANNOT tear or disocclude (killing the failure mode of all three rejected renderers) and reframes coarse DepthAnything-Small depth as a narrative reveal beat �� a branded signature, not a commodity parallax.
|
| 255 |
+
|
| 256 |
+
RISKS:
|
| 257 |
+
- Two-lane decoupling is the only Large-effort item and touches the live path's core: a long-lived self-clocked painter that re-grants across ZeroGPU forks. It depends on disk-carried continuity (frame+depth PATHS, never CUDA tensors — ZeroGPU refuses to unpickle them) and on never assuming the warm worker survives a scheduler reassignment. Get the generation-token fencing right or stale strips splice into a wiped world (controller.js section/depth handlers have NO _liveOn guard today; reset() can't even cancel an in-flight streamScrollLive because its job is never stored in _job).
|
| 258 |
+
- Moonshine swap is one string but unproven in THIS tree: verify dtype/device load, WASM real-time on a mid laptop, and that the WebGPU path doesn't contend with the WebGL scroll (run it on device:'wasm' to be safe). transformers.js model-load gates exist (60s init) but a failed download must fall back silently to no-interim, not break the mic.
|
| 259 |
+
- Raising concurrency_limit above 1 assumes the 2025-2026 ZeroGPU concurrent-by-default model AND that module-scope pipelines are read-only under concurrent inference. Confirm no shared-state mutation before lifting the limit, and right-size every @spaces.GPU duration (several declare 160-200s; over-declaring lowers queue priority AND can pre-emptively fail with 'quota exceeded' during a live judged demo).
|
| 260 |
+
- Caption divergence risk: the interim (client Moonshine) text and the final (server turbo) text MUST converge on the same words that drive painting, or the judge sees the caption say one thing and the world paint another. Atomically replace interim with the server final per clause; never let the two paths drift.
|
| 261 |
+
- Do NOT adopt FastRTC or default to Web Speech API. FastRTC needs a persistent server handler + Cloudflare TURN on firewalled Spaces (fights ZeroGPU fork-per-call) and ships a stock Gradio Stream component (violates the no-stock-components rule). Web Speech API streams audio to Google's cloud by default — a direct contradiction of the '100% local / off-the-grid' brand and privacy posture. Both add risk for zero advantage over the layered local design; Web Speech is at best an optional third tier behind a 'cloud assist' toggle, never authoritative.
|
| 262 |
+
|
| 263 |
+
REPORT SECTION:
|
| 264 |
+
# Real-time UX, Captions & Fluidity
|
| 265 |
+
|
| 266 |
+
## Verdict: weak as shipped — and the gap is wiring, not capability
|
| 267 |
+
|
| 268 |
+
The single feature the user explicitly asked for — interim text appearing **as you speak** — is fully built and **100% dead code**. Verified in the tree: `recorder.js` exposes an `onInterim` hook and fires it (`recorder.js:125-131`), `asr-worker.js` runs in-browser ASR, and `asr.js` wraps it — yet `index.html` never imports `asr.js`/`asr-worker.js`, and `controller.js._initRecorder()` (`controller.js:155-177`) wires `onState`/`onSegment`/`onTranscript` but **not** `onInterim`. The only caption today is the server `transcript` event echoed at `controller.js:272`, which lands *after* a full ZeroGPU round-trip. The honest answer to "is the transcript shown instantly?" is **no**.
|
| 269 |
+
|
| 270 |
+
Worse, even when wired the current interim path is the *wrong* one. `recorder.js:125-131` re-fires `onInterim(this._segTo16k())` every ~1.1s over a **growing** segment buffer. Whisper has no incremental cache and pads every input to a fixed 30s mel window, so a 5s clause ticked 5 times = 5 full padded forward passes — work grows **quadratically** with clause length. That is the documented lag + 3-word-fragment symptom that got client ASR reverted. The correct hook already exists one function down: `_cutLiveSegment()` (`recorder.js:168`), which fires **once** on the closed clause.
|
| 271 |
+
|
| 272 |
+
## The two non-negotiable moves
|
| 273 |
+
|
| 274 |
+
**1. Caption once per closed clause, on Moonshine.** Delete the 1.1s growing-buffer tick; call `onInterim(this._segTo16k())` exactly once inside `_cutLiveSegment()`; wire `onInterim` in `_initRecorder` to render a non-destructive `is-interim` span in `#transcript`; let the server `transcript` event overwrite it in place. One transcribe per clause, never over a growing buffer — this kills the O(n²) bug *by construction*, not by tuning. Then swap `asr-worker.js:11` from `onnx-community/whisper-base.en` to `onnx-community/moonshine-tiny-ONNX` (same `pipeline('automatic-speech-recognition', id)` call). Moonshine is **linear in actual audio** (no 30s pad tax), ~34–50ms vs Whisper's ~277ms, lower WER, and its provisional→finalized streaming encoder gives stable interims natively. Run it on `device:'wasm'` so a ~1s ASR forward never steals frame-time from the three.js scroll. This is **S** effort and the highest impact-per-line in the whole report.
|
| 275 |
+
|
| 276 |
+
**2. Words and paint are two clocks, not one.** The transcript inherits the painter's latency because words and paint share one serialized GPU generator. `_scroll_live_phrase` (`app.py:1149-1192`) runs ASR → Director-once → paint 2 strips → depth **serially** under one `@spaces.GPU(concurrency_limit=1)`, and the client `_drainLive` loop (`controller.js:251-265`) awaits each phrase's *full* paint before starting the next. A clause spoken while painting is invisible for 5–15s. Split it: **Lane A** = ASR-only per closed clause (~200ms) appending `pending-NNN.txt` to the on-disk session queue; **Lane B** = one long-lived, self-clocked painter generator that pops pending lines, paints one strip, yields, and re-grants on the duration cap (disk-carried frame/depth **paths** — never CUDA tensors — already make it stateless across forks). Delete the serial `_drainLive` await-loop and stop hardcoding `concurrency_limit=1` now that ZeroGPU runs handlers concurrently by default. Caption latency drops from "phrase-painting-time" to ~one in-flight strip (~0.8s), and ~0 with the client caption in front. This is **L** effort and load-bearing — and it is already fully specified with line numbers in the project's own `PROJECT_EVALUATION.md`, just never implemented.
|
| 277 |
+
|
| 278 |
+
## Make the flow *feel* continuous
|
| 279 |
+
|
| 280 |
+
Three cheaper wins compound the architectural fix:
|
| 281 |
+
|
| 282 |
+
- **Unblock depth.** `controller.js:325` (`if (!s.depth && !force) return`) makes a <100ms depth call gate the *visible* strip and the *next* strip's start. Render the section flat the instant it lands (the renderer already accepts `depthUrl:null`) and hot-swap depth on the later event. Server-side, overlap `estimate_depth(strip k)` with `paint(strip k+1)` on a second CUDA stream — DepthAnything-Small (25M) hides behind the 4B klein forward.
|
| 283 |
+
- **Kill the per-section hitch.** `stage-scroll.js` re-uploads the entire 8192×512 RGBA strip for *both* image and depth textures via `CanvasTexture.needsUpdate` (~33.6MB/section) and runs the depth gaussian + affine match synchronously on the rAF thread. Switch to `gl.texSubImage2D` over only the ~384px written band (~0.8MB) and move the pixel loops to a Worker or compute them server-side (the depth array is already numpy). This is the most *visible* flaw in the live demo and polish is a judged axis.
|
| 284 |
+
- **Cache the bundle.** Static routes (`app.py:100-127`) return bare `FileResponse` with no `Cache-Control`/`ETag`. Add `max-age=31536000, immutable` with content-hashed filenames — a free repeat-load win for judges who reload the Space.
|
| 285 |
+
|
| 286 |
+
Also: make prewarm pre-emptible (it fires fire-and-forget on a 1200ms timer and can serialize the user's first phrase behind a full warm on the single slot), bound `_phraseQ` (a continuous talker outruns the GPU forever today), and make `quota_exceeded` terminal — stop the recorder and route to the showcase instead of silently re-firing into exhausted quota.
|
| 287 |
+
|
| 288 |
+
## The branded innovation (for BFL)
|
| 289 |
+
|
| 290 |
+
The distinctive move is not the ASR — it is a **wet-ink caption bound to a two-clock pipeline**. Interim words appear instantly as luminous "wet ink" writing onto the **leading right edge** of the scroll — the exact place the next strip paints — then *dry* to crisp when the server turbo finalizes; each finalized clause is spatially anchored to the strip of world it generated and drifts left with the scroll. This makes speech→world causality physically visible and is genuinely hard to clone because it couples streaming-ASR provisional-state semantics to the custom WebGL scroll. Render interims with stabilization (Google's 123-user study: diff tokens, repaint only when semantically different, fade new tokens ~250ms) so they *materialize* rather than flicker. Underneath, the honest systems contribution is **"words and paint are two clocks"** — the StreamDiT / Rolling Forcing / StreamDiffusionV2 control-decoupling principle ported to a new regime (real-time painterly outpaint under fork-per-call, 120s cap, concurrency), backed by a measured latency table and paired with the already-verified negative result (klein adaLN is a scalar broadcast → no per-column LoRA streaming). Fully on-device, off-the-grid — a clean privacy story Moonshine reinforces.
|
| 291 |
+
|
| 292 |
+
## What NOT to do
|
| 293 |
+
|
| 294 |
+
Do not adopt **FastRTC** (needs a persistent handler + Cloudflare TURN on firewalled Spaces, fights ZeroGPU's fork-per-call, ships a stock Gradio Stream component you've banned). Do not make **Web Speech API** the default or authoritative path (Chrome streams audio to Google's cloud by default — a direct contradiction of the "100% local / off-the-grid" brand); allow it only as an optional third tier behind a "cloud assist" toggle. Both add transport risk for zero advantage over the layered local design you already have half-built.
|
| 295 |
+
==========================================================================================
|
| 296 |
+
THEME: Audit consolidation & Closure plan
|
| 297 |
+
VERDICT: SHIP-BLOCKED, NOT BROKEN. The system runs on the deployed Space and the painter is sound at the pinned diffusers commit (the "FATAL import" audit is a local-env false alarm — verified below). But the submission FORFEITS its largest prize today: zero Codex-attributed commits + empty git remote = the $5k OpenAI track is 100% unmet with ~48h to close. Three honesty landmines (false Codex claim, fabricated/nonexistent badge names, README depth-renderer description matching code that was rejected) will be caught by the exact judges who wrote the rules. The depth problem is a POLISH lever, not a gate — stop building renderer #4. Close in this fixed order: (1) Codex history + repo, (2) honesty truth-pass + correct badges, (3) demo video, (4) one Space smoke-test of the painter, (5) the BFL-legible research artifact. Everything else is post-submission.
|
| 298 |
+
|
| 299 |
+
TOP RECOMMENDATIONS:
|
| 300 |
+
[high/M] Establish Codex-attributed git history TODAY: create a private GitHub repo, link it to the Space (private repo is explicitly allowed by the Field Guide FAQ), and route the remaining commits (docs/COMMIT_PLAN.md groups) through Codex configured for AUTHOR-level identity (GIT_AUTHOR_NAME=Codex / GIT_AUTHOR_EMAIL=codex@openai.com) with NO Co-Authored-By trailer. Never squash the existing Efrain commits; add Codex commits on top. upload_folder deploys do NOT produce Codex history — these must be real `git commit` calls.
|
| 301 |
+
WHY: The ONLY binary disqualifier for the $5k OpenAI track, currently 0% met (verified: 8/8 recent commits authored by Efrain, `git remote -v` empty). Author-level attribution is the single path that satisfies BOTH the prize rule AND the user's permanent no-Co-Authored-By rule. Highest-EV unmet item in the entire audit.
|
| 302 |
+
[high/M] Run a single honesty truth-pass and make it a hard gate (release_check.py): (a) DELETE the 'built with Codex' claim until the history above is real; (b) replace the fabricated badge list (Off-the-Grid/Field-Notes/Well-Tuned/Llama-Champion) with the REAL Field Guide badges and CLAIM the unclaimed Tiny Titan ($1.5k — every runtime model is <=4B) + Off Brand ($1.5k — custom WebGL UI ships); (c) rewrite the README depth description ONCE to match the shipped renderer (geometry-only screen-anchored depth-LEAN, static camera) and fix the stale stage-scroll.js:8 'depth-displaced MESH' header; (d) state the art style is RANDOM-per-world by default, not Director-inferred; (e) reconcile lora.py default (code 1.0 vs doc 0.75) and the '5.83B vs 10.73B' params number.
|
| 303 |
+
WHY: Honesty is the explicitly-judged moat and the judges WROTE the badge list and the FLUX architecture. Each false claim (Codex authorship, nonexistent badges, a README describing the two rejected renderers, Director-picks-style) is a direct, catchable credibility hit. These are mostly text edits with near-zero risk and high downside if left.
|
| 304 |
+
[high/M] Record the demo video over the LIVE Space (pre-warmed off-camera) as a single unbroken 'wow' shot — one evocative spoken sentence, the painterly world unrolls with the depth-lean and captions dissolving in, NO UI tour or voiceover over the magic moment. Put architecture/honesty/negative-result in a SECOND segment. Write the social post and paste video+social+repo links into the three README ⟦PLACEHOLDER⟧ slots.
|
| 305 |
+
WHY: The demo video is a REQUIRED, judged artifact and currently does not exist; it is also the primary judging surface when ZeroGPU quota is exhausted. Every winning-video source converges on hook-first/show-don't-tell, and the unrolling-world moment is Lightloom's single biggest emotional asset ('would you show a friend?').
|
| 306 |
+
[high/S] Before building anything on the painter, run ONE Space smoke-test: load_fast_pipeline() + a single outpaint_right() call on the deployed Space and paste the result image. This discharges the painter audit's 'FATAL import' finding, which I verified is a LOCAL-ENV false alarm (local has stable diffusers 0.36.0 which lacks the Klein classes; the pinned commit 784fa62 DOES export Flux2KleinInpaintPipeline + Flux2KleinKVPipeline — confirmed against the GitHub source).
|
| 307 |
+
WHY: Resolves the single scariest audit verdict ('painter broken: verdict broken'). It is NOT broken at the pinned commit, but the local env cannot test it, which is why ~6 audits independently flagged 'unverified against real pipeline.' One Space run converts that whole cluster of doubt into a green checkmark and proves mask_image/strength/guidance semantics against reality (CLAUDE rule 4).
|
| 308 |
+
[medium/M] Add the global-session depth normalization fix (the only depth code change worth doing pre-submission): kill the per-strip double-normalize (estimate_depth min-max THEN per-strip 2/98 percentile), return raw disparity and normalize every strip against a per-session EMA depth window persisted in _live_save_state. Keep renderer #3 as-is and REBRAND it as the intentional 'living horizon-lean' aesthetic. Do NOT build renderer #4.
|
| 309 |
+
WHY: Per-strip independent normalization makes cross-strip seam alignment structurally impossible (the exact tearing that killed renderer #2) and is the root cause behind 'too flat' (75-87% of each strip reads as 'near'). The global window is the highest-leverage depth fix and is disocclusion-free. A 4th novel renderer with <48h left is the highest-risk time sink in the whole audit and threatens the submission.
|
| 310 |
+
[medium/M] Ship the BFL-legible research artifact as a tight Field Notes write-up: the VERIFIED negative result (klein adaLN modulation is a scalar broadcast — single temb per sample, [B,1,D] over the sequence, cite transformer_flux2.py:1021-1036/836/910 — so LoRA-only per-column streaming is impossible by construction) PLUS the constructive corollary (the KV-cache _blend_mod_params path at lines 180-190 proves a [B,seq,D] modulation flows through, so a ~50-line forward-patch CAN do it). Position against StreamDiT (2507.03745) and Rolling Diffusion (2402.09470); cite Rokeby Voice Scroll (2023) and the IDCIoT 2024 voice->panorama paper so you NEVER lead with 'voice->panorama' (taken). Target NeurIPS 2026 ICBINB.
|
| 311 |
+
WHY: This is the most BFL-legible, most-verified artifact the project owns — an honest, source-grounded claim about THEIR flagship model's internals, which BFL explicitly 'invites scrutiny' on. It pre-empts the 'why not stream a LoRA?' question and out-impresses a Voice-Scroll clone precisely because it reports what did NOT work on the judges' model. Two independent negatives (architecture + 4-step SDEdit drift) make a stronger note than one.
|
| 312 |
+
|
| 313 |
+
BOLD MOVES:
|
| 314 |
+
* Latent-native depth on BFL's own axis: linearly probe / lightweight-decode the FLUX.2-klein VAE latent (REPA/DINOv2-aligned — BFL's #1 research thread, Nov-2025 + Mar-2026 papers; ReaLS 2502.00359 / REPA-E 2504.10483 show depth is readable from aligned latents) for a depth/disparity map, and benchmark it head-to-head against the separate DepthAnything-V2 pass. Either outcome wins: usable latent-depth drops a model from the 5.83B ledger AND proves the semantic richness BFL claims; worse latent-depth is an honest measured negative on their exact claim. If it's sharper, feed it into renderer #3 so the visible 3D becomes a BRANDED FLUX.2 innovation, not a generic 3D-photo trick. (POST-SUBMISSION — do not let it threaten the 48h freeze.)
|
| 315 |
+
* Self-compile and PUBLISH an sm120 (Blackwell) AOTI artifact for FLUX.2-klein-4B on ZeroGPU. HF's own zerogpu-aoti/FLUX.2-klein blob targets sm90 (H200) ONLY and will not load on the actual judging hardware — a working public Blackwell AOTI build of klein used in a real-time streaming-outpaint loop is something BFL has not packaged themselves, is citable/reusable, and yields ~1.3-1.8x on the per-strip paint (the gating cost of the whole live experience). Frame as deep mastery of their model on the real hardware.
|
| 316 |
+
* Rebrand the entire project around klein's own slogan — 'interactive visual intelligence, made literal.' One-line README/video thesis: Lightloom is what FLUX.2 klein was BUILT for (live, on-device, sub-second, voice-driven canvas). Add one slide: 'what here only works because it's FLUX.2 klein' (4-step CFG-free outpaint streaming, KV multi-reference continuity, latent semantics). Directly answers BFL's #1 Kontext criterion ('showcases a strength competing models could not replicate') at near-zero cost.
|
| 317 |
+
* Depth-as-ATMOSPHERE, not geometry: pair the lean with disocclusion-FREE aerial-perspective cues (depth-graded haze/fog tint, depth-modulated desaturation+contrast, scroll-tracking depth-of-field). Screen-space, geometry-only, CANNOT tear/vibrate/seam, reads as strong depth on coarse maps, and becomes a named branded look ('the Lightloom depth-of-painting'). Coarse DepthAnything-Small is well-suited to atmospheric cues (tolerate low contrast) and badly suited to hard parallax (exposes speckle) — lean into the medium's strength.
|
| 318 |
+
* Publish two HF artifacts for free badges + real research credit: flip Efradeca/lightloom-style-lora PUBLIC with a proper FLUX.2-klein LoRA card (base_model tag, trigger 'lghtlm style', before/after grid) for Well-Tuned/BFL-Krea-aesthetic alignment; and push the Director GBNF traces (beat -> JSON shot + style) as a Croissant dataset with a free HF DOI for Sharing-is-Caring + citable closure.
|
| 319 |
+
|
| 320 |
+
RISKS:
|
| 321 |
+
- DISQUALIFIER (binary, ~48h): zero Codex-attributed commits + empty git remote. Verified locally. If unfixed, the $5k OpenAI track is forfeit. upload_folder deploys can never satisfy this — must be real git commits authored as Codex.
|
| 322 |
+
- HONESTY CATCH by the people who wrote the rules: (a) README 'built with Codex' is currently false; (b) README claims badges that DO NOT EXIST in the Field Guide (Off-the-Grid/Field-Notes/Well-Tuned/Llama-Champion); (c) README describes the depth renderer two contradictory ways and BOTH match the REJECTED renderers, not the shipped lean; (d) 'Director picks the style' is false (default is SystemRandom); (e) lora.py doc says 0.75, code uses 1.0. Each is independently catchable.
|
| 323 |
+
- FABRICATED-VERIFICATION risk: config.py/preflight.json pin 'openbmb/MiniCPM5-1B' and 'FLUX.2-klein-4B' to 40-char SHAs and mark all_green:true, but preflight.json's ASR entry resolves a DIFFERENT model (CohereLabs/cohere-transcribe) than the README/config (whisper-turbo, revision=None). Re-resolve every repo_id+SHA against the live Hub (HfApi().model_info) and regenerate preflight.json BEFORE submit — a fabricated ledger is fatal with BFL/OpenBMB on the panel.
|
| 324 |
+
- SECURITY/PRIVACY at deploy: deploy ships folder_path=ROOT with a fail-open deny-list that does NOT exclude secrets/*.token/*.pem/*.key/.modal_*.log; and the deploy sets LIGHTLOOM_ENABLE_INTERNAL_BENCH=1 in PROD, exposing traceback.format_exc() + un-ledgered model loads + GPU-quota DoS to any unauthenticated caller. Switch to a `git archive HEAD` staging-dir deploy and NEVER set the bench flag in the Space. 67 uncommitted files + privacy-mode media mean a naive `git add -A` could leak private screen recordings to the new repo.
|
| 325 |
+
- PAINTER UNVERIFIABLE LOCALLY: the live painter cannot be run/tested on the dev machine (stable diffusers 0.36.0 lacks the Klein classes; the Space's pinned commit has them). This is WHY ~6 audits flagged 'unverified.' One Space smoke-test is mandatory before trusting mask_image/strength/guidance semantics or building on top.
|
| 326 |
+
- TIME-SINK TRAP: chasing user-acceptable depth via a 4th renderer, or deepening Cohere/BFL integration for prize money (neither sponsor pays cash — partners/judges only), burns the last hours. Redirect to Codex ($5k) + OpenBMB ($2.5k) + Modal credits, which actually pay and are nearly met.
|
| 327 |
+
- ROBUSTNESS gaps that surface in a live judge session: client-trusted session id (IDOR, 32-bit space), _sweep_scroll_sessions can delete the ACTIVE session mid-narration, non-atomic state.json write can silently re-roll the random style mid-session, unbounded base64 audio decode (DoS on the single GPU slot), and the live phrase queue is unbounded behind concurrency=1 so a continuous talker outruns the painter forever. None disqualify, but any can produce a visible failure on stage.
|
| 328 |
+
|
| 329 |
+
REPORT SECTION:
|
| 330 |
+
# Audit Consolidation & Closure Plan
|
| 331 |
+
|
| 332 |
+
## Verdict
|
| 333 |
+
|
| 334 |
+
Lightloom is **ship-blocked, not broken**. The deployed Space works; the live painter is structurally **sound at the pinned diffusers commit** (the painter audit's "FATAL import" verdict is a local-environment false alarm — see *Contradictions Resolved*). But the submission **forfeits its single largest prize today**: with ~48 hours to the June 15 close, there are **zero Codex-attributed commits and no git remote**, so the $5k OpenAI track is 100% unmet. Three honesty landmines will be caught by the exact judges (BFL, OpenBMB, Cohere) who wrote the rules and the architecture. Depth is a polish lever, not a gate — building a fourth renderer now is the highest-risk move in the entire audit.
|
| 335 |
+
|
| 336 |
+
## Contradictions Resolved (verified, not asserted)
|
| 337 |
+
|
| 338 |
+
1. **"Painter is BROKEN — imports don't resolve" vs "live demo is deployed and working."** Resolved by direct test. The dev machine runs **stable diffusers 0.36.0**, which exports only `Flux2Pipeline` — so `Flux2KleinInpaintPipeline`/`Flux2KleinKVPipeline` genuinely `ImportError` *locally*. But `requirements.txt` pins commit `784fa62`, and I confirmed against that exact commit's GitHub source that it **does** export both Klein classes (`pipeline_flux2_klein_inpaint`, `pipeline_flux2_klein_kv`). **Conclusion:** the painter is not broken on the Space; it is **unverifiable on the dev machine**, which is the real reason ~6 independent audits flagged "unverified against the real pipeline." The fix is not a rewrite — it is **one Space smoke-test**.
|
| 339 |
+
|
| 340 |
+
2. **"`gr.Server` does not exist / app won't import" vs "build random-cover-1 is deployed."** Both true and consistent: `gr.Server` is absent from the *local* gradio, present in the HF-injected build. The risk is real but mischaracterized as "currently broken" — it is "**unpinned and will break on the next HF gradio bump.**" Fix: pin `sdk_version` in README front-matter + a boot-time `assert hasattr(gr,'Server')` + a CI import-smoke test.
|
| 341 |
+
|
| 342 |
+
3. **Codex attribution vs the user's permanent no-Co-Authored-By rule.** Not a true conflict. The prize requires *Codex-attributed* commits; Codex's *default* is a `Co-authored-by:` trailer (forbidden), but **author-level** identity (`GIT_AUTHOR_NAME=Codex`) satisfies the prize **without** a trailer.
|
| 343 |
+
|
| 344 |
+
## Consolidated, Severity-Ranked Fix List
|
| 345 |
+
|
| 346 |
+
**P0 — Disqualifier / submission-incomplete (do first, in order):**
|
| 347 |
+
- **Codex history + private repo linked to Space.** Zero Codex commits today; binary forfeit of $5k. Author-level attribution, no squash of Efrain history, real `git commit`s (not `upload_folder`).
|
| 348 |
+
- **Honesty truth-pass (hard-gated by `release_check.py`):** delete the false "built with Codex" line; replace fabricated badges with the real Field Guide list and **claim the unclaimed Tiny Titan ($1.5k, every model ≤4B) + Off Brand ($1.5k)**; rewrite the README depth description and `stage-scroll.js:8` header to the **shipped** lean; state style is **random-per-world by default**; reconcile lora.py (1.0 vs 0.75) and the 5.83B-vs-10.73B params number.
|
| 349 |
+
- **Demo video over the live Space** (single unbroken wow shot) + social post + fill three README ⟦PLACEHOLDER⟧ links.
|
| 350 |
+
- **Regenerate `preflight.json` against the live Hub** — resolve every repo_id+SHA (`HfApi().model_info`); the current artifact pins a different ASR than the README and risks reading as fabricated verification.
|
| 351 |
+
|
| 352 |
+
**P1 — Will surface on a judge's screen / security at deploy:**
|
| 353 |
+
- **Deploy from `git archive HEAD`** (tracked files only), never `folder_path=ROOT` with a fail-open deny-list; **never set `LIGHTLOOM_ENABLE_INTERNAL_BENCH=1` in the Space** (it exposes tracebacks + un-ledgered models + GPU-DoS). Verify no private media leaks into the new repo.
|
| 354 |
+
- **One Space smoke-test of `outpaint_right`** to discharge the painter doubt cluster and verify `mask_image`/`strength`/`guidance_scale` against the real `__call__`.
|
| 355 |
+
- **Global-session depth normalization** (kill the per-strip double-normalize; EMA window persisted in live state) — root cause of both "too flat" and seam tearing. Keep renderer #3, rebrand it.
|
| 356 |
+
- **Bound the live phrase queue + make `quota_exceeded` terminal**; **cap base64 audio pre-decode**; **exclude the active session from `_sweep_scroll_sessions`**; **atomic `state.json` write** so a partial write can't re-roll the style mid-session.
|
| 357 |
+
|
| 358 |
+
**P2 — Robustness / correctness debt (post-submission):** harden `_coerce_shot` to repair palette + fill empty required strings (converts most lost beats to graceful successes); add constrained JSON decoding to the GPU Director path; unify the two divergent scroll engines + the three default scene states; delete inert code (`slerp_embeds`, `encode_scroll_embeds`, `sdedit_strength` branch, `_introspect_klein`); add the missing test suites (`test_guards.py`, `test_asr.py`) per CLAUDE rule 5; wire the already-built interim-caption path (one import + one callback).
|
| 359 |
+
|
| 360 |
+
## Closure + Publication Plan (48-hour critical path)
|
| 361 |
+
|
| 362 |
+
**Hour 0–6 — Eligibility & honesty (no GPU):** create private GitHub repo, link to Space, configure Codex author identity, commit the `COMMIT_PLAN.md` groups through Codex. In parallel, run the honesty truth-pass text edits and make `release_check.py` gate them.
|
| 363 |
+
|
| 364 |
+
**Hour 6–18 — Artifacts:** regenerate `preflight.json` against the Hub; switch deploy to `git archive`; one Space smoke-test of the painter; land the global-depth-normalization fix and rebrand renderer #3 as the "living horizon-lean."
|
| 365 |
+
|
| 366 |
+
**Hour 18–30 — Demo:** pre-warm the Space off-camera, record the single-shot wow video, write the social post, fill the README links, run a quota-exhausted incognito judge-sim (desktop + mobile) so the Showcase fallback is exercised.
|
| 367 |
+
|
| 368 |
+
**Hour 30–48 — Buffer + research note:** ship the Field Notes negative-result write-up (scalar-broadcast wall + constructive forward-patch corollary, cited to `transformer_flux2.py`, positioned against StreamDiT/Rolling Diffusion, citing Rokeby 2023 + IDCIoT 2024). Flip the LoRA public with a proper klein card.
|
| 369 |
+
|
| 370 |
+
## Publication Targets (closure)
|
| 371 |
+
- **Negative result → NeurIPS 2026 ICBINB** (negative-results venue; CFP explicitly accepts architecture/efficiency-limitation findings).
|
| 372 |
+
- **Live system → Creative-AI / Real-Time-Live demo track** (separate home; judged as a system, not on SOTA).
|
| 373 |
+
- **LoRA + Director-traces dataset → HF Hub** with a free DOI for citable credit (Well-Tuned + Sharing-is-Caring).
|
| 374 |
+
|
| 375 |
+
## The BFL-Impressing Frame (do this, not a Voice-Scroll clone)
|
| 376 |
+
Lead with **"interactive visual intelligence, made literal"** — klein's own slogan. The contribution is the **system** (local, single-slot, continuous streaming outpaint, 2.5D, disk-derived continuity across worker recycles) **+ a verified, citable negative result about BFL's own model**. The post-submission bold moves that genuinely out-impress a clone are **latent-native depth read straight from the REPA/DINOv2-aligned klein VAE** (their #1 research thread) and a **self-compiled sm120 Blackwell AOTI build of klein** (which HF has not packaged for the actual judging hardware). Both are honest, both are on BFL's exact frontier, and neither should be allowed to threaten the 48-hour freeze.
|
app.py
CHANGED
|
@@ -145,7 +145,7 @@ def health() -> dict[str, Any]:
|
|
| 145 |
"privacy_mode": os.getenv("LIGHTLOOM_PRIVACY_MODE", "1") == "1",
|
| 146 |
"cpu_count": os.cpu_count(),
|
| 147 |
"director_backend": CONFIG.director_backend,
|
| 148 |
-
"build": "
|
| 149 |
}
|
| 150 |
|
| 151 |
|
|
@@ -1025,6 +1025,28 @@ def _live_save_state(out_dir: Path, **updates: Any) -> None:
|
|
| 1025 |
pass
|
| 1026 |
|
| 1027 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1028 |
def _scroll_transition_prompt(prev_subject: str, subject: str) -> str:
|
| 1029 |
"""The boundary strip between two scenes: paint the FIRST as it DISSOLVES into the
|
| 1030 |
SECOND, so the panorama morphs instead of cutting hard."""
|
|
@@ -1184,10 +1206,8 @@ def _scroll_live_phrase(session: str, text: str, audio_b64: str, lang: str):
|
|
| 1184 |
"caption": phrase[:160] if k == 0 else "",
|
| 1185 |
"w": sec.width,
|
| 1186 |
}
|
| 1187 |
-
|
| 1188 |
-
|
| 1189 |
-
dmap = (dmap - lo) / (hi - lo) if hi > lo else np.full_like(dmap, 0.5)
|
| 1190 |
-
dmap = np.clip(dmap, 0.0, 1.0) # constant/low-variance depth must not wrap mod 256
|
| 1191 |
Image.fromarray((dmap * 255).astype(np.uint8)).save(out_dir / f"section-{g}-depth.png")
|
| 1192 |
yield {"stage": "depth", "index": g, "depth": f"/frames-scroll/{session}/section-{g}-depth.png"}
|
| 1193 |
g += 1
|
|
|
|
| 145 |
"privacy_mode": os.getenv("LIGHTLOOM_PRIVACY_MODE", "1") == "1",
|
| 146 |
"cpu_count": os.cpu_count(),
|
| 147 |
"director_backend": CONFIG.director_backend,
|
| 148 |
+
"build": "raymarch-depth-1",
|
| 149 |
}
|
| 150 |
|
| 151 |
|
|
|
|
| 1025 |
pass
|
| 1026 |
|
| 1027 |
|
| 1028 |
+
def _depth_global_norm(raw: Any, state: dict[str, Any], out_dir: Path) -> Any:
|
| 1029 |
+
"""Normalize a RAW depth/disparity map against ONE session-global [lo,hi] (an EMA of per-strip
|
| 1030 |
+
2/98 percentiles, persisted across GPU-worker recycles), so EVERY strip shares the SAME depth
|
| 1031 |
+
scale -> adjacent strips align (no seam) and 'near' is consistent (no per-strip flattening — the
|
| 1032 |
+
structural cause of the 'flat 2.5D' + seam shear). Returns [0,1] with 1 = near (renderer convention)."""
|
| 1033 |
+
import numpy as np
|
| 1034 |
+
|
| 1035 |
+
p2, p98 = float(np.percentile(raw, 2)), float(np.percentile(raw, 98))
|
| 1036 |
+
lo, hi = state.get("depth_lo"), state.get("depth_hi")
|
| 1037 |
+
if not (isinstance(lo, (int, float)) and isinstance(hi, (int, float)) and hi > lo):
|
| 1038 |
+
lo, hi = p2, p98 # the first strip seeds the session-global scale
|
| 1039 |
+
else:
|
| 1040 |
+
a = 0.25 # ease the global range toward each new strip's percentiles
|
| 1041 |
+
lo, hi = (1 - a) * lo + a * p2, (1 - a) * hi + a * p98
|
| 1042 |
+
if hi - lo < 1e-6:
|
| 1043 |
+
hi = lo + 1e-6
|
| 1044 |
+
state["depth_lo"], state["depth_hi"] = lo, hi
|
| 1045 |
+
_live_save_state(out_dir, depth_lo=lo, depth_hi=hi)
|
| 1046 |
+
norm = np.clip((raw - lo) / (hi - lo), 0.0, 1.0)
|
| 1047 |
+
return 1.0 - norm # 1 = near, 0 = far
|
| 1048 |
+
|
| 1049 |
+
|
| 1050 |
def _scroll_transition_prompt(prev_subject: str, subject: str) -> str:
|
| 1051 |
"""The boundary strip between two scenes: paint the FIRST as it DISSOLVES into the
|
| 1052 |
SECOND, so the panorama morphs instead of cutting hard."""
|
|
|
|
| 1206 |
"caption": phrase[:160] if k == 0 else "",
|
| 1207 |
"w": sec.width,
|
| 1208 |
}
|
| 1209 |
+
raw_d = estimate_depth(depth_pipe, sec, normalize=False) # RAW disparity (un-scaled)
|
| 1210 |
+
dmap = _depth_global_norm(raw_d, state, out_dir) # ONE session-global scale, 1=near
|
|
|
|
|
|
|
| 1211 |
Image.fromarray((dmap * 255).astype(np.uint8)).save(out_dir / f"section-{g}-depth.png")
|
| 1212 |
yield {"stage": "depth", "index": g, "depth": f"/frames-scroll/{session}/section-{g}-depth.png"}
|
| 1213 |
g += 1
|
docs/DEEP_RESEARCH_REPORT.md
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Lightloom — Deep Research & Audit Report (60-agent, June 2026)
|
| 2 |
+
|
| 3 |
+
Source: a 60-agent orchestration (40 research + 15 audit + 5 synthesis), 3.9M tokens. Every critical
|
| 4 |
+
finding below was **independently re-verified against the real code/env** before being written here —
|
| 5 |
+
where the agents were wrong (they audited the stale local `.venv`), it is flagged.
|
| 6 |
+
|
| 7 |
+
---
|
| 8 |
+
|
| 9 |
+
## TL;DR — the honest headline
|
| 10 |
+
|
| 11 |
+
The system **runs and is real**, but the deep audit is not flattering, and that's the point. Three things
|
| 12 |
+
matter more than anything else with ~48h to close:
|
| 13 |
+
|
| 14 |
+
1. **The depth problem is now SOLVED on paper — and it becomes our signature.** Renderer #3 (the lean) is
|
| 15 |
+
**mathematically flat by construction** (verified: `stage-scroll.js:96` is a center-anchored horizontal
|
| 16 |
+
zoom with *zero* parallax where the eye looks and *no* scroll term). The user's instinct was correct.
|
| 17 |
+
The fix is a single, decisive path (a fragment-shader **steep-parallax raymarch driven by scroll
|
| 18 |
+
velocity** on **globally-normalized** depth), and the *branded innovation* is **"the painter paints what
|
| 19 |
+
it hides"** — klein doing a second pass to inpaint the hidden layer, so depth is generated **by FLUX**.
|
| 20 |
+
2. **The submission currently FORFEITS its biggest prize and ships honesty landmines.** 0 Codex-attributed
|
| 21 |
+
commits + empty git remote ⇒ the OpenAI track is 100% unmet. The README claims "built with Codex"
|
| 22 |
+
(false), lists badge names that don't exist, and describes a depth renderer that was rejected. These are
|
| 23 |
+
exactly what the rule-writing judges will catch. **R7 (public repo + Codex commits) is the only true
|
| 24 |
+
disqualifier and it is unmet.**
|
| 25 |
+
3. **Gradio is confirmed but FRAGILE, and Cohere (the host sponsor) runs zero live weights.** `gr.Server`
|
| 26 |
+
is not in standard gradio 6.5.1 (`gr.Server is None`) — the app can't even boot locally; it works only
|
| 27 |
+
on the Space's injected SDK build. One silent SDK bump = total failure. Both are fixable into *wins*.
|
| 28 |
+
|
| 29 |
+
The depth is a **polish lever, not a gate**. The gates are R7 + honesty. We fix both, then we make the
|
| 30 |
+
depth + Gradio + Cohere into the things that *win*.
|
| 31 |
+
|
| 32 |
+
---
|
| 33 |
+
|
| 34 |
+
## PART 1 — Depth / 3D & the branded innovation (the #1 problem)
|
| 35 |
+
|
| 36 |
+
### Why every depth attempt failed (root causes, verified)
|
| 37 |
+
- **Renderer #3 (lean) is flat by construction.** `p = vec3(position.x*(1.0+W), position.y, 0.0)` is a zoom
|
| 38 |
+
about screen-center: **no parallax at frame center, no scroll-coupled motion** ⇒ structurally "too flat".
|
| 39 |
+
- **Depth is normalized TWICE, independently per strip** (`depth.py:17-30` min-max+invert, then `app.py`
|
| 40 |
+
re-stretches each strip's 2/98 percentiles). No shared scale ⇒ "near" means something different every
|
| 41 |
+
section ⇒ the seam shear of renderer #2 AND a contributor to "flat".
|
| 42 |
+
- **DepthAnything-V2-Small keys on LUMINANCE, not geometry, on painterly strips** — bright skies read as
|
| 43 |
+
*near*, ~75-87% of every strip reads as "near" after the contrast curve ⇒ near-uniform stretch = flat.
|
| 44 |
+
This is a *named, published gap*: depth models are trained to **ignore** painterly media (Illustrator's
|
| 45 |
+
Depth, arXiv:2511.17454, Nov 2025).
|
| 46 |
+
|
| 47 |
+
### The ship-now depth path (retires all 3 rejected renderers)
|
| 48 |
+
1. **Replace the lean with a fragment-shader STEEP-PARALLAX RAYMARCH** (DepthFlow recipe) over the single
|
| 49 |
+
continuous 8192px strip+depth texture: march the view ray into the heightfield (16-24 coarse + 6-8
|
| 50 |
+
binary-refine steps), take the **first surface hit**, sample the image there. Drive the eyepoint offset
|
| 51 |
+
from **scroll VELOCITY** (never a time/Lissajous term), bounded ~4-6% of view width, eased to 0 at rest.
|
| 52 |
+
→ tear-free (first-hit occlusion, not forward UV smear), a TRUE viewpoint change (reads as 3D at frame
|
| 53 |
+
center), seam-free (one strip), proven real-time (DepthFlow ships 8K@50fps). *No per-frame network.*
|
| 54 |
+
2. **Normalize depth ONCE, globally, per session** (EMA of 2/98 percentiles persisted in live state). Kill
|
| 55 |
+
the double per-strip normalize. Single highest-leverage fix for BOTH "flat" and the seam shear.
|
| 56 |
+
3. **Add a depth-driven AERIAL-PERSPECTIVE pass** (desaturate + lerp toward a per-world sky tint + tiny
|
| 57 |
+
blur as depth→far) + a disocclusion stretch-fill clamp. Aerial perspective is a primary monocular cue we
|
| 58 |
+
completely lack (`vShade` is only a 0.90–1.06 brightness nudge). Cheap, **cannot tear**, and is the
|
| 59 |
+
clean fallback that makes depth visible even before the raymarch lands.
|
| 60 |
+
|
| 61 |
+
### The BRANDED innovation (what out-impresses a Voice-Scroll clone, for BFL)
|
| 62 |
+
- **"The painter paints what it hides" — the Streaming Layered Depth Strip.** Per appended strip, run a
|
| 63 |
+
SECOND klein pass masked to a soft-disocclusion map (SLIDE: `S = tanh(depth-gradient discontinuity)`) to
|
| 64 |
+
**inpaint the hidden background once at generate-time**; store it as a BG layer the raymarch reveals as
|
| 65 |
+
content sweeps. Depth is generated **BY FLUX** (klein doing double duty: outpaint forward + inpaint
|
| 66 |
+
behind), amortized into a call we already pay for, ZeroGPU-survivable. **No Voice-Scroll clone can have
|
| 67 |
+
this.** It gives true accretion-deletion — the cue the lean structurally lacks.
|
| 68 |
+
- **FLUX.2 4D position-IDs as a streaming stitch primitive** (verified in installed source): subclass the
|
| 69 |
+
pipeline to override `_prepare_image_ids` so the previous strip's W-coords sit immediately LEFT of the new
|
| 70 |
+
target (instead of resetting to W=0) ⇒ RoPE encodes true horizontal adjacency ⇒ **born-consistent seams
|
| 71 |
+
at generation time.** This is the positive mirror of our verified-negative adaLN result.
|
| 72 |
+
- **Research artifact (BFL-grade):** ship the matched pair — VERIFIED NEGATIVE (klein adaLN is a scalar
|
| 73 |
+
broadcast ⇒ per-column LoRA streaming impossible) + VERIFIED POSITIVE (klein inpaint synthesizes
|
| 74 |
+
disocclusion in real time; W-offset reference IDs stitch neighbors).
|
| 75 |
+
- **Modal stretch (headline fine-tune):** train klein to **co-generate a painterly depth channel in the
|
| 76 |
+
same 4-step pass** (joint RGB+depth LoRA, target a `[RGB | sharp-depth]` 2-up tile, depth labeled by
|
| 77 |
+
Depth Pro / DAv2-Large). The painter emits geometry **aligned to its brushwork**, seam-consistent by
|
| 78 |
+
construction, and **deletes the separate depth model from the runtime.** Attacks depth at the source.
|
| 79 |
+
- **Demo finale (not live):** ONE pre-baked **Spark.js** Gaussian-splat fly-through of the finished spoken
|
| 80 |
+
world — the most shareable artifact — while the live loop stays on the safe raymarch. Frame explicitly
|
| 81 |
+
AGAINST World Labs *Marble* (cloud, minutes, one-shot, static): **Lightloom is live, local, continuous,
|
| 82 |
+
voice-streamed on one ZeroGPU slot.** Do NOT do live per-strip 3DGS / 32-64-plane MPI (budget + seams).
|
| 83 |
+
|
| 84 |
+
---
|
| 85 |
+
|
| 86 |
+
## PART 2 — Gradio: confirmed, fragile, and a huge unclaimed win
|
| 87 |
+
|
| 88 |
+
- **Confirmed:** the app is a real `gr.Server` app (Gradio's custom-frontend "Server mode", shipped ~April
|
| 89 |
+
2026), with `@gradio/client` transport load-bearing for ZeroGPU quota attribution. This is a *strong*
|
| 90 |
+
reading of the "must use Gradio" rule.
|
| 91 |
+
- **BUT FRAGILE (critical):** `gr.Server` does **not** exist in standard gradio 6.5.1 (`gr.Server is None`)
|
| 92 |
+
— the app **cannot boot locally**; it runs only on the Space's injected SDK build. One silent HF SDK bump
|
| 93 |
+
= total failure, and the win is invisible to a video-only judge.
|
| 94 |
+
- **Fixes (do both):**
|
| 95 |
+
1. **PIN `sdk_version`** in README front-matter to the exact `gr.Server`-providing build the live Space
|
| 96 |
+
runs **today** (read it off the Space — do **not** guess) + a **boot-time assert** on `gr.Server`.
|
| 97 |
+
2. **THE single most impressive move:** extract the WebGL voice-scroll/depth renderer into a **publishable
|
| 98 |
+
custom Gradio component** (`gradio cc create`, 3D template) → publish to **PyPI + the Gradio Gallery**.
|
| 99 |
+
The first live, voice-driven, depth-displaced painterly-panorama component in existence — a permanent,
|
| 100 |
+
citable, `pip install`-able artifact a clone cannot produce.
|
| 101 |
+
- **Bonus:** stack `@app.mcp.tool()` over the existing endpoints ⇒ "an agent paints a continuous world by
|
| 102 |
+
speaking" (agentic category, one-line decorators). Add a `gradio_client` repro snippet to the README so
|
| 103 |
+
judges can drive the live ZeroGPU pipeline programmatically.
|
| 104 |
+
- **Do NOT** adopt FastRTC/WebRTC for the core voice→paint loop (document as a considered-rejected verified
|
| 105 |
+
negative); keep `@app.api` + `@gradio/client`.
|
| 106 |
+
|
| 107 |
+
---
|
| 108 |
+
|
| 109 |
+
## PART 3 — Sponsors, Modal & fine-tuning (thin-to-false today)
|
| 110 |
+
|
| 111 |
+
- **Cohere (HOST sponsor) runs ZERO live weights** — only "evaluated". The agents found our incompat
|
| 112 |
+
excuse is now **obsolete**: on **transformers 5.12.x**, `CohereLabs/cohere-transcribe-03-2026` loads via
|
| 113 |
+
the **native** `CohereAsrForConditionalGeneration` (no `trust_remote_code`). **Make Cohere load-bearing**:
|
| 114 |
+
replace Whisper as the runtime ASR (it's #1 on Open ASR). This single move turns the weakest sponsor story
|
| 115 |
+
into a strong one + lets us say "we run the host sponsor's own reference pipeline."
|
| 116 |
+
- **PRIMARY fine-tune (BFL headline):** the joint RGB+depth "paint-the-depth" LoRA (Part 1).
|
| 117 |
+
- **Ship-first depth fix (no training, <48h):** layered-depth + klein disocclusion inpaint (Part 1's
|
| 118 |
+
Streaming Layered Depth Strip, the cheap version).
|
| 119 |
+
- **Modal → a warm "Deep Loom" microservice** (`@modal.asgi_app` + `min_containers=1` + memory snapshot):
|
| 120 |
+
takes a finished strip, returns a **sharp Depth-Pro/MoGe depth map + feed-forward splats** for the
|
| 121 |
+
fly-through finale. Turns Modal from a LoRA toy into a real, on-brand two-tier architecture (live local on
|
| 122 |
+
ZeroGPU; the one thing ZeroGPU physically can't do offloaded to a warm Modal endpoint).
|
| 123 |
+
- **MiniCPM upgrade:** adopt **MiniCPM-V 4.6** (vision, transformers-5.11-native, +0.3B) as a **"Seeing
|
| 124 |
+
Director"** — reads the cover strip to lock palette/style, inspects every Nth beat for drift. Strengthens
|
| 125 |
+
the OpenBMB story (the Director actually SEES the world it's directing).
|
| 126 |
+
- **Reproducibility (must fix before any retrain):** pin diffusers/transformers/peft/accelerate to the
|
| 127 |
+
runtime's exact commit (`784fa62…`) in the Modal image; the `validate/stylegrid/covers` fuse the LoRA at
|
| 128 |
+
scale 1.0 but the live path is also 1.0 now — reconcile the stale `0.75` comment in `lora.py`.
|
| 129 |
+
|
| 130 |
+
---
|
| 131 |
+
|
| 132 |
+
## PART 4 — Real-time UX, captions & fluidity (the user's ask — mostly DELETION + WIRING)
|
| 133 |
+
|
| 134 |
+
- **The "words as you speak" feature is 100% DEAD CODE** — `recorder.js` + `asr-worker.js` implement the
|
| 135 |
+
full client interim path; `controller.js` never wires `onInterim`. **Just wire it.**
|
| 136 |
+
- **Wire it correctly (kills the old O(n²) bug by construction):** fire the client caption **once per CLOSED
|
| 137 |
+
VAD clause** (delete the 1.1s growing-buffer tick), on **Moonshine-tiny** (`onnx-community/moonshine-tiny`,
|
| 138 |
+
~34ms, on-device WebGPU→WASM). Interim = greyed "wet ink"; server-whisper final hardens it.
|
| 139 |
+
- **Two clocks (the real fluidity fix):** decouple **words** (Lane A: ASR-only, ~200ms, append
|
| 140 |
+
`pending-NNN.txt` to the disk queue) from **paint** (Lane B: ONE long-lived self-clocked `@spaces.GPU`
|
| 141 |
+
painter generator per mic session that pops the queue) ⇒ the transcript NEVER queues behind the painter.
|
| 142 |
+
Surface a dual-track HUD ("you said / I'm painting"). This is also a clean **systems contribution**.
|
| 143 |
+
- **Decouple depth from the visible strip:** drop the `!s.depth → return` gate (`controller.js _tryAdd`);
|
| 144 |
+
render the strip flat instantly, hot-swap depth on the later event.
|
| 145 |
+
- **Stop the hitching:** add `Cache-Control: immutable` + content-hashed filenames to the static routes;
|
| 146 |
+
stop re-uploading the **entire 8192×512 RGBA** texture on every `addSection` (only ~384px changed).
|
| 147 |
+
- **Signature UX:** the **WET-INK CAPTION** writes interim words onto the **leading right edge** (where the
|
| 148 |
+
next strip will paint), then "dries" to crisp — the exact moment "the word becomes world."
|
| 149 |
+
|
| 150 |
+
---
|
| 151 |
+
|
| 152 |
+
## PART 5 — Honesty landmines + the closure / R7 critical path
|
| 153 |
+
|
| 154 |
+
The audit's verdict: **SHIP-BLOCKED, NOT BROKEN.** The painter is sound at the pinned commit (the "FATAL
|
| 155 |
+
import" finding is a **local-env false alarm** — verified: `requirements.txt` pins diffusers to git commit
|
| 156 |
+
`784fa62…` which has the class; the local `.venv` has stale 0.36.0).
|
| 157 |
+
|
| 158 |
+
**Critical (must fix before submission):**
|
| 159 |
+
- **R7 — the only true disqualifier:** 0 Codex-attributed commits, empty git remote. Create a (private is
|
| 160 |
+
allowed) GitHub repo, link it to the Space, route the `docs/COMMIT_PLAN.md` groups through **Codex**.
|
| 161 |
+
- **Honesty truth-pass (hard-gate in `release_check.py`):**
|
| 162 |
+
- **DELETE the "built with Codex" claim** until the Codex history is real (README:83 — currently false).
|
| 163 |
+
- **Fix the badge list** — several advertised badges are fabricated / don't exist; claim only real ones.
|
| 164 |
+
- **Fix the README depth description** — it describes the rejected renderer, twice, two different wrong
|
| 165 |
+
ways; align it to whatever ships.
|
| 166 |
+
- **Own the style claim I broke:** README still sells "MiniCPM *infers* the per-scene style" but it's now
|
| 167 |
+
**RANDOM** (the inferring path is env-gated off). Make the README say "random per world" (truthful).
|
| 168 |
+
- Reconcile the ledger ASR row vs `preflight.json`; the stale LoRA-scale comment; un-pinned LoRA revision.
|
| 169 |
+
- **SECURITY (real):** the deploy sets `LIGHTLOOM_ENABLE_INTERNAL_BENCH=1` as a **production** Space variable
|
| 170 |
+
(`deploy_private_space.py:65`) ⇒ it **silently re-enables** `/world`, `/panorama`, and every `/internal/*`
|
| 171 |
+
endpoint I gated (which load **un-ledgered** models) ⇒ directly contradicts the "only four models" ledger.
|
| 172 |
+
Also: the deploy ships the **entire repo root** via a hand-maintained deny-list (leak risk); the
|
| 173 |
+
client-supplied `session` is an **IDOR** key into shared `/tmp`; `_sweep_scroll_sessions()` can delete a
|
| 174 |
+
**live** session mid-narration.
|
| 175 |
+
- **Perf/latency (real, high):** `load_asr()` is **not cached** ⇒ Whisper reloads from disk on **every**
|
| 176 |
+
transcription; per-clause path is fully sequential (ASR → director-once → 2 strips → depth, concurrency 1).
|
| 177 |
+
|
| 178 |
+
**Closure plan (sequenced for ~48h):**
|
| 179 |
+
1. **R7 now:** public/linked repo + Codex commits (user/Codex). *Biggest prize, only disqualifier.*
|
| 180 |
+
2. **Honesty truth-pass** + harden `release_check.py` to gate on it (Claude can do).
|
| 181 |
+
3. **Depth ship-now path** (global normalize + raymarch + aerial perspective) → deploy → user accepts depth.
|
| 182 |
+
4. **Wire interim captions + two clocks** (mostly deletion + wiring) → fluidity.
|
| 183 |
+
5. **Fix the bench-flag-in-prod + ASR cache + IDOR + sweep race** (security/perf, Claude can do).
|
| 184 |
+
6. **Demo video** over the live, pre-warmed Space — one unbroken "wow" shot (user).
|
| 185 |
+
7. **Make Cohere load-bearing** (transformers 5.12 + native Cohere Transcribe) — host-sponsor story.
|
| 186 |
+
8. **Branded artifacts:** publish the LoRA public (Well-Tuned), the Field Notes write-up (negative+positive),
|
| 187 |
+
the custom Gradio component (the single biggest BFL/Gradio flex), the Spark.js fly-through finale.
|
| 188 |
+
|
| 189 |
+
**Stretch (if time):** the joint RGB+depth "paint-the-depth" LoRA (headline BFL innovation); the Modal
|
| 190 |
+
"Deep Loom" microservice; MiniCPM-V "Seeing Director"; a Blackwell (sm120) AOTI build of klein (HF's own
|
| 191 |
+
zerogpu-aoti/FLUX.2-klein targets sm90/H200 only — won't load on the judging hardware; a working sm120 build
|
| 192 |
+
in a streaming loop is itself a real contribution).
|
| 193 |
+
|
| 194 |
+
---
|
| 195 |
+
|
| 196 |
+
### The one-line thesis to brand everything
|
| 197 |
+
**"Lightloom is what FLUX.2 [klein] was built for: a live, on-device, sub-second, voice-driven canvas —
|
| 198 |
+
where the painter even paints what it hides."** Live + local + continuous + voice + FLUX-native depth is the
|
| 199 |
+
axis a Voice-Scroll clone and a cloud world-model both lose on.
|
frontend/js/stage-scroll.js
CHANGED
|
@@ -5,11 +5,12 @@
|
|
| 5 |
* The world is ONE continuous horizontal panorama that keeps EXTENDING to the
|
| 6 |
* right as you speak (each section is outpainted to continue the last, seamlessly)
|
| 7 |
* and SCROLLS LEFT continuously, so it flows like a living mural. Real 2.5D comes
|
| 8 |
-
* from a
|
| 9 |
-
*
|
| 10 |
-
*
|
| 11 |
-
*
|
| 12 |
-
*
|
|
|
|
| 13 |
*
|
| 14 |
* Sections stream in via addSection({imageUrl, depthUrl}); they are composited onto
|
| 15 |
* a growing strip canvas (so the texture is ONE continuous panorama, no quad seams).
|
|
@@ -27,95 +28,136 @@ const Scroll = (() => {
|
|
| 27 |
let raf = 0, t0 = 0, lastT = 0, ok = false, reduced = false, aspect = 16 / 9;
|
| 28 |
// backing strip canvases (one continuous panorama + its depth), grown to the right
|
| 29 |
const STRIP_H = 512, STRIP_W = 8192; // ~21 sections of 384px; recycled when full
|
| 30 |
-
// 2.5D via
|
| 31 |
-
//
|
| 32 |
-
|
| 33 |
const CAM_FOV = 38; // degrees
|
| 34 |
const CAM_DIST = 2.0; // camera distance from the plane center (the camera is STATIC)
|
| 35 |
const DEPTH_GAIN = 1.45; // contrast stretch for coarse, low-contrast DepthAnything-Small depth
|
| 36 |
const DEPTH_GAMMA = 0.85; // <1 lifts mid-depths so near content separates from far
|
| 37 |
-
const OVERSCAN = 1.
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
const
|
| 43 |
-
|
| 44 |
-
const
|
| 45 |
-
const
|
|
|
|
|
|
|
| 46 |
let imgCanvas = null, depCanvas = null, imgCtx = null, depCtx = null;
|
| 47 |
let imgTex = null, depTex = null;
|
| 48 |
let writeX = 0; // where the next section is pasted (px on the strip)
|
| 49 |
let scrollX = 0; // current left edge of the view window (px on the strip)
|
|
|
|
|
|
|
|
|
|
| 50 |
let osReduced = false;
|
| 51 |
const queue = []; // sections {img, dep, meta, x} in arrival order, for focus/captions
|
| 52 |
let onFocus = null, focusX = -1;
|
| 53 |
const loader = new THREE.TextureLoader();
|
| 54 |
loader.crossOrigin = "anonymous";
|
| 55 |
|
| 56 |
-
// VERT:
|
| 57 |
-
// focus, bound it, edge-attenuate the steepest cliffs, push +Z (toward camera = near).
|
| 58 |
const VERT = `
|
| 59 |
precision highp float;
|
| 60 |
-
|
| 61 |
-
uniform float uU0; // window left edge, normalized on the strip
|
| 62 |
-
uniform float uViewU; // window width, normalized
|
| 63 |
-
uniform float uFocus; // depth pivot for the contrast stretch
|
| 64 |
-
uniform float uGain; // depth contrast gain (>1 expands near/far separation)
|
| 65 |
-
uniform float uGamma; // depth gamma (<1 lifts mids)
|
| 66 |
-
uniform float uTexel; // 1.0 / STRIP_W
|
| 67 |
-
uniform float uTexelV; // 1.0 / STRIP_H
|
| 68 |
-
uniform float uLeanX; // max horizontal lean of nearest content (0 => flat)
|
| 69 |
-
uniform float uNearLo; // depth where the lean starts ramping up
|
| 70 |
-
uniform float uNearHi; // depth where the lean is full (nearest)
|
| 71 |
-
varying vec2 vUv; // true window uv handed to the fragment shader (UNDISPLACED)
|
| 72 |
-
varying float vShade; // subtle depth-cued shading
|
| 73 |
-
float sampleDepth(vec2 wuv){
|
| 74 |
-
return texture2D(uDep, vec2(clamp(wuv.x,0.0,1.0), clamp(wuv.y,0.001,0.999))).r;
|
| 75 |
-
}
|
| 76 |
void main(){
|
| 77 |
-
vUv = uv;
|
| 78 |
-
|
| 79 |
-
float d0 = clamp(sampleDepth(wuv), 0.0, 1.0);
|
| 80 |
-
// contrast stretch (coarse DepthAnything-Small is low-contrast): gamma-lift, then expand about focus.
|
| 81 |
-
float d = pow(d0, uGamma);
|
| 82 |
-
d = clamp(uFocus + (d - uFocus) * uGain, 0.0, 1.0);
|
| 83 |
-
// edge-aware attenuation: kills the lean across depth cliffs -> strictly monotone, no fold/tear.
|
| 84 |
-
float dxp = sampleDepth(wuv + vec2(uTexel*2.0, 0.0));
|
| 85 |
-
float dxn = sampleDepth(wuv - vec2(uTexel*2.0, 0.0));
|
| 86 |
-
float dyp = sampleDepth(wuv + vec2(0.0, uTexelV*2.0));
|
| 87 |
-
float dyn = sampleDepth(wuv - vec2(0.0, uTexelV*2.0));
|
| 88 |
-
float slope = abs(dxp-dxn) + abs(dyp-dyn);
|
| 89 |
-
float edgeAtt = 1.0 - smoothstep(0.18, 0.55, slope); // 1 on flats -> 0 on cliffs
|
| 90 |
-
// near-ness in [0,1]: 0 = far (no lean), 1 = nearest (full lean). NON-NEGATIVE -> coverage only expands.
|
| 91 |
-
float relNear = smoothstep(uNearLo, uNearHi, d);
|
| 92 |
-
float W = uLeanX * relNear * edgeAtt; // W in [0, uLeanX], 0 at cliffs
|
| 93 |
-
// SCREEN-ANCHORED HORIZONTAL LEAN: position.x in [-1,1] is the on-screen X (0 at centre). p.x =
|
| 94 |
-
// base_x*(1+W) is a per-vertex zoom about centre. Near (high W) leans outward more than far, so
|
| 95 |
-
// near LEADS far as the scroll sweeps content across the frame; bounded by uLeanX; identity at centre.
|
| 96 |
-
vec3 p = vec3(position.x * (1.0 + W), position.y, 0.0);
|
| 97 |
-
vShade = mix(1.06, 0.90, smoothstep(0.0, 1.0, 1.0 - relNear)); // near a touch brighter (aerial cue)
|
| 98 |
-
gl_Position = projectionMatrix * modelViewMatrix * vec4(p, 1.0);
|
| 99 |
}`;
|
| 100 |
-
// FRAG:
|
|
|
|
|
|
|
|
|
|
| 101 |
const FRAG = `
|
| 102 |
precision highp float;
|
| 103 |
-
varying vec2
|
| 104 |
-
|
| 105 |
-
uniform sampler2D
|
| 106 |
-
uniform float uU0;
|
| 107 |
-
uniform float uViewU;
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
c = max(c, vec3(0.0));
|
| 113 |
-
// linear -> sRGB
|
| 114 |
vec3 lo = c * 12.92; vec3 hi = 1.055 * pow(c, vec3(1.0/2.4)) - 0.055;
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
c *= mix(0.82, 1.0, vy);
|
| 120 |
gl_FragColor = vec4(c, 1.0);
|
| 121 |
}`;
|
|
@@ -127,36 +169,37 @@ const Scroll = (() => {
|
|
| 127 |
renderer = new THREE.WebGLRenderer({ canvas, antialias: true, alpha: false });
|
| 128 |
renderer.setPixelRatio(Math.min(devicePixelRatio || 1, 2));
|
| 129 |
scene = new THREE.Scene();
|
| 130 |
-
camera = new THREE.PerspectiveCamera(CAM_FOV, aspect, 0.05, 100);
|
| 131 |
camera.position.set(0, 0, CAM_DIST);
|
| 132 |
camera.lookAt(0, 0, 0);
|
| 133 |
imgCanvas = document.createElement("canvas"); imgCanvas.width = STRIP_W; imgCanvas.height = STRIP_H;
|
| 134 |
depCanvas = document.createElement("canvas"); depCanvas.width = STRIP_W; depCanvas.height = STRIP_H;
|
| 135 |
-
imgCtx = imgCanvas.getContext("2d"); depCtx = depCanvas.getContext("2d");
|
| 136 |
imgCtx.fillStyle = "#0b0d12"; imgCtx.fillRect(0, 0, STRIP_W, STRIP_H);
|
| 137 |
depCtx.fillStyle = "#808080"; depCtx.fillRect(0, 0, STRIP_W, STRIP_H);
|
| 138 |
imgTex = new THREE.CanvasTexture(imgCanvas); imgTex.colorSpace = THREE.SRGBColorSpace;
|
| 139 |
depTex = new THREE.CanvasTexture(depCanvas);
|
| 140 |
for (const tx of [imgTex, depTex]) { tx.minFilter = THREE.LinearFilter; tx.magFilter = THREE.LinearFilter; tx.wrapS = THREE.ClampToEdgeWrapping; }
|
| 141 |
-
depTex.generateMipmaps = false; depTex.wrapT = THREE.ClampToEdgeWrapping;
|
| 142 |
-
// Fallback: if the GPU has no vertex texture units, render flat (= depth-off, a clean
|
| 143 |
-
// image) instead of breaking. Universal on real hardware; gate anyway.
|
| 144 |
-
const canVTF = !renderer.capabilities || renderer.capabilities.vertexTextures !== false;
|
| 145 |
mat = new THREE.ShaderMaterial({
|
| 146 |
uniforms: {
|
| 147 |
uImg: { value: imgTex }, uDep: { value: depTex },
|
| 148 |
uU0: { value: 0 }, uViewU: { value: 0.25 },
|
| 149 |
-
uFocus: { value: 0.5 }, uGain: { value: DEPTH_GAIN }, uGamma: { value: DEPTH_GAMMA },
|
| 150 |
uTexel: { value: 1 / STRIP_W }, uTexelV: { value: 1 / STRIP_H },
|
| 151 |
-
|
| 152 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
},
|
| 154 |
vertexShader: VERT, fragmentShader: FRAG,
|
| 155 |
-
depthWrite: false, depthTest: false,
|
| 156 |
});
|
| 157 |
-
mesh = new THREE.Mesh(new THREE.PlaneGeometry(2, 2,
|
| 158 |
scene.add(mesh);
|
| 159 |
-
fitMeshToFrustum();
|
| 160 |
resize();
|
| 161 |
window.addEventListener("resize", resize);
|
| 162 |
const mq = window.matchMedia && window.matchMedia("(prefers-reduced-motion: reduce)");
|
|
@@ -166,8 +209,7 @@ const Scroll = (() => {
|
|
| 166 |
} catch (e) { ok = false; }
|
| 167 |
}
|
| 168 |
|
| 169 |
-
// size the plane to fill the frustum at z=0
|
| 170 |
-
// bulge never reveal the plane's edge.
|
| 171 |
function fitMeshToFrustum() {
|
| 172 |
if (!camera || !mesh || !camera.isPerspectiveCamera) return;
|
| 173 |
const halfH = Math.tan((camera.fov * Math.PI / 180) / 2) * CAM_DIST;
|
|
@@ -184,12 +226,28 @@ const Scroll = (() => {
|
|
| 184 |
aspect = w / h;
|
| 185 |
if (camera && camera.isPerspectiveCamera) {
|
| 186 |
camera.aspect = aspect;
|
| 187 |
-
camera.updateProjectionMatrix();
|
| 188 |
fitMeshToFrustum();
|
| 189 |
}
|
| 190 |
}
|
| 191 |
function reducedNow() { return reduced || osReduced || (document.body && document.body.classList.contains("reduced-motion")); }
|
| 192 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
function loadImg(url) {
|
| 194 |
return new Promise((resolve) => {
|
| 195 |
if (!url) return resolve(null);
|
|
@@ -236,9 +294,8 @@ const Scroll = (() => {
|
|
| 236 |
|
| 237 |
// Pre-smooth the DEPTH STRIP ONLY (never the picture): a separable 5-tap Gaussian with
|
| 238 |
// edge-clamp, run once per appended section, kills DepthAnything-Small's speckle and softens
|
| 239 |
-
// hard depth steps into gentle ramps BEFORE the
|
| 240 |
-
//
|
| 241 |
-
// falls back to a raw draw if pixel access is blocked (relief stays bounded by the shader).
|
| 242 |
function smoothDepthToStrip(dep, dx, w, h, blend = 0) {
|
| 243 |
const off = document.createElement("canvas"); off.width = w; off.height = h;
|
| 244 |
const oc = off.getContext("2d", { willReadFrequently: true });
|
|
@@ -247,8 +304,7 @@ const Scroll = (() => {
|
|
| 247 |
const src = oc.getImageData(0, 0, w, h);
|
| 248 |
const a = src.data, out = src.data, n = w * h;
|
| 249 |
// SEAM AFFINE MATCH: pin this section's depth onto the EXISTING strip's depth scale over the
|
| 250 |
-
// FEATHER overlap so there is no DC/contrast cliff at the boundary
|
| 251 |
-
// (kills the strip-seam shear). Adjacent depth maps then share one depth scale.
|
| 252 |
if (blend > 1 && dx > 0) {
|
| 253 |
try {
|
| 254 |
const bw = Math.min(blend, w);
|
|
@@ -323,6 +379,7 @@ const Scroll = (() => {
|
|
| 323 |
depCtx.globalCompositeOperation = "copy";
|
| 324 |
depCtx.drawImage(depCanvas, -shift, 0); depCtx.globalCompositeOperation = "source-over";
|
| 325 |
writeX = Math.max(0, writeX - shift); scrollX = Math.max(0, scrollX - shift);
|
|
|
|
| 326 |
for (const s of queue) s.x -= shift;
|
| 327 |
}
|
| 328 |
|
|
@@ -331,7 +388,6 @@ const Scroll = (() => {
|
|
| 331 |
if (!renderer || !scene || !camera) return;
|
| 332 |
const now = performance.now();
|
| 333 |
const dt = Math.min(0.05, (now - (lastT || now)) / 1000); lastT = now;
|
| 334 |
-
const t = (now - t0) / 1000;
|
| 335 |
const rm = reducedNow();
|
| 336 |
const viewU = Math.min(1, (aspect * STRIP_H) / STRIP_W); // window width (normalized)
|
| 337 |
mat.uniforms.uViewU.value = viewU;
|
|
@@ -341,12 +397,19 @@ const Scroll = (() => {
|
|
| 341 |
const speed = rm ? 0 : 110; // px/sec — the fast loop generates ~350px/s, so this flows
|
| 342 |
if (scrollX < maxScroll) scrollX = Math.min(maxScroll, scrollX + speed * dt);
|
| 343 |
mat.uniforms.uU0.value = scrollX / STRIP_W;
|
| 344 |
-
// 2.5D parallax
|
| 345 |
-
//
|
| 346 |
-
//
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 350 |
// focus = the section under the centre of the view -> caption
|
| 351 |
const centerX = scrollX + viewU * STRIP_W * 0.5;
|
| 352 |
let f = null;
|
|
@@ -361,6 +424,8 @@ const Scroll = (() => {
|
|
| 361 |
depCtx.fillStyle = "#808080"; depCtx.fillRect(0, 0, STRIP_W, STRIP_H);
|
| 362 |
imgTex.needsUpdate = depTex.needsUpdate = true;
|
| 363 |
writeX = 0; scrollX = 0; queue.length = 0; focusX = -1;
|
|
|
|
|
|
|
| 364 |
}
|
| 365 |
function setReducedMotion(f) { reduced = !!f; }
|
| 366 |
function dispose() { cancelAnimationFrame(raf); window.removeEventListener("resize", resize); if (renderer) renderer.dispose(); ok = false; }
|
|
|
|
| 5 |
* The world is ONE continuous horizontal panorama that keeps EXTENDING to the
|
| 6 |
* right as you speak (each section is outpainted to continue the last, seamlessly)
|
| 7 |
* and SCROLLS LEFT continuously, so it flows like a living mural. Real 2.5D comes
|
| 8 |
+
* from a per-fragment STEEP-PARALLAX RAYMARCH (DepthFlow relief recipe): the
|
| 9 |
+
* DepthAnything map is a heightfield, and each pixel marches the view ray to the
|
| 10 |
+
* FIRST surface hit — so it is occlusion-correct (no tear), the eyepoint shifts with
|
| 11 |
+
* SCROLL VELOCITY (no time term → no vibration; perfectly flat at rest), and it reads
|
| 12 |
+
* over ONE continuous strip (no seams). A depth aerial-perspective pass (haze +
|
| 13 |
+
* desaturation toward far) makes depth read even in a still frame.
|
| 14 |
*
|
| 15 |
* Sections stream in via addSection({imageUrl, depthUrl}); they are composited onto
|
| 16 |
* a growing strip canvas (so the texture is ONE continuous panorama, no quad seams).
|
|
|
|
| 28 |
let raf = 0, t0 = 0, lastT = 0, ok = false, reduced = false, aspect = 16 / 9;
|
| 29 |
// backing strip canvases (one continuous panorama + its depth), grown to the right
|
| 30 |
const STRIP_H = 512, STRIP_W = 8192; // ~21 sections of 384px; recycled when full
|
| 31 |
+
// 2.5D via a per-fragment STEEP-PARALLAX RAYMARCH (DepthFlow): depth is a heightfield; each pixel
|
| 32 |
+
// marches the view ray to the FIRST hit (occlusion-correct -> no tear), the eyepoint shifts with
|
| 33 |
+
// SCROLL VELOCITY (no time term -> no vibration; flat at rest), one continuous strip -> no seams.
|
| 34 |
const CAM_FOV = 38; // degrees
|
| 35 |
const CAM_DIST = 2.0; // camera distance from the plane center (the camera is STATIC)
|
| 36 |
const DEPTH_GAIN = 1.45; // contrast stretch for coarse, low-contrast DepthAnything-Small depth
|
| 37 |
const DEPTH_GAMMA = 0.85; // <1 lifts mid-depths so near content separates from far
|
| 38 |
+
const OVERSCAN = 1.0; // plane fills the frustum exactly so vUv maps 1:1 to the visible frame
|
| 39 |
+
const MESH_SEG = 1; // pure full-screen quad; relief is per-fragment now
|
| 40 |
+
const PARALLAX = 0.05; // eyepoint horizontal budget = 5% of view width (4-6% range)
|
| 41 |
+
const MARCH_LIN = 20; // coarse linear forward steps (first-hit bracket)
|
| 42 |
+
const MARCH_BIN = 7; // binary-refine steps inside the bracketed interval
|
| 43 |
+
const VEL_TAU = 0.18; // s — exponential smoothing time-const for scroll velocity
|
| 44 |
+
const VEL_REF = 220.0; // px/s mapped to full +/-1 parallax (scroll ~110px/s => ~0.5 budget)
|
| 45 |
+
const AERIAL_DESAT = 0.45; // max desaturation at the far plane
|
| 46 |
+
const AERIAL_HAZE = 0.38; // max lerp toward the haze tint at the far plane
|
| 47 |
+
const AERIAL_BLUR = 0.9; // texel-scale of the far-plane 4-tap micro-blur
|
| 48 |
+
const HAZE_TINT = [0.62, 0.68, 0.78]; // cool sky/haze tint
|
| 49 |
let imgCanvas = null, depCanvas = null, imgCtx = null, depCtx = null;
|
| 50 |
let imgTex = null, depTex = null;
|
| 51 |
let writeX = 0; // where the next section is pasted (px on the strip)
|
| 52 |
let scrollX = 0; // current left edge of the view window (px on the strip)
|
| 53 |
+
let prevScrollX = 0; // scrollX last frame, for d(scrollX)/dt
|
| 54 |
+
let velSmooth = 0; // smoothed px/s scroll velocity (drives the parallax eyepoint)
|
| 55 |
+
let _focusCache = 0.5, _focusFrame = 0; // CPU-pinned view-centre median depth (throttled)
|
| 56 |
let osReduced = false;
|
| 57 |
const queue = []; // sections {img, dep, meta, x} in arrival order, for focus/captions
|
| 58 |
let onFocus = null, focusX = -1;
|
| 59 |
const loader = new THREE.TextureLoader();
|
| 60 |
loader.crossOrigin = "anonymous";
|
| 61 |
|
| 62 |
+
// VERT: trivial pass-through full-screen quad (all the depth work is per-fragment now).
|
|
|
|
| 63 |
const VERT = `
|
| 64 |
precision highp float;
|
| 65 |
+
varying vec2 vUv;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
void main(){
|
| 67 |
+
vUv = uv;
|
| 68 |
+
gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
}`;
|
| 70 |
+
// FRAG: a steep-parallax (relief) raymarch over the strip-window. The depth strip is a heightfield
|
| 71 |
+
// (global [0,1], 0=far 1=near); for each pixel we march the view ray to the FIRST below-surface
|
| 72 |
+
// crossing (occlusion-correct -> no tear) and sample the painted strip there. The eyepoint offset
|
| 73 |
+
// P comes from smoothed scroll velocity (uHParallax). + aerial perspective from the hit's depth.
|
| 74 |
const FRAG = `
|
| 75 |
precision highp float;
|
| 76 |
+
varying vec2 vUv;
|
| 77 |
+
uniform sampler2D uImg; // continuous painted strip
|
| 78 |
+
uniform sampler2D uDep; // continuous depth strip, global [0,1], 0=far 1=near
|
| 79 |
+
uniform float uU0; // window left edge, normalized on the strip
|
| 80 |
+
uniform float uViewU; // window width, normalized
|
| 81 |
+
uniform float uTexel; // 1.0 / STRIP_W
|
| 82 |
+
uniform float uTexelV; // 1.0 / STRIP_H
|
| 83 |
+
uniform float uFocus; // CPU-pinned view-centre median depth (aerial pivot + swim guard)
|
| 84 |
+
uniform float uGain; // depth contrast gain
|
| 85 |
+
uniform float uGamma; // depth gamma (<1 lifts mids)
|
| 86 |
+
uniform float uHParallax; // signed eyepoint offset in [-1,1] from smoothed scroll velocity
|
| 87 |
+
uniform float uParallax; // budget (fraction of view width)
|
| 88 |
+
uniform float uAerialDesat;
|
| 89 |
+
uniform float uAerialHaze;
|
| 90 |
+
uniform float uAerialBlur;
|
| 91 |
+
uniform vec3 uHazeTint;
|
| 92 |
+
|
| 93 |
+
const int LIN = ${MARCH_LIN};
|
| 94 |
+
const int BIN = ${MARCH_BIN};
|
| 95 |
+
|
| 96 |
+
// window-uv (vUv) -> strip-uv, clamped into the painted canvas
|
| 97 |
+
vec2 toStrip(vec2 w){
|
| 98 |
+
return vec2(clamp(uU0 + w.x * uViewU, 0.0, 1.0), clamp(w.y, 0.001, 0.999));
|
| 99 |
+
}
|
| 100 |
+
// HEIGHTFIELD height in FULL [0,1]: contrast-stretched depth used directly as surface height so
|
| 101 |
+
// the ray can intersect anywhere along [1->0]. 0 = far (deep), 1 = near (tall).
|
| 102 |
+
float heightAt(vec2 w){
|
| 103 |
+
float d0 = texture2D(uDep, toStrip(w)).r;
|
| 104 |
+
float d = pow(clamp(d0, 0.0, 1.0), uGamma);
|
| 105 |
+
return clamp(uFocus + (d - uFocus) * uGain, 0.0, 1.0);
|
| 106 |
+
}
|
| 107 |
+
vec3 toSRGB(vec3 c){
|
| 108 |
c = max(c, vec3(0.0));
|
|
|
|
| 109 |
vec3 lo = c * 12.92; vec3 hi = 1.055 * pow(c, vec3(1.0/2.4)) - 0.055;
|
| 110 |
+
return mix(lo, hi, step(vec3(0.0031308), c));
|
| 111 |
+
}
|
| 112 |
+
void main(){
|
| 113 |
+
// view ray (parallax-occlusion): eyepoint shifts horizontally by P across the ray. ray param
|
| 114 |
+
// t in [0,1]; ray height descends 1 -> 0; surface = heightAt(baseUv - P*t). First t where
|
| 115 |
+
// rayH < surfH is the FIRST hit => occlusion-correct (no smear).
|
| 116 |
+
float P = uHParallax * uParallax; // signed, |P| <= uParallax
|
| 117 |
+
vec2 baseUv = vUv;
|
| 118 |
+
float prevT = 0.0;
|
| 119 |
+
bool hit = false;
|
| 120 |
+
float hitT = 1.0;
|
| 121 |
+
for (int i = 1; i <= LIN; i++){
|
| 122 |
+
float t = float(i) / float(LIN);
|
| 123 |
+
float rayH = 1.0 - t;
|
| 124 |
+
float surfH = heightAt(baseUv - vec2(P * t, 0.0));
|
| 125 |
+
if (rayH < surfH){ hitT = t; hit = true; break; }
|
| 126 |
+
prevT = t;
|
| 127 |
+
}
|
| 128 |
+
float tA = prevT, tB = hit ? hitT : 1.0;
|
| 129 |
+
for (int j = 0; j < BIN; j++){
|
| 130 |
+
float tm = 0.5 * (tA + tB);
|
| 131 |
+
float rayH = 1.0 - tm;
|
| 132 |
+
float surfH = heightAt(baseUv - vec2(P * tm, 0.0));
|
| 133 |
+
if (rayH < surfH) tB = tm; else tA = tm;
|
| 134 |
+
}
|
| 135 |
+
float tHit = 0.5 * (tA + tB);
|
| 136 |
+
vec2 hitUv = baseUv - vec2(P * tHit, 0.0);
|
| 137 |
+
|
| 138 |
+
vec2 sStrip = toStrip(hitUv);
|
| 139 |
+
vec3 c = texture2D(uImg, sStrip).rgb;
|
| 140 |
+
|
| 141 |
+
// AERIAL PERSPECTIVE: cue from the hit's height (0 far -> 1 near)
|
| 142 |
+
float dHit = texture2D(uDep, sStrip).r;
|
| 143 |
+
dHit = pow(clamp(dHit, 0.0, 1.0), uGamma);
|
| 144 |
+
dHit = clamp(uFocus + (dHit - uFocus) * uGain, 0.0, 1.0);
|
| 145 |
+
float farness = 1.0 - dHit;
|
| 146 |
+
if (uAerialBlur > 0.0 && farness > 0.01){
|
| 147 |
+
float r = uAerialBlur * farness;
|
| 148 |
+
vec3 b = texture2D(uImg, sStrip + vec2( uTexel*r, 0.0)).rgb
|
| 149 |
+
+ texture2D(uImg, sStrip + vec2(-uTexel*r, 0.0)).rgb
|
| 150 |
+
+ texture2D(uImg, sStrip + vec2(0.0, uTexelV*r)).rgb
|
| 151 |
+
+ texture2D(uImg, sStrip + vec2(0.0, -uTexelV*r)).rgb;
|
| 152 |
+
c = mix(c, b * 0.25, farness * 0.6);
|
| 153 |
+
}
|
| 154 |
+
float luma = dot(max(c, vec3(0.0)), vec3(0.299, 0.587, 0.114));
|
| 155 |
+
c = mix(c, vec3(luma), uAerialDesat * farness);
|
| 156 |
+
c = mix(c, uHazeTint, uAerialHaze * farness * farness);
|
| 157 |
+
|
| 158 |
+
c = toSRGB(c);
|
| 159 |
+
c *= mix(0.92, 1.05, dHit); // near a touch brighter (aerial cue)
|
| 160 |
+
float vy = smoothstep(0.0, 0.12, vUv.y) * smoothstep(1.0, 0.88, vUv.y);
|
| 161 |
c *= mix(0.82, 1.0, vy);
|
| 162 |
gl_FragColor = vec4(c, 1.0);
|
| 163 |
}`;
|
|
|
|
| 169 |
renderer = new THREE.WebGLRenderer({ canvas, antialias: true, alpha: false });
|
| 170 |
renderer.setPixelRatio(Math.min(devicePixelRatio || 1, 2));
|
| 171 |
scene = new THREE.Scene();
|
| 172 |
+
camera = new THREE.PerspectiveCamera(CAM_FOV, aspect, 0.05, 100);
|
| 173 |
camera.position.set(0, 0, CAM_DIST);
|
| 174 |
camera.lookAt(0, 0, 0);
|
| 175 |
imgCanvas = document.createElement("canvas"); imgCanvas.width = STRIP_W; imgCanvas.height = STRIP_H;
|
| 176 |
depCanvas = document.createElement("canvas"); depCanvas.width = STRIP_W; depCanvas.height = STRIP_H;
|
| 177 |
+
imgCtx = imgCanvas.getContext("2d"); depCtx = depCanvas.getContext("2d", { willReadFrequently: true });
|
| 178 |
imgCtx.fillStyle = "#0b0d12"; imgCtx.fillRect(0, 0, STRIP_W, STRIP_H);
|
| 179 |
depCtx.fillStyle = "#808080"; depCtx.fillRect(0, 0, STRIP_W, STRIP_H);
|
| 180 |
imgTex = new THREE.CanvasTexture(imgCanvas); imgTex.colorSpace = THREE.SRGBColorSpace;
|
| 181 |
depTex = new THREE.CanvasTexture(depCanvas);
|
| 182 |
for (const tx of [imgTex, depTex]) { tx.minFilter = THREE.LinearFilter; tx.magFilter = THREE.LinearFilter; tx.wrapS = THREE.ClampToEdgeWrapping; }
|
| 183 |
+
depTex.generateMipmaps = false; depTex.wrapT = THREE.ClampToEdgeWrapping;
|
|
|
|
|
|
|
|
|
|
| 184 |
mat = new THREE.ShaderMaterial({
|
| 185 |
uniforms: {
|
| 186 |
uImg: { value: imgTex }, uDep: { value: depTex },
|
| 187 |
uU0: { value: 0 }, uViewU: { value: 0.25 },
|
|
|
|
| 188 |
uTexel: { value: 1 / STRIP_W }, uTexelV: { value: 1 / STRIP_H },
|
| 189 |
+
uFocus: { value: 0.5 }, uGain: { value: DEPTH_GAIN }, uGamma: { value: DEPTH_GAMMA },
|
| 190 |
+
uHParallax: { value: 0 }, // driven by smoothed scroll velocity in loop()
|
| 191 |
+
uParallax: { value: PARALLAX },
|
| 192 |
+
uAerialDesat: { value: AERIAL_DESAT },
|
| 193 |
+
uAerialHaze: { value: AERIAL_HAZE },
|
| 194 |
+
uAerialBlur: { value: AERIAL_BLUR },
|
| 195 |
+
uHazeTint: { value: new THREE.Vector3(HAZE_TINT[0], HAZE_TINT[1], HAZE_TINT[2]) },
|
| 196 |
},
|
| 197 |
vertexShader: VERT, fragmentShader: FRAG,
|
| 198 |
+
depthWrite: false, depthTest: false,
|
| 199 |
});
|
| 200 |
+
mesh = new THREE.Mesh(new THREE.PlaneGeometry(2, 2, MESH_SEG, MESH_SEG), mat); // pure quad
|
| 201 |
scene.add(mesh);
|
| 202 |
+
fitMeshToFrustum(); // with OVERSCAN=1.0 it fits exactly -> vUv maps 1:1 to the frame
|
| 203 |
resize();
|
| 204 |
window.addEventListener("resize", resize);
|
| 205 |
const mq = window.matchMedia && window.matchMedia("(prefers-reduced-motion: reduce)");
|
|
|
|
| 209 |
} catch (e) { ok = false; }
|
| 210 |
}
|
| 211 |
|
| 212 |
+
// size the plane to fill the frustum at z=0 (OVERSCAN=1.0 -> exact fit, vUv 1:1 with the frame).
|
|
|
|
| 213 |
function fitMeshToFrustum() {
|
| 214 |
if (!camera || !mesh || !camera.isPerspectiveCamera) return;
|
| 215 |
const halfH = Math.tan((camera.fov * Math.PI / 180) / 2) * CAM_DIST;
|
|
|
|
| 226 |
aspect = w / h;
|
| 227 |
if (camera && camera.isPerspectiveCamera) {
|
| 228 |
camera.aspect = aspect;
|
| 229 |
+
camera.updateProjectionMatrix();
|
| 230 |
fitMeshToFrustum();
|
| 231 |
}
|
| 232 |
}
|
| 233 |
function reducedNow() { return reduced || osReduced || (document.body && document.body.classList.contains("reduced-motion")); }
|
| 234 |
|
| 235 |
+
// Pin the focus/aerial plane to the view-centre median-ish depth (CPU, cheap, throttled 1-in-6),
|
| 236 |
+
// so the picture doesn't swim and aerial perspective tracks the current scene.
|
| 237 |
+
function focusDepthTarget() {
|
| 238 |
+
if ((_focusFrame++ % 6) !== 0) return _focusCache;
|
| 239 |
+
try {
|
| 240 |
+
const viewU = Math.min(1, (aspect * STRIP_H) / STRIP_W);
|
| 241 |
+
const cx = Math.round(scrollX + viewU * STRIP_W * 0.5); // centre column on the strip (px)
|
| 242 |
+
if (cx < 0 || cx >= STRIP_W) return _focusCache;
|
| 243 |
+
const col = depCtx.getImageData(cx, Math.round(STRIP_H * 0.25), 1, Math.round(STRIP_H * 0.5)).data;
|
| 244 |
+
let s = 0, n = 0;
|
| 245 |
+
for (let i = 0; i < col.length; i += 4) { s += col[i]; n++; }
|
| 246 |
+
_focusCache = n ? (s / n) / 255 : 0.5;
|
| 247 |
+
} catch (_) { /* tainted/blocked -> keep last */ }
|
| 248 |
+
return _focusCache;
|
| 249 |
+
}
|
| 250 |
+
|
| 251 |
function loadImg(url) {
|
| 252 |
return new Promise((resolve) => {
|
| 253 |
if (!url) return resolve(null);
|
|
|
|
| 294 |
|
| 295 |
// Pre-smooth the DEPTH STRIP ONLY (never the picture): a separable 5-tap Gaussian with
|
| 296 |
// edge-clamp, run once per appended section, kills DepthAnything-Small's speckle and softens
|
| 297 |
+
// hard depth steps into gentle ramps BEFORE the raymarch reads them. Touches only depCanvas;
|
| 298 |
+
// imgCanvas is never blurred. Same-origin -> no taint; falls back to a raw draw if blocked.
|
|
|
|
| 299 |
function smoothDepthToStrip(dep, dx, w, h, blend = 0) {
|
| 300 |
const off = document.createElement("canvas"); off.width = w; off.height = h;
|
| 301 |
const oc = off.getContext("2d", { willReadFrequently: true });
|
|
|
|
| 304 |
const src = oc.getImageData(0, 0, w, h);
|
| 305 |
const a = src.data, out = src.data, n = w * h;
|
| 306 |
// SEAM AFFINE MATCH: pin this section's depth onto the EXISTING strip's depth scale over the
|
| 307 |
+
// FEATHER overlap so there is no DC/contrast cliff at the boundary (kills the strip-seam shear).
|
|
|
|
| 308 |
if (blend > 1 && dx > 0) {
|
| 309 |
try {
|
| 310 |
const bw = Math.min(blend, w);
|
|
|
|
| 379 |
depCtx.globalCompositeOperation = "copy";
|
| 380 |
depCtx.drawImage(depCanvas, -shift, 0); depCtx.globalCompositeOperation = "source-over";
|
| 381 |
writeX = Math.max(0, writeX - shift); scrollX = Math.max(0, scrollX - shift);
|
| 382 |
+
prevScrollX = Math.max(0, prevScrollX - shift); // so next-frame velocity isn't a spike
|
| 383 |
for (const s of queue) s.x -= shift;
|
| 384 |
}
|
| 385 |
|
|
|
|
| 388 |
if (!renderer || !scene || !camera) return;
|
| 389 |
const now = performance.now();
|
| 390 |
const dt = Math.min(0.05, (now - (lastT || now)) / 1000); lastT = now;
|
|
|
|
| 391 |
const rm = reducedNow();
|
| 392 |
const viewU = Math.min(1, (aspect * STRIP_H) / STRIP_W); // window width (normalized)
|
| 393 |
mat.uniforms.uViewU.value = viewU;
|
|
|
|
| 397 |
const speed = rm ? 0 : 110; // px/sec — the fast loop generates ~350px/s, so this flows
|
| 398 |
if (scrollX < maxScroll) scrollX = Math.min(maxScroll, scrollX + speed * dt);
|
| 399 |
mat.uniforms.uU0.value = scrollX / STRIP_W;
|
| 400 |
+
// 2.5D parallax: the eyepoint offset is driven by SCROLL VELOCITY (NOT time -> no vibration;
|
| 401 |
+
// NOT cumulative position -> no divergence). A cascade of one-pole filters can only monotonically
|
| 402 |
+
// approach its target, so it never rings; at rest it eases to 0 -> a clean still flat image.
|
| 403 |
+
const instVel = dt > 1e-4 ? (scrollX - prevScrollX) / dt : 0; // px/s this frame
|
| 404 |
+
prevScrollX = scrollX;
|
| 405 |
+
const aV = 1 - Math.exp(-dt / VEL_TAU); // one-pole smoothing
|
| 406 |
+
velSmooth += (instVel - velSmooth) * aV;
|
| 407 |
+
const clampedVel = Math.max(-2 * VEL_REF, Math.min(2 * VEL_REF, velSmooth)); // reject recycle jump
|
| 408 |
+
const targetVel = rm ? 0 : clampedVel;
|
| 409 |
+
const hTarget = Math.max(-1, Math.min(1, targetVel / VEL_REF)); // signed, bounded [-1,1]
|
| 410 |
+
mat.uniforms.uHParallax.value += (hTarget - mat.uniforms.uHParallax.value) * Math.min(1, dt * 6);
|
| 411 |
+
// pin the FOCUS/aerial plane to the view-centre median depth (slow ease, throttled probe)
|
| 412 |
+
mat.uniforms.uFocus.value += (focusDepthTarget() - mat.uniforms.uFocus.value) * Math.min(1, dt * 1.5);
|
| 413 |
// focus = the section under the centre of the view -> caption
|
| 414 |
const centerX = scrollX + viewU * STRIP_W * 0.5;
|
| 415 |
let f = null;
|
|
|
|
| 424 |
depCtx.fillStyle = "#808080"; depCtx.fillRect(0, 0, STRIP_W, STRIP_H);
|
| 425 |
imgTex.needsUpdate = depTex.needsUpdate = true;
|
| 426 |
writeX = 0; scrollX = 0; queue.length = 0; focusX = -1;
|
| 427 |
+
prevScrollX = 0; velSmooth = 0; _focusCache = 0.5; _focusFrame = 0;
|
| 428 |
+
if (mat) { mat.uniforms.uHParallax.value = 0; mat.uniforms.uFocus.value = 0.5; }
|
| 429 |
}
|
| 430 |
function setReducedMotion(f) { reduced = !!f; }
|
| 431 |
function dispose() { cancelAnimationFrame(raf); window.removeEventListener("resize", resize); if (renderer) renderer.dispose(); ok = false; }
|
scripts/deploy_private_space.py
CHANGED
|
@@ -62,7 +62,12 @@ def main() -> int:
|
|
| 62 |
for key, value in {
|
| 63 |
"LIGHTLOOM_PROFILE": "space",
|
| 64 |
"LIGHTLOOM_PRIVACY_MODE": "1",
|
| 65 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
"LIGHTLOOM_G1_REPS": os.getenv("LIGHTLOOM_G1_REPS", "5"),
|
| 67 |
}.items():
|
| 68 |
api.add_space_variable(space_id, key, value, token=token)
|
|
|
|
| 62 |
for key, value in {
|
| 63 |
"LIGHTLOOM_PROFILE": "space",
|
| 64 |
"LIGHTLOOM_PRIVACY_MODE": "1",
|
| 65 |
+
# OFF in production: this flag gates /internal/* probes AND the legacy /world + /panorama
|
| 66 |
+
# generators that load klein-BASE + a 3rd-party 360 LoRA NOT in the runtime ledger. Leaving
|
| 67 |
+
# it "1" silently re-enabled those un-ledgered endpoints for any judge -> a ledger/honesty
|
| 68 |
+
# contradiction. The live experience (/scroll_live) does not need it. Flip to "1" only for a
|
| 69 |
+
# local/offline bench run.
|
| 70 |
+
"LIGHTLOOM_ENABLE_INTERNAL_BENCH": os.getenv("LIGHTLOOM_ENABLE_INTERNAL_BENCH", "0"),
|
| 71 |
"LIGHTLOOM_G1_REPS": os.getenv("LIGHTLOOM_G1_REPS", "5"),
|
| 72 |
}.items():
|
| 73 |
api.add_space_variable(space_id, key, value, token=token)
|
src/lightloom/audio_in/asr.py
CHANGED
|
@@ -68,9 +68,16 @@ def _resample_to_16k(audio: Any, sr: int) -> Any:
|
|
| 68 |
return np.interp(x, xp, audio).astype("float32")
|
| 69 |
|
| 70 |
|
|
|
|
|
|
|
|
|
|
| 71 |
def load_asr() -> tuple[Any, Any]:
|
| 72 |
-
"""Load Whisper-turbo on CUDA (fp16). GPU only — call inside
|
| 73 |
-
Returns ``(processor, model)``.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
import torch
|
| 75 |
from transformers import WhisperForConditionalGeneration, WhisperProcessor
|
| 76 |
|
|
@@ -80,7 +87,8 @@ def load_asr() -> tuple[Any, Any]:
|
|
| 80 |
ref.repo_id, revision=ref.revision, torch_dtype=torch.float16
|
| 81 |
).to("cuda")
|
| 82 |
model.eval()
|
| 83 |
-
|
|
|
|
| 84 |
|
| 85 |
|
| 86 |
def _collapse_repeats(text: str) -> str:
|
|
|
|
| 68 |
return np.interp(x, xp, audio).astype("float32")
|
| 69 |
|
| 70 |
|
| 71 |
+
_ASR_CACHE: tuple[Any, Any] | None = None
|
| 72 |
+
|
| 73 |
+
|
| 74 |
def load_asr() -> tuple[Any, Any]:
|
| 75 |
+
"""Load Whisper-turbo on CUDA (fp16), CACHED at module scope. GPU only — call inside
|
| 76 |
+
`@spaces.GPU`. Returns ``(processor, model)``. (Without the cache Whisper was reloaded from
|
| 77 |
+
disk on EVERY transcription — a large per-clause latency hit; this mirrors load_fast_pipeline.)"""
|
| 78 |
+
global _ASR_CACHE
|
| 79 |
+
if _ASR_CACHE is not None:
|
| 80 |
+
return _ASR_CACHE
|
| 81 |
import torch
|
| 82 |
from transformers import WhisperForConditionalGeneration, WhisperProcessor
|
| 83 |
|
|
|
|
| 87 |
ref.repo_id, revision=ref.revision, torch_dtype=torch.float16
|
| 88 |
).to("cuda")
|
| 89 |
model.eval()
|
| 90 |
+
_ASR_CACHE = (processor, model)
|
| 91 |
+
return _ASR_CACHE
|
| 92 |
|
| 93 |
|
| 94 |
def _collapse_repeats(text: str) -> str:
|
src/lightloom/depth/depth.py
CHANGED
|
@@ -44,10 +44,13 @@ def load_depth_pipeline(device: str = "cuda") -> Any:
|
|
| 44 |
)
|
| 45 |
|
| 46 |
|
| 47 |
-
def estimate_depth(depth_pipeline: Any, image: Any, invert: bool = True) -> np.ndarray:
|
| 48 |
-
"""Run depth estimation on a PIL image
|
|
|
|
|
|
|
|
|
|
| 49 |
out = depth_pipeline(image)
|
| 50 |
predicted = out["predicted_depth"]
|
| 51 |
array = predicted.detach().cpu().numpy() if hasattr(predicted, "detach") else np.asarray(predicted)
|
| 52 |
-
array = np.squeeze(array)
|
| 53 |
-
return normalize_depth(array, invert=invert)
|
|
|
|
| 44 |
)
|
| 45 |
|
| 46 |
|
| 47 |
+
def estimate_depth(depth_pipeline: Any, image: Any, invert: bool = True, normalize: bool = True) -> np.ndarray:
|
| 48 |
+
"""Run depth estimation on a PIL image. With ``normalize`` (default) returns a per-image [0,1]
|
| 49 |
+
map (legacy). With ``normalize=False`` returns the RAW model disparity (un-scaled) so the caller
|
| 50 |
+
can apply ONE session-global scale across all strips — the only way adjacent strips share a depth
|
| 51 |
+
scale (per-strip min-max makes 'near' mean a different thing every section -> seams + flatness)."""
|
| 52 |
out = depth_pipeline(image)
|
| 53 |
predicted = out["predicted_depth"]
|
| 54 |
array = predicted.detach().cpu().numpy() if hasattr(predicted, "detach") else np.asarray(predicted)
|
| 55 |
+
array = np.squeeze(array).astype("float32")
|
| 56 |
+
return normalize_depth(array, invert=invert) if normalize else array
|