chore: deploy private lightloom build

app.py

@@ -1470,6 +1470,136 @@ def scroll_live_api(session: str = "", text: str = "", audio_b64: str = "", lang
         yield {"stage": stage, "error": str(exc)[:200], "session": sess}
 
 
+@spaces.GPU(duration=170)
+def _postprocess_world(session: str, lang: str = "en"):
+    """Director's Cut POST-PROCESS (Tier-1). 100% SEPARATE from the live painter: never calls
+    _scroll_live_phrase, only READS the finished section-*.webp off disk, fail-isolated. Stitches the panorama,
+    lets the MiniCPM-V-4.6 "Art Director" name it from the PIXELS (env-gated, try/except center-fallback to the
+    live text title), then renders a gentle clamped depth-parallax pan-through to world.mp4 (served by the
+    existing /frames-scroll route). torchvision is required by the seer's processor (in requirements.txt)."""
+    import math
+
+    import numpy as np
+    import torch
+    from PIL import Image
+
+    out_dir = _SCROLL_ROOT / session
+    strips = (
+        sorted(out_dir.glob("section-*.webp"), key=lambda p: int(re.match(r"section-(\d+)", p.stem).group(1)))
+        if out_dir.is_dir()
+        else []
+    )
+    if not strips:
+        yield {"stage": "postprocess_error", "error": "no_strips"}
+        return
+    # 1) STITCH the finished strips into ONE panorama
+    imgs = [Image.open(p).convert("RGB") for p in strips]
+    h0 = max(i.height for i in imgs)
+    w0 = sum(i.width for i in imgs)
+    pano = Image.new("RGB", (w0, h0))
+    x = 0
+    for im in imgs:
+        pano.paste(im, (x, 0))
+        x += im.width
+    pano.save(out_dir / "panorama.webp", "WEBP", quality=92)
+    yield {"stage": "stitched", "w": w0, "h": h0}
+
+    # 2) ART DIRECTOR (MiniCPM-V-4.6) — env-gated, try/except CENTER fallback so it can NEVER break the MP4
+    state = _live_state(out_dir)
+    seer = {
+        "title": (state.get("title") or "").strip(),
+        "caption": "",
+        "field_notes": [],
+        "focal_points": [{"x": 0.5, "y": 0.5, "label": ""}],
+    }
+    if os.getenv("LIGHTLOOM_WORLD_SEER", "1") == "1":
+        try:
+            from lightloom.paint.prompt import scrub_unsafe
+            from lightloom.vision.world_seer import load_world_seer, see_world
+
+            proc, vmodel = load_world_seer()
+            see_in = pano.copy()
+            see_in.thumbnail((1536, 1536))  # bound the VLM token count
+            seer = see_world(proc, vmodel, see_in, seed=len(strips))
+            seer["title"] = scrub_unsafe(seer.get("title") or "")[0]
+            seer["caption"] = scrub_unsafe(seer.get("caption") or "")[0]
+        except Exception as exc:  # noqa: BLE001 - naming is optional; never block the keepsake MP4
+            yield {"stage": "seer_skipped", "error": str(exc)[:160]}
+    yield {
+        "stage": "titled",
+        "title": seer["title"],
+        "caption": seer["caption"],
+        "field_notes": seer["field_notes"],
+    }
+
+    # 3) RAW float32 depth on the (downscaled) panorama -> one near/far scale (NOT the live 8-bit PNG)
+    from lightloom.depth.depth import estimate_depth, load_depth_pipeline
+    from lightloom.export.ldi import encode_frames_ffmpeg, normalize_disparity, warp_layer
+
+    depth_pipe = load_depth_pipeline()
+    render_h = 540
+    render_w = max(2, int(round(w0 * render_h / h0)))
+    render_w -= render_w % 2
+    render_pano = pano.resize((render_w, render_h))
+    raw_d = estimate_depth(depth_pipe, render_pano, normalize=False)
+    nd = normalize_disparity(np.asarray(raw_d, dtype=np.float32))
+
+    # 4) RENDER: a 16:9 window PANS across the wide panorama (ease-in-out) with a gentle depth-parallax
+    #    breathing -> "fly through your world". Tier-1: single plane, border-fill on the tiny gaps, no FLUX heal.
+    img_full = torch.from_numpy(np.asarray(render_pano, dtype=np.float32).transpose(2, 0, 1)[None] / 255.0).to("cuda")
+    nd_full = torch.from_numpy(nd[None, None].astype(np.float32)).to("cuda")
+    win_w = min(render_w, int(round(render_h * 16 / 9)))
+    win_w -= win_w % 2
+    travel = max(0, render_w - win_w)
+    n_frames = 144
+    frames_dir = out_dir / "frames"
+    frames_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        for k in range(n_frames):
+            t = k / (n_frames - 1) if n_frames > 1 else 0.0
+            ease = 0.5 - 0.5 * math.cos(math.pi * t)  # ease the pan
+            x0 = int(round(ease * travel))
+            win_img = img_full[:, :, :, x0 : x0 + win_w]
+            win_nd = nd_full[:, :, :, x0 : x0 + win_w]
+            dolly = 0.05 * (0.5 - 0.5 * math.cos(2 * math.pi * t))  # subtle parallax breathing
+            with torch.inference_mode():
+                warped, _gap = warp_layer(win_img, win_nd, 0.0, 0.0, dolly, padding_mode="border")
+            frame = warped[0].clamp(0, 1).mul(255).byte().cpu().numpy().transpose(1, 2, 0)
+            Image.fromarray(frame).save(frames_dir / f"frame-{k:04d}.webp", "WEBP", quality=88)
+            if k % 36 == 0:
+                yield {"stage": "rendering", "frame": k, "total": n_frames}
+        yield {"stage": "encoding"}
+        encode_frames_ffmpeg(frames_dir, out_dir / "world.mp4", fps=24, height=1080)
+    finally:
+        for f in frames_dir.glob("frame-*.webp"):
+            try:
+                f.unlink()
+            except OSError:
+                pass
+    yield {
+        "stage": "ready",
+        "video": f"/frames-scroll/{session}/world.mp4",
+        "title": seer["title"],
+        "caption": seer["caption"],
+    }
+
+
+@app.api(name="postprocess", concurrency_id="gpu", concurrency_limit=1, stream_every=0.05)
+def postprocess_api(session: str = "", lang: str = "en") -> dict:
+    sess = (session or "").strip()
+    if not re.fullmatch(r"[a-f0-9]{6,32}", sess):
+        yield {"stage": "postprocess_error", "error": "bad_session"}
+        return
+    try:
+        for event in _postprocess_world(sess, lang or "en"):
+            event.setdefault("session", sess)
+            yield event
+    except Exception as exc:  # noqa: BLE001 - quota guard; CANNOT affect the live painter (separate endpoint)
+        message = str(exc).lower()
+        stage = "quota_exceeded" if any(t in message for t in ("gpu", "quota", "zerogpu", "exceeded")) else "postprocess_error"
+        yield {"stage": stage, "error": str(exc)[:200], "session": sess}
+
+
 @spaces.GPU(duration=120)
 def _introspect_klein_on_gpu() -> dict[str, Any]:
     """Report the klein pipeline's call signature + LoRA/embeds capabilities, so we can

requirements.txt

@@ -7,6 +7,10 @@ pydantic==2.12.5
 numpy==2.2.6
 Pillow==12.2.0
 torch==2.11.0
+# torchvision is REQUIRED by the MiniCPM-V-4.6 "Art Director" processor (the AutoProcessor pulls a
+# MiniCPMV4_6VideoProcessor that requires_backends torchvision) — verified by the F0 smoke-load. 0.26.0
+# matches torch 2.11.0. Post-process (Director's Cut) only; the live painter never imports it.
+torchvision==0.26.0
 transformers==5.11.0
 diffusers @ git+https://github.com/huggingface/diffusers.git@784fa62652fb2719d415830f918fc32a49ecc7a1
 peft>=0.14

src/lightloom/core/config.py

@@ -120,6 +120,17 @@ MODEL_REFS: dict[str, ModelRef] = {
         notes="Runtime via transformers on GPU (LlamaForCausalLM). G3: llama.cpp CPU "
         "measured ~1 tok/s on the ZeroGPU Space regardless of threads.",
     ),
+    "world_seer": ModelRef(
+        role="Art Director / world-seer (Director's Cut post-process)",
+        repo_id="openbmb/MiniCPM-V-4.6",
+        revision=None,  # pin after the on-Space smoke-load (F0 PASSED locally on transformers 5.11.0)
+        license="apache-2.0",
+        params=1_300_000_000,
+        runtime=False,  # POST-PROCESS only — loaded at session-end, NEVER on the live painter slot
+        notes="MiniCPM-V-4.6 vision-language (MiniCPMV4_6ForConditionalGeneration). Native in transformers 5.11 "
+        "(no trust_remote_code) but REQUIRES torchvision (the AutoProcessor pulls a video processor). Names + "
+        "describes the FINISHED panorama and aims the keepsake fly-through camera. F0 smoke-load PASSED local.",
+    ),
     "director_gguf": ModelRef(
         role="Director (local llama.cpp GGUF)",
         repo_id="openbmb/MiniCPM5-1B-GGUF",

src/lightloom/export/ldi.py

@@ -0,0 +1,134 @@
+"""Layered-Depth-Image (LDI) Ken Burns fly-through helpers for the Director's Cut keepsake MP4.
+
+Runs in the session-end POST-PROCESS endpoint over the FINISHED panorama + its re-estimated float32 depth —
+NEVER on the live painter path. Pure torch / numpy / PIL, and a SYSTEM-ffmpeg encode (ffmpeg is in
+packages.txt -> ZERO new pip deps). The grid_sample warp runs on GPU at runtime; the depth-normalize /
+layer-split / camera-path / grid math here is pure CPU and unit-tested.
+
+Design constraints (verified):
+  - Depth source = `estimate_depth(..., normalize=False)` RAW float32 disparity (near=high), NOT the live
+    8-bit `section-*-depth.png` (session-renormalized -> wrong-scaled for back-projection).
+  - Motion is GENTLE and HARD-CLAMPED (the `stage-scroll.js` PARALLAX=0 scar): one focal point can never
+    widen the camera past the envelope, so disocclusion gaps stay small and FLUX heals them in-style.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+from typing import Any
+
+
+def normalize_disparity(disp: Any, lo_pct: float = 2.0, hi_pct: float = 98.0) -> Any:
+    """Robustly normalize RAW float32 disparity to [0,1] (1 = nearest) via percentiles, so ONE near/far scale
+    spans the whole panorama. Returns a float32 array; a flat input maps to all-zeros."""
+    import numpy as np
+
+    d = np.asarray(disp, dtype=np.float32)
+    lo, hi = np.percentile(d, lo_pct), np.percentile(d, hi_pct)
+    if hi - lo < 1e-6:
+        return np.zeros_like(d)
+    return np.clip((d - lo) / (hi - lo), 0.0, 1.0).astype(np.float32)
+
+
+def depth_to_layers(norm_disp: Any, n_layers: int = 3) -> list:
+    """Split normalized disparity [0,1] into ``n_layers`` boolean masks, NEAR (index 0) -> FAR, equal-width
+    bands. Every pixel lands in exactly one layer; the disoccludable background is the union of farther
+    layers behind each nearer one."""
+    import numpy as np
+
+    n = max(1, int(n_layers))
+    d = np.asarray(norm_disp, dtype=np.float32)
+    edges = np.linspace(0.0, 1.0, n + 1)  # edges[0]=0 .. edges[n]=1
+    masks = []
+    for i in range(n):  # i=0 -> nearest band (highest disparity)
+        hi = edges[n - i]
+        lo = edges[n - i - 1]
+        if i == n - 1:  # farthest band is inclusive at the bottom so disparity==0 is covered
+            masks.append((d <= hi) & (d >= lo))
+        else:
+            masks.append((d <= hi) & (d > lo))
+    return masks
+
+
+def clamped_camera_path(
+    n_frames: int,
+    focal_x: float = 0.5,
+    focal_y: float = 0.5,
+    max_shift: float = 0.06,
+    max_dolly: float = 0.10,
+) -> list:
+    """A gentle, CLAMPED dolly-in + small lateral path (ease-in-out), softly biased toward (focal_x, focal_y).
+    Returns ``n_frames`` ``(shift_x, shift_y, dolly)`` with amplitudes HARD-bounded by max_shift/max_dolly so
+    one focal point can never widen motion past the gentle envelope (the PARALLAX=0 scar). Frame 0 is the
+    rest pose (0,0,0)."""
+    import numpy as np
+
+    n = max(1, int(n_frames))
+    t = np.linspace(0.0, 1.0, n)
+    ease = 0.5 - 0.5 * np.cos(np.pi * t)  # smooth 0..1 ease-in-out, ease[0]=0
+    bx = (min(1.0, max(0.0, float(focal_x))) - 0.5) * 2.0  # focal bias in [-1,1]
+    by = (min(1.0, max(0.0, float(focal_y))) - 0.5) * 2.0
+    sx = bx * max_shift * ease
+    sy = by * (max_shift * 0.5) * ease  # vertical motion is half the horizontal envelope
+    dolly = max_dolly * ease
+    return [(float(sx[i]), float(sy[i]), float(dolly[i])) for i in range(n)]
+
+
+def build_sample_grid(h: int, w: int) -> Any:
+    """Base identity sampling grid in [-1,1] for ``torch.nn.functional.grid_sample`` — shape (1, h, w, 2),
+    last dim = (x, y)."""
+    import torch
+
+    ys = torch.linspace(-1.0, 1.0, h)
+    xs = torch.linspace(-1.0, 1.0, w)
+    gy, gx = torch.meshgrid(ys, xs, indexing="ij")
+    return torch.stack((gx, gy), dim=-1).unsqueeze(0)
+
+
+def warp_layer(
+    img_chw: Any,
+    norm_disp_hw: Any,
+    shift_x: float,
+    shift_y: float,
+    dolly: float,
+    padding_mode: str = "border",
+) -> tuple:
+    """Depth-displaced backward warp of one RGB layer (GPU at runtime). ``img_chw`` (1,3,H,W) in [0,1],
+    ``norm_disp_hw`` (1,1,H,W) in [0,1]. Returns ``(warped (1,3,H,W), gap_mask (1,1,H,W))`` where gap_mask=1
+    marks disoccluded pixels (sampled from outside the source). Tier-1 uses ``padding_mode="border"`` so the
+    tiny clamped-motion gaps are filled by the edge (no black / no heal); Tier-2 LDI uses the gap_mask to FLUX-
+    heal the disoccluded background. The gap_mask is computed from grid bounds, independent of padding_mode."""
+    import torch
+    import torch.nn.functional as F
+
+    _, _, h, w = img_chw.shape
+    base = build_sample_grid(h, w).to(img_chw.device)
+    # parallax: nearer pixels (high disparity) move MORE; dolly scales the whole field outward.
+    disp = norm_disp_hw[0, 0]
+    off_x = disp * shift_x + base[0, :, :, 0] * dolly
+    off_y = disp * shift_y + base[0, :, :, 1] * dolly
+    grid = base.clone()
+    grid[0, :, :, 0] = base[0, :, :, 0] + off_x
+    grid[0, :, :, 1] = base[0, :, :, 1] + off_y
+    warped = F.grid_sample(img_chw, grid, mode="bilinear", padding_mode=padding_mode, align_corners=True)
+    inb = ((grid[..., 0].abs() <= 1.0) & (grid[..., 1].abs() <= 1.0)).float().unsqueeze(1)
+    gap = 1.0 - inb
+    return warped, gap
+
+
+def encode_frames_ffmpeg(frames_dir: Any, out_path: Any, fps: int = 25, height: int = 1080) -> Path:
+    """Encode ``frame-*.webp`` in ``frames_dir`` into an H.264 yuv420p MP4 via the SYSTEM ffmpeg
+    (packages.txt) — zero new pip deps. Scales to ``height`` p (width auto, even). Raises
+    ``subprocess.CalledProcessError`` if ffmpeg fails."""
+    d = Path(frames_dir)
+    out = Path(out_path)
+    cmd = [
+        "ffmpeg", "-y", "-framerate", str(int(fps)),
+        "-pattern_type", "glob", "-i", str(d / "frame-*.webp"),
+        "-vf", f"scale=-2:{int(height)}:flags=lanczos,format=yuv420p",
+        "-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
+        str(out),
+    ]
+    subprocess.run(cmd, check=True, capture_output=True)
+    return out

src/lightloom/vision/world_seer.py

@@ -0,0 +1,162 @@
+"""MiniCPM-V-4.6 "Art Director" world-seer (Director's Cut POST-PROCESS only — NEVER the live painter path).
+
+Reads the FINISHED painted panorama at session end and names + describes it from the ACTUAL pixels, and flags
+the focal regions that aim the keepsake fly-through camera. This is the Best-MiniCPM-Build contribution.
+
+F0 smoke-load VERIFIED on transformers 5.11.0 (local, 2026-06-14): `AutoModelForImageTextToText` loads
+natively (class MiniCPMV4_6ForConditionalGeneration, 1.30B, NO trust_remote_code); the 2-call processor form +
+greedy decode returned vision-grounded JSON in ~2.7s warm. REQUIRES `torchvision` (the AutoProcessor pulls a
+MiniCPMV4_6VideoProcessor) -> must be pinned in requirements.txt; adding it forces ONE full Space rebuild.
+
+Parameter contribution: openbmb/MiniCPM-V-4.6 ~1.30B (register in the ledger). Loading/inference run under
+`@spaces.GPU` in the session-end endpoint; `parse_world_json`/`_coerce` are pure CPU and unit-tested.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+WORLD_SEER_MODEL = "openbmb/MiniCPM-V-4.6"
+
+_SEER_PROMPT = (
+    "You are looking at a finished painted panorama — one continuous imagined world. "
+    "Reply with EXACTLY ONE JSON object and NOTHING else, in this shape:\n"
+    '{"title": "a 2-4 word evocative world name in Title Case",'
+    ' "caption": "one short evocative sentence about the world",'
+    ' "field_notes": ["3-5 short noun phrases naming things actually visible"],'
+    ' "focal_points": [{"x": 0.0-1.0, "y": 0.0-1.0, "label": "the most striking thing here"}]}\n'
+    "x,y are the fractional centre of each striking subject (0,0 = top-left). List 1-3 focal_points."
+)
+
+_FALLBACK_TITLES = (
+    "A World Unfolding",
+    "Where the Light Remembers",
+    "The Long Bright Drift",
+    "A Quiet Country",
+)
+
+_SEER_CACHE: tuple[Any, Any] | None = None
+
+
+def load_world_seer() -> tuple[Any, Any]:
+    """Load MiniCPM-V-4.6 on CUDA (bf16), CACHED at module scope. GPU only — call inside the session-end
+    `@spaces.GPU` post-process endpoint, NEVER on the live painter path. Returns ``(processor, model)``.
+    Native in transformers 5.11 (no trust_remote_code); requires torchvision (the AutoProcessor pulls a
+    video processor)."""
+    global _SEER_CACHE
+    if _SEER_CACHE is not None:
+        return _SEER_CACHE
+    import torch
+    from transformers import AutoModelForImageTextToText, AutoProcessor
+
+    processor = AutoProcessor.from_pretrained(WORLD_SEER_MODEL)
+    model = (
+        AutoModelForImageTextToText.from_pretrained(WORLD_SEER_MODEL, dtype=torch.bfloat16)
+        .to("cuda")
+        .eval()
+    )
+    _SEER_CACHE = (processor, model)
+    return _SEER_CACHE
+
+
+def _iter_json_objects(text: str):
+    """Yield every top-level balanced-brace ``{...}`` object in ``text``. MiniCPM-V sometimes emits several
+    separate JSON objects (e.g. ``{"title":..}{"caption":..}``) instead of one; ``parse_world_json`` merges
+    them. String-aware so a brace inside a quoted value never miscounts the depth."""
+    depth = 0
+    start = -1
+    in_str = False
+    esc = False
+    for i, ch in enumerate(text):
+        if in_str:
+            if esc:
+                esc = False
+            elif ch == "\\":
+                esc = True
+            elif ch == '"':
+                in_str = False
+            continue
+        if ch == '"':
+            in_str = True
+        elif ch == "{":
+            if depth == 0:
+                start = i
+            depth += 1
+        elif ch == "}":
+            if depth > 0:
+                depth -= 1
+                if depth == 0 and start >= 0:
+                    yield text[start : i + 1]
+                    start = -1
+
+
+def parse_world_json(text: str) -> dict:
+    """Tolerantly parse the seer decode: merge every balanced-brace JSON object found (the model may split
+    fields across several ``{}``), keeping the FIRST value seen for each key, ignoring unparseable fragments."""
+    merged: dict = {}
+    for blob in _iter_json_objects(text or ""):
+        try:
+            obj = json.loads(blob)
+        except Exception:  # noqa: BLE001 - skip a malformed fragment, keep the rest
+            continue
+        if isinstance(obj, dict):
+            for key, value in obj.items():
+                if key not in merged:
+                    merged[key] = value
+    return merged
+
+
+def _coerce(merged: dict, seed: int = 0) -> dict:
+    """Fill every field with a safe default so the post-process always has a usable, clean result (a VLM
+    failure can never blank or break the keepsake)."""
+    title = re.sub(r"[\"'“”]", "", str(merged.get("title") or "")).strip()
+    title = re.sub(r"[.;:!?,]+$", "", title).strip()
+    if len(title.split()) > 6:  # runaway -> first 6 words
+        title = " ".join(title.split()[:6])
+    if not title or not re.search(r"[A-Za-z]", title):
+        title = _FALLBACK_TITLES[abs(seed) % len(_FALLBACK_TITLES)]
+    title = title[:60]
+
+    caption = re.sub(r"\s+", " ", str(merged.get("caption") or "")).strip()[:200]
+
+    notes: list[str] = []
+    if isinstance(merged.get("field_notes"), list):
+        for note in merged["field_notes"]:
+            cleaned = re.sub(r"\s+", " ", str(note)).strip()[:60]
+            if cleaned:
+                notes.append(cleaned)
+    notes = notes[:5]
+
+    focal: list[dict] = []
+    if isinstance(merged.get("focal_points"), list):
+        for fp in merged["focal_points"]:
+            if not isinstance(fp, dict):
+                continue
+            try:
+                x = min(1.0, max(0.0, float(fp.get("x", 0.5))))
+                y = min(1.0, max(0.0, float(fp.get("y", 0.5))))
+            except (TypeError, ValueError):
+                continue
+            label = re.sub(r"\s+", " ", str(fp.get("label") or "")).strip()[:60]
+            focal.append({"x": x, "y": y, "label": label})
+    focal = focal[:3] or [{"x": 0.5, "y": 0.5, "label": ""}]  # CENTER fallback: the camera always has a target
+
+    return {"title": title, "caption": caption, "field_notes": notes, "focal_points": focal}
+
+
+def see_world(processor: Any, model: Any, panorama: Any, max_new_tokens: int = 192, seed: int = 0) -> dict:
+    """Read the finished panorama (PIL.Image) -> ``{title, caption, field_notes, focal_points}``. Uses the
+    F0-verified 2-call form (``apply_chat_template(tokenize=False)`` -> ``processor(images=, text=)`` ->
+    ``generate``); greedy decode (``do_sample=False``) was verified to emit cleanly. ALWAYS returns a coerced
+    dict (never raises on bad JSON) so a VLM failure cannot break the post-process."""
+    import torch
+
+    messages = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": _SEER_PROMPT}]}]
+    text = processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
+    inputs = processor(images=[panorama], text=text, return_tensors="pt").to(model.device)
+    with torch.inference_mode():
+        out = model.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=False)
+    decoded = processor.batch_decode(out[:, inputs["input_ids"].shape[1] :], skip_special_tokens=True)[0]
+    return _coerce(parse_world_json(decoded), seed=seed)