Spaces:
Running on Zero
Running on Zero
Download lot/flow.py from hugging-apps/mpeg-diffusion: direct link, hf CLI and curl.
- Browser
- Download file 12.7 kB
-
https://huggingface.co/spaces/hugging-apps/mpeg-diffusion/resolve/main/lot/flow.py
- Command line
-
hf download hf://spaces/hugging-apps/mpeg-diffusion/lot/flow.py
-
curl -L -o flow.py https://huggingface.co/spaces/hugging-apps/mpeg-diffusion/resolve/main/lot/flow.py
12.7 kB
| """Patch-wise asymmetric flow for Level-of-Token Diffusion. | |
| The velocity algebra is LakonLab's ``AsymFlowMixin.asymflow_velocity`` with | |
| scale ``s = 1`` and calibration ``k = 1``. That special case is equation 9 of | |
| Kiyohiro Nakayama, Brian Chao, Jan Ackermann, Hansheng Chen, Federico Tombari, Leonidas Guibas, Lior Yariv and Gordon Wetzstein, 2026. LoT does not use AsymFlow's per-extent timestep | |
| calibration (eq. 18). Extent scales are applied to the clean latent instead | |
| (eq. 16), which keeps a single timestep for every token. | |
| LakonLab is loaded from the ``common.py`` file directly. Importing the | |
| ``lakonlab`` package pulls in ``mmcv``, which this path does not need. | |
| """ | |
| from __future__ import annotations | |
| import importlib.util | |
| import os | |
| from functools import lru_cache | |
| from pathlib import Path | |
| import torch | |
| from layout import LotLayout, TokenRect | |
| def _lakonlab_common_path() -> Path: | |
| override = os.environ.get("LAKONLAB_ROOT") | |
| candidates: list[Path] = [] | |
| if override: | |
| candidates.append(Path(override)) | |
| repo = Path(__file__).resolve().parents[2] | |
| candidates.append(repo.parent / "LakonLab") | |
| for root in candidates: | |
| path = root / "lakonlab" / "models" / "architectures" / "asymflow" / "common.py" | |
| if path.is_file(): | |
| return path | |
| searched = ", ".join(str(path) for path in candidates) | |
| raise FileNotFoundError(f"LakonLab asymflow/common.py not found. Looked in: {searched}") | |
| def _asymflow(): | |
| path = _lakonlab_common_path() | |
| spec = importlib.util.spec_from_file_location("lakonlab_asymflow_common", path) | |
| if spec is None or spec.loader is None: | |
| raise ImportError(f"could not load {path}") | |
| module = importlib.util.module_from_spec(spec) | |
| spec.loader.exec_module(module) | |
| class _VelocityKernel(module.AsymFlowMixin): | |
| """One Procrustes basis, otherwise the LakonLab velocity step.""" | |
| def __init__(self, basis: torch.Tensor): | |
| self.proj_buffer = basis | |
| self.sigma_min = 1e-6 | |
| self.training = True | |
| return module.AsymFlowCalibration, _VelocityKernel, path | |
| def mean_basis(token_dim: int, extent: tuple[int, int, int]) -> torch.Tensor: | |
| """Semi-orthonormal lift that averages the uniform tokens inside a patch. | |
| ``A`` has shape ``(et*eh*ew*D, D)`` and ``A^T A = I``. For a unit extent | |
| this is the identity, so a dense layout is unchanged. This is the | |
| stand-in used before a Procrustes fit; it is not the paper's data-fit basis. | |
| """ | |
| et, eh, ew = extent | |
| sites = et * eh * ew | |
| if token_dim < 1 or sites < 1: | |
| raise ValueError("token dim and extent must be positive") | |
| eye = torch.eye(token_dim) | |
| return eye.repeat(sites, 1) / math_sqrt(sites) | |
| def math_sqrt(value: int) -> float: | |
| return value ** 0.5 | |
| def fit_procrustes(dense: torch.Tensor, reference: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: | |
| """Orthogonal Procrustes basis and RMS scale, eqs. 14–15. | |
| ``dense`` is ``(N, D_e)`` and ``reference`` is ``(N, D)``, the aligned | |
| multi-scale VAE tokens. The SVD matches | |
| ``LakonLab/tools/asymflow_subspace_procrustes.py``: ``A = V U^T`` from | |
| ``Z^T X = U S V^T``. ``A`` is ``(D_e, D)`` with ``A^T A = I``, and ``s`` | |
| matches the RMS of ``A^T x`` to the RMS of ``z``. | |
| """ | |
| if dense.ndim != 2 or reference.ndim != 2 or dense.shape[0] != reference.shape[0]: | |
| raise ValueError("dense and reference must be (N, D_e) and (N, D)") | |
| if reference.shape[1] > dense.shape[1]: | |
| raise ValueError("reference dimension exceeds the dense patch dimension") | |
| dense64 = dense.detach().double() | |
| reference64 = reference.detach().double() | |
| cross = reference64.T @ dense64 | |
| pixel_gram = dense64.T @ dense64 | |
| latent_norm_sq = (reference64 * reference64).sum().clamp(min=1e-12) | |
| u, _, vh = torch.linalg.svd(cross, full_matrices=False) | |
| rank = cross.shape[0] | |
| basis = (vh[:rank].T @ u[:, :rank].T).contiguous() | |
| projected = torch.trace(basis.T @ pixel_gram @ basis) | |
| scale = (projected / latent_norm_sq).clamp(min=1e-12).sqrt() | |
| return basis.float(), scale.float().reshape(()) | |
| def compress_patches(dense: torch.Tensor, basis: torch.Tensor) -> torch.Tensor: | |
| """Eq. 7. ``dense`` is ``(..., D_e)`` and the result is ``(..., D)``.""" | |
| return dense @ basis.to(dtype=dense.dtype, device=dense.device) | |
| def asymmetric_target(clean: torch.Tensor, noise: torch.Tensor, basis: torch.Tensor) -> torch.Tensor: | |
| """Eq. 8, row-vector form: ``P eps - x0`` with ``P = A A^T``.""" | |
| projector = basis @ basis.T | |
| return noise @ projector.to(dtype=noise.dtype, device=noise.device) - clean | |
| def recover_dense_velocity( | |
| u_a: torch.Tensor, | |
| x_t: torch.Tensor, | |
| basis: torch.Tensor, | |
| sigma: torch.Tensor | float, | |
| ) -> torch.Tensor: | |
| """Recover a full-rank patch velocity from an asymmetric prediction (eq. 9). | |
| ``u_a`` and ``x_t`` share shape ``(..., D_e)``. ``sigma`` is a scalar or a | |
| batch vector of length ``u_a.shape[0]``. The denominator is clamped at | |
| ``1e-6``, as in the paper. | |
| """ | |
| if u_a.shape != x_t.shape: | |
| raise ValueError(f"u_a shape {tuple(u_a.shape)} != x_t shape {tuple(x_t.shape)}") | |
| if basis.ndim != 2 or basis.shape[0] != u_a.shape[-1]: | |
| raise ValueError("basis must be (D_e, D) for this patch") | |
| sigma_t = _broadcast_sigma(sigma, u_a) | |
| basis_f = basis.detach().to(device=u_a.device, dtype=torch.float32) | |
| if os.environ.get("LOT_NO_LAKONLAB") == "1": | |
| return asymflow_velocity_torch(u_a, x_t, basis_f, sigma_t).to(dtype=u_a.dtype) | |
| try: | |
| calibration_cls, kernel_cls, _path = _asymflow() | |
| except FileNotFoundError: | |
| # No LakonLab checkout (for example inside ComfyUI): the same s = k = 1 arithmetic. | |
| return asymflow_velocity_torch(u_a, x_t, basis_f, sigma_t).to(dtype=u_a.dtype) | |
| ones = torch.ones((), device=u_a.device, dtype=torch.float32) | |
| calibration = calibration_cls(s=ones, k=ones, timestep=ones, sigma=sigma_t) | |
| recovered = kernel_cls(basis_f).asymflow_velocity(u_a, x_t, calibration) | |
| return recovered.to(dtype=u_a.dtype) | |
| def asymflow_velocity_torch( | |
| u_a: torch.Tensor, | |
| x_t: torch.Tensor, | |
| basis: torch.Tensor, | |
| sigma: torch.Tensor, | |
| sigma_min: float = 1e-6, | |
| ) -> torch.Tensor: | |
| """LakonLab ``asymflow_velocity`` with ``s = k = 1``, in plain torch (eq. 9). | |
| ``sub(v) = v A A^T`` and ``comp(v) = v - sub(v)``; the result is | |
| ``sub(u_a) + (comp(x_t) + comp(u_a)) / max(sigma, sigma_min)``. Matches the | |
| kernel's training-mode ``sigma_min``, which is what ``_VelocityKernel`` sets. | |
| """ | |
| u = u_a.float() | |
| x = x_t.float() | |
| projector = basis @ basis.T | |
| u_sub = u @ projector | |
| x_sub = x @ projector | |
| sig = sigma.float().clamp(min=sigma_min) | |
| return u_sub + ((x - x_sub) + (u - u_sub)) / sig | |
| def _broadcast_sigma(sigma: torch.Tensor | float, like: torch.Tensor) -> torch.Tensor: | |
| if not torch.is_tensor(sigma): | |
| sigma = torch.tensor(float(sigma), dtype=torch.float32) | |
| sigma = sigma.to(device=like.device, dtype=torch.float32) | |
| if sigma.ndim == 0: | |
| return sigma | |
| if sigma.ndim == 1: | |
| if sigma.shape[0] != like.shape[0]: | |
| raise ValueError(f"sigma batch {sigma.shape[0]} != {like.shape[0]}") | |
| return sigma.reshape(like.shape[0], *([1] * (like.ndim - 1))) | |
| if sigma.shape == like.shape[: sigma.ndim]: | |
| return sigma.reshape(*sigma.shape, *([1] * (like.ndim - sigma.ndim))) | |
| raise ValueError(f"cannot broadcast sigma shape {tuple(sigma.shape)} onto {tuple(like.shape)}") | |
| def sample_noisy( | |
| y0: torch.Tensor, | |
| sigma: torch.Tensor | float, | |
| noise: torch.Tensor | None = None, | |
| ) -> tuple[torch.Tensor, torch.Tensor]: | |
| """``y_t = (1 - sigma) y0 + sigma eps`` on the scaled lattice.""" | |
| if noise is None: | |
| noise = torch.randn_like(y0) | |
| sig = _broadcast_sigma(sigma, y0).to(dtype=y0.dtype) | |
| return (1 - sig) * y0 + sig * noise, noise | |
| def clean_from_velocity( | |
| y_t: torch.Tensor, | |
| velocity: torch.Tensor, | |
| sigma: torch.Tensor | float, | |
| ) -> torch.Tensor: | |
| """``y0_hat = y_t - sigma u``.""" | |
| sig = _broadcast_sigma(sigma, y_t).to(dtype=y_t.dtype) | |
| return y_t - sig * velocity | |
| def lot_clean_loss( | |
| y0_hat: torch.Tensor, | |
| y0: torch.Tensor, | |
| sigma: torch.Tensor | float, | |
| sigma_floor: float = 0.05, | |
| ) -> torch.Tensor: | |
| """Weighted clean-data loss, eq. 17. | |
| The paper writes a squared Euclidean norm inside the expectation. This | |
| returns the mean over elements of that weighted square, which has the | |
| same minimizer on a fixed grid and does not grow with the lattice size. | |
| """ | |
| sig = _broadcast_sigma(sigma, y0).clamp(min=sigma_floor) | |
| weight = 1.0 / sig.square() | |
| return ((y0_hat - y0).float().square() * weight).mean() | |
| def lot_h3_clean_loss( | |
| out: torch.Tensor, | |
| y_t: torch.Tensor, | |
| y0: torch.Tensor, | |
| sigma: torch.Tensor | float, | |
| sigma_floor: float = 0.05, | |
| ) -> torch.Tensor: | |
| """Eq. 17 for H3's head, which predicts ``x0 - eps`` (H3's scheduler: ``x0 = x_t + sigma * v``). | |
| The clean estimate is ``y_t + sigma * out``, not ``clean_from_velocity`` (that | |
| is the ``eps - x0`` convention). For ``sigma >= sigma_floor`` this equals | |
| ``mean((out - (y0 - eps))^2)``: the ordinary H3 velocity MSE, taken in y-space. | |
| """ | |
| sig = _broadcast_sigma(sigma, y0).to(dtype=torch.float32) | |
| y0_hat = y_t.float() + sig * out.float() | |
| weight = 1.0 / sig.clamp(min=sigma_floor).square() | |
| return ((y0_hat - y0.float()).square() * weight).mean() | |
| def _one_extent(rects: list[TokenRect]) -> tuple[int, int, int]: | |
| if not rects: | |
| raise ValueError("expected at least one rectangle") | |
| extent = rects[0].extent | |
| for rect in rects[1:]: | |
| if rect.extent != extent: | |
| raise ValueError(f"rectangles mix extents {extent} and {rect.extent}") | |
| return extent | |
| def _windows(rects: list[TokenRect], device: torch.device) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: | |
| """Broadcast indices ``(n, et, eh, ew)`` for every rectangle of one extent.""" | |
| et, eh, ew = _one_extent(rects) | |
| t = torch.tensor([rect.t for rect in rects], dtype=torch.long, device=device) | |
| u = torch.tensor([rect.u for rect in rects], dtype=torch.long, device=device) | |
| v = torch.tensor([rect.v for rect in rects], dtype=torch.long, device=device) | |
| dt = torch.arange(et, device=device) | |
| du = torch.arange(eh, device=device) | |
| dv = torch.arange(ew, device=device) | |
| tt = t[:, None, None, None] + dt[None, :, None, None] | |
| uu = u[:, None, None, None] + du[None, None, :, None] | |
| vv = v[:, None, None, None] + dv[None, None, None, :] | |
| return tt, uu, vv | |
| def apply_extent_scales( | |
| latent: torch.Tensor, | |
| layout: LotLayout, | |
| scales: dict[tuple[int, int, int], torch.Tensor], | |
| *, | |
| invert: bool = False, | |
| ) -> torch.Tensor: | |
| """Divide each patch by its extent scale. ``invert=True`` multiplies it back. | |
| One index write per extent. A unit scale hides a swapped multiply, so tests | |
| use a scale other than 1. | |
| """ | |
| out = latent.clone() | |
| buckets: dict[tuple[int, int, int], list[TokenRect]] = {} | |
| for rect in layout.rects: | |
| buckets.setdefault(rect.extent, []).append(rect) | |
| for extent, rects in buckets.items(): | |
| scale = scales[extent].to(device=latent.device, dtype=latent.dtype) | |
| tt, uu, vv = _windows(rects, latent.device) | |
| region = out[:, tt, uu, vv, :] | |
| region = region * scale if invert else region / scale | |
| out[:, tt, uu, vv, :] = region | |
| return out | |
| def gather_extent(tokens: torch.Tensor, rects: list[TokenRect]) -> torch.Tensor: | |
| """Pack patches of one extent to ``(B, n, D_e)``. | |
| Site order is C-order over ``(et, eh, ew, D)``, with the token channel | |
| fastest. Procrustes rows must use this same order. One index read for the | |
| whole extent. | |
| """ | |
| tt, uu, vv = _windows(rects, tokens.device) | |
| patches = tokens[:, tt, uu, vv, :] | |
| return patches.reshape(tokens.shape[0], len(rects), -1) | |
| def scatter_extent(canvas: torch.Tensor, values: torch.Tensor, rects: list[TokenRect]) -> None: | |
| """Write ``(B, n, D_e)`` patches back onto a fresh ``(B, T, H, W, D)`` canvas.""" | |
| tt, uu, vv = _windows(rects, canvas.device) | |
| et, eh, ew = _one_extent(rects) | |
| patches = values.reshape(canvas.shape[0], len(rects), et, eh, ew, canvas.shape[-1]) | |
| canvas[:, tt, uu, vv, :] = patches | |
| def euler_step(y_t: torch.Tensor, velocity: torch.Tensor, t: float, t_next: float) -> torch.Tensor: | |
| """One explicit step of ``dy/dt = u`` from ``t`` to ``t_next``.""" | |
| return y_t + (float(t_next) - float(t)) * velocity | |