File size: 12,677 Bytes
6febba8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
"""Patch-wise asymmetric flow for Level-of-Token Diffusion.

The velocity algebra is LakonLab's ``AsymFlowMixin.asymflow_velocity`` with
scale ``s = 1`` and calibration ``k = 1``. That special case is equation 9 of
Kiyohiro Nakayama, Brian Chao, Jan Ackermann, Hansheng Chen, Federico Tombari, Leonidas Guibas, Lior Yariv and Gordon Wetzstein, 2026. LoT does not use AsymFlow's per-extent timestep
calibration (eq. 18). Extent scales are applied to the clean latent instead
(eq. 16), which keeps a single timestep for every token.

LakonLab is loaded from the ``common.py`` file directly. Importing the
``lakonlab`` package pulls in ``mmcv``, which this path does not need.
"""

from __future__ import annotations

import importlib.util
import os
from functools import lru_cache
from pathlib import Path

import torch

from layout import LotLayout, TokenRect


def _lakonlab_common_path() -> Path:
    override = os.environ.get("LAKONLAB_ROOT")
    candidates: list[Path] = []
    if override:
        candidates.append(Path(override))
    repo = Path(__file__).resolve().parents[2]
    candidates.append(repo.parent / "LakonLab")
    for root in candidates:
        path = root / "lakonlab" / "models" / "architectures" / "asymflow" / "common.py"
        if path.is_file():
            return path
    searched = ", ".join(str(path) for path in candidates)
    raise FileNotFoundError(f"LakonLab asymflow/common.py not found. Looked in: {searched}")


@lru_cache(maxsize=1)
def _asymflow():
    path = _lakonlab_common_path()
    spec = importlib.util.spec_from_file_location("lakonlab_asymflow_common", path)
    if spec is None or spec.loader is None:
        raise ImportError(f"could not load {path}")
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)

    class _VelocityKernel(module.AsymFlowMixin):
        """One Procrustes basis, otherwise the LakonLab velocity step."""

        def __init__(self, basis: torch.Tensor):
            self.proj_buffer = basis
            self.sigma_min = 1e-6
            self.training = True

    return module.AsymFlowCalibration, _VelocityKernel, path


def mean_basis(token_dim: int, extent: tuple[int, int, int]) -> torch.Tensor:
    """Semi-orthonormal lift that averages the uniform tokens inside a patch.

    ``A`` has shape ``(et*eh*ew*D, D)`` and ``A^T A = I``. For a unit extent
    this is the identity, so a dense layout is unchanged. This is the
    stand-in used before a Procrustes fit; it is not the paper's data-fit basis.
    """
    et, eh, ew = extent
    sites = et * eh * ew
    if token_dim < 1 or sites < 1:
        raise ValueError("token dim and extent must be positive")
    eye = torch.eye(token_dim)
    return eye.repeat(sites, 1) / math_sqrt(sites)


def math_sqrt(value: int) -> float:
    return value ** 0.5


def fit_procrustes(dense: torch.Tensor, reference: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
    """Orthogonal Procrustes basis and RMS scale, eqs. 14–15.

    ``dense`` is ``(N, D_e)`` and ``reference`` is ``(N, D)``, the aligned
    multi-scale VAE tokens. The SVD matches
    ``LakonLab/tools/asymflow_subspace_procrustes.py``: ``A = V U^T`` from
    ``Z^T X = U S V^T``. ``A`` is ``(D_e, D)`` with ``A^T A = I``, and ``s``
    matches the RMS of ``A^T x`` to the RMS of ``z``.
    """
    if dense.ndim != 2 or reference.ndim != 2 or dense.shape[0] != reference.shape[0]:
        raise ValueError("dense and reference must be (N, D_e) and (N, D)")
    if reference.shape[1] > dense.shape[1]:
        raise ValueError("reference dimension exceeds the dense patch dimension")
    dense64 = dense.detach().double()
    reference64 = reference.detach().double()
    cross = reference64.T @ dense64
    pixel_gram = dense64.T @ dense64
    latent_norm_sq = (reference64 * reference64).sum().clamp(min=1e-12)
    u, _, vh = torch.linalg.svd(cross, full_matrices=False)
    rank = cross.shape[0]
    basis = (vh[:rank].T @ u[:, :rank].T).contiguous()
    projected = torch.trace(basis.T @ pixel_gram @ basis)
    scale = (projected / latent_norm_sq).clamp(min=1e-12).sqrt()
    return basis.float(), scale.float().reshape(())


def compress_patches(dense: torch.Tensor, basis: torch.Tensor) -> torch.Tensor:
    """Eq. 7. ``dense`` is ``(..., D_e)`` and the result is ``(..., D)``."""
    return dense @ basis.to(dtype=dense.dtype, device=dense.device)


def asymmetric_target(clean: torch.Tensor, noise: torch.Tensor, basis: torch.Tensor) -> torch.Tensor:
    """Eq. 8, row-vector form: ``P eps - x0`` with ``P = A A^T``."""
    projector = basis @ basis.T
    return noise @ projector.to(dtype=noise.dtype, device=noise.device) - clean


def recover_dense_velocity(
    u_a: torch.Tensor,
    x_t: torch.Tensor,
    basis: torch.Tensor,
    sigma: torch.Tensor | float,
) -> torch.Tensor:
    """Recover a full-rank patch velocity from an asymmetric prediction (eq. 9).

    ``u_a`` and ``x_t`` share shape ``(..., D_e)``. ``sigma`` is a scalar or a
    batch vector of length ``u_a.shape[0]``. The denominator is clamped at
    ``1e-6``, as in the paper.
    """
    if u_a.shape != x_t.shape:
        raise ValueError(f"u_a shape {tuple(u_a.shape)} != x_t shape {tuple(x_t.shape)}")
    if basis.ndim != 2 or basis.shape[0] != u_a.shape[-1]:
        raise ValueError("basis must be (D_e, D) for this patch")
    sigma_t = _broadcast_sigma(sigma, u_a)
    basis_f = basis.detach().to(device=u_a.device, dtype=torch.float32)
    if os.environ.get("LOT_NO_LAKONLAB") == "1":
        return asymflow_velocity_torch(u_a, x_t, basis_f, sigma_t).to(dtype=u_a.dtype)
    try:
        calibration_cls, kernel_cls, _path = _asymflow()
    except FileNotFoundError:
        # No LakonLab checkout (for example inside ComfyUI): the same s = k = 1 arithmetic.
        return asymflow_velocity_torch(u_a, x_t, basis_f, sigma_t).to(dtype=u_a.dtype)
    ones = torch.ones((), device=u_a.device, dtype=torch.float32)
    calibration = calibration_cls(s=ones, k=ones, timestep=ones, sigma=sigma_t)
    recovered = kernel_cls(basis_f).asymflow_velocity(u_a, x_t, calibration)
    return recovered.to(dtype=u_a.dtype)


def asymflow_velocity_torch(
    u_a: torch.Tensor,
    x_t: torch.Tensor,
    basis: torch.Tensor,
    sigma: torch.Tensor,
    sigma_min: float = 1e-6,
) -> torch.Tensor:
    """LakonLab ``asymflow_velocity`` with ``s = k = 1``, in plain torch (eq. 9).

    ``sub(v) = v A A^T`` and ``comp(v) = v - sub(v)``; the result is
    ``sub(u_a) + (comp(x_t) + comp(u_a)) / max(sigma, sigma_min)``. Matches the
    kernel's training-mode ``sigma_min``, which is what ``_VelocityKernel`` sets.
    """
    u = u_a.float()
    x = x_t.float()
    projector = basis @ basis.T
    u_sub = u @ projector
    x_sub = x @ projector
    sig = sigma.float().clamp(min=sigma_min)
    return u_sub + ((x - x_sub) + (u - u_sub)) / sig


def _broadcast_sigma(sigma: torch.Tensor | float, like: torch.Tensor) -> torch.Tensor:
    if not torch.is_tensor(sigma):
        sigma = torch.tensor(float(sigma), dtype=torch.float32)
    sigma = sigma.to(device=like.device, dtype=torch.float32)
    if sigma.ndim == 0:
        return sigma
    if sigma.ndim == 1:
        if sigma.shape[0] != like.shape[0]:
            raise ValueError(f"sigma batch {sigma.shape[0]} != {like.shape[0]}")
        return sigma.reshape(like.shape[0], *([1] * (like.ndim - 1)))
    if sigma.shape == like.shape[: sigma.ndim]:
        return sigma.reshape(*sigma.shape, *([1] * (like.ndim - sigma.ndim)))
    raise ValueError(f"cannot broadcast sigma shape {tuple(sigma.shape)} onto {tuple(like.shape)}")


def sample_noisy(
    y0: torch.Tensor,
    sigma: torch.Tensor | float,
    noise: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
    """``y_t = (1 - sigma) y0 + sigma eps`` on the scaled lattice."""
    if noise is None:
        noise = torch.randn_like(y0)
    sig = _broadcast_sigma(sigma, y0).to(dtype=y0.dtype)
    return (1 - sig) * y0 + sig * noise, noise


def clean_from_velocity(
    y_t: torch.Tensor,
    velocity: torch.Tensor,
    sigma: torch.Tensor | float,
) -> torch.Tensor:
    """``y0_hat = y_t - sigma u``."""
    sig = _broadcast_sigma(sigma, y_t).to(dtype=y_t.dtype)
    return y_t - sig * velocity


def lot_clean_loss(
    y0_hat: torch.Tensor,
    y0: torch.Tensor,
    sigma: torch.Tensor | float,
    sigma_floor: float = 0.05,
) -> torch.Tensor:
    """Weighted clean-data loss, eq. 17.

    The paper writes a squared Euclidean norm inside the expectation. This
    returns the mean over elements of that weighted square, which has the
    same minimizer on a fixed grid and does not grow with the lattice size.
    """
    sig = _broadcast_sigma(sigma, y0).clamp(min=sigma_floor)
    weight = 1.0 / sig.square()
    return ((y0_hat - y0).float().square() * weight).mean()



def lot_h3_clean_loss(
    out: torch.Tensor,
    y_t: torch.Tensor,
    y0: torch.Tensor,
    sigma: torch.Tensor | float,
    sigma_floor: float = 0.05,
) -> torch.Tensor:
    """Eq. 17 for H3's head, which predicts ``x0 - eps`` (H3's scheduler: ``x0 = x_t + sigma * v``).

    The clean estimate is ``y_t + sigma * out``, not ``clean_from_velocity`` (that
    is the ``eps - x0`` convention). For ``sigma >= sigma_floor`` this equals
    ``mean((out - (y0 - eps))^2)``: the ordinary H3 velocity MSE, taken in y-space.
    """
    sig = _broadcast_sigma(sigma, y0).to(dtype=torch.float32)
    y0_hat = y_t.float() + sig * out.float()
    weight = 1.0 / sig.clamp(min=sigma_floor).square()
    return ((y0_hat - y0.float()).square() * weight).mean()

def _one_extent(rects: list[TokenRect]) -> tuple[int, int, int]:
    if not rects:
        raise ValueError("expected at least one rectangle")
    extent = rects[0].extent
    for rect in rects[1:]:
        if rect.extent != extent:
            raise ValueError(f"rectangles mix extents {extent} and {rect.extent}")
    return extent


def _windows(rects: list[TokenRect], device: torch.device) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    """Broadcast indices ``(n, et, eh, ew)`` for every rectangle of one extent."""
    et, eh, ew = _one_extent(rects)
    t = torch.tensor([rect.t for rect in rects], dtype=torch.long, device=device)
    u = torch.tensor([rect.u for rect in rects], dtype=torch.long, device=device)
    v = torch.tensor([rect.v for rect in rects], dtype=torch.long, device=device)
    dt = torch.arange(et, device=device)
    du = torch.arange(eh, device=device)
    dv = torch.arange(ew, device=device)
    tt = t[:, None, None, None] + dt[None, :, None, None]
    uu = u[:, None, None, None] + du[None, None, :, None]
    vv = v[:, None, None, None] + dv[None, None, None, :]
    return tt, uu, vv


def apply_extent_scales(
    latent: torch.Tensor,
    layout: LotLayout,
    scales: dict[tuple[int, int, int], torch.Tensor],
    *,
    invert: bool = False,
) -> torch.Tensor:
    """Divide each patch by its extent scale. ``invert=True`` multiplies it back.

    One index write per extent. A unit scale hides a swapped multiply, so tests
    use a scale other than 1.
    """
    out = latent.clone()
    buckets: dict[tuple[int, int, int], list[TokenRect]] = {}
    for rect in layout.rects:
        buckets.setdefault(rect.extent, []).append(rect)
    for extent, rects in buckets.items():
        scale = scales[extent].to(device=latent.device, dtype=latent.dtype)
        tt, uu, vv = _windows(rects, latent.device)
        region = out[:, tt, uu, vv, :]
        region = region * scale if invert else region / scale
        out[:, tt, uu, vv, :] = region
    return out


def gather_extent(tokens: torch.Tensor, rects: list[TokenRect]) -> torch.Tensor:
    """Pack patches of one extent to ``(B, n, D_e)``.

    Site order is C-order over ``(et, eh, ew, D)``, with the token channel
    fastest. Procrustes rows must use this same order. One index read for the
    whole extent.
    """
    tt, uu, vv = _windows(rects, tokens.device)
    patches = tokens[:, tt, uu, vv, :]
    return patches.reshape(tokens.shape[0], len(rects), -1)


def scatter_extent(canvas: torch.Tensor, values: torch.Tensor, rects: list[TokenRect]) -> None:
    """Write ``(B, n, D_e)`` patches back onto a fresh ``(B, T, H, W, D)`` canvas."""
    tt, uu, vv = _windows(rects, canvas.device)
    et, eh, ew = _one_extent(rects)
    patches = values.reshape(canvas.shape[0], len(rects), et, eh, ew, canvas.shape[-1])
    canvas[:, tt, uu, vv, :] = patches


def euler_step(y_t: torch.Tensor, velocity: torch.Tensor, t: float, t_next: float) -> torch.Tensor:
    """One explicit step of ``dy/dt = u`` from ``t`` to ``t_next``."""
    return y_t + (float(t_next) - float(t)) * velocity