mpeg-diffusion / lot /layout_sources.py
multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
6febba8 verified
Raw History Blame Contribute Delete
9.39 kB
"""Content-aware LoT layouts: semantic mask, bounding boxes, depth of field, texture.
The LoT paper's supplementary (layout-adaptive generation) derives per-frame
layouts from these four sources with one trained model: fine tokens on the
primary action region, coarser ones on secondary objects and background. Each
source here becomes a per-token *desired extent* map (the coarsest square a
token may join: 1, 2 or 4), and ``layout_from_desired_any`` turns that into a
quadtree layout per latent frame, so the layout follows the content over time.
Pixel inputs are ``(F, H, W)`` (or ``(H, W)`` for a single frame) in [0, 1] at
pixel resolution; they are area-averaged onto the token lattice (one H3 token =
32×32 pixels) and mapped from pixel frames onto H3's latent frames
(``(1, 4, 4, 4, 4)`` pixel frames per latent frame, repeating).
"""
from __future__ import annotations
import torch
import torch.nn.functional as F
from h3_positions import FRAME_PER_TOKEN
from layout import LotLayout, TokenRect, area_average, blur_radius, layout_from_desired
EXTENTS = (1, 2, 4)
def latent_frame_spans(latent_t: int) -> list[tuple[int, int]]:
"""``[start, stop)`` pixel frames covered by each H3 latent frame."""
spans, start = [], 0
for k in range(latent_t):
n = FRAME_PER_TOKEN[k % 5]
spans.append((start, start + n))
start += n
return spans
def to_latent_frames(field: torch.Tensor, latent_t: int, reduce: str = "max") -> torch.Tensor:
"""``(F, h, w)`` per pixel frame -> ``(latent_t, h, w)`` per latent frame.
A single frame is broadcast. Any other frame count is stretched onto the
expected pixel-frame count first, so a mask from a shorter or longer clip
still lines up in time.
"""
if field.ndim == 2:
field = field[None]
spans = latent_frame_spans(latent_t)
total = spans[-1][1]
if field.shape[0] == 1:
return field.expand(latent_t, *field.shape[1:]).clone()
if field.shape[0] != total:
index = torch.linspace(0, field.shape[0] - 1, total).round().long()
field = field[index]
pool = (lambda x: x.amax(0)) if reduce == "max" else (lambda x: x.mean(0))
return torch.stack([pool(field[a:b]) for a, b in spans])
def layout_from_desired_any(desired: torch.Tensor) -> LotLayout:
"""Quadtree (4 → 2 → 1) for any ``(T, H, W)`` grid.
A 4×4 cell cut by the far edge keeps the clipped rectangle when its cells
allow it (4×2 on the 2-token strip of H3's 24×42, 2×4 on a 2-row strip), so
edges coarsen like the interior instead of falling back to 1×1. Every
clipped extent is in H3's bank (FINE + MID + COARSE).
"""
return layout_from_desired(desired.long().clamp(1, 4), root=4, edge_rects=True)
def desired_from_levels(fine: torch.Tensor, mid: torch.Tensor | None, background: int) -> torch.Tensor:
"""Boolean token maps -> desired extents: fine 1, mid 2, everything else ``background``."""
if background not in EXTENTS:
raise ValueError(f"background extent must be one of {EXTENTS}")
desired = torch.full(fine.shape, background, dtype=torch.long)
if mid is not None:
desired[mid & (desired > 2)] = 2
desired[fine] = 1
return desired
def _tokens(pixel: torch.Tensor, latent_t: int, token_h: int, token_w: int, reduce: str = "max") -> torch.Tensor:
pixel = pixel.float()
if pixel.ndim == 2:
pixel = pixel[None]
return to_latent_frames(area_average(pixel, token_h, token_w), latent_t, reduce)
def _dilate(field: torch.Tensor, tokens: int) -> torch.Tensor:
if tokens <= 0:
return field
k = 2 * tokens + 1
return F.max_pool2d(field.float()[:, None], k, stride=1, padding=tokens)[:, 0] > 0.5
def mask_layout(mask: torch.Tensor, latent_t: int, token_h: int, token_w: int, *,
secondary: torch.Tensor | None = None, background: int = 4,
coverage: float = 0.05, dilate: int = 1) -> LotLayout:
"""Semantic mask (or rasterised boxes): 1×1 on the mask, 2×2 on ``secondary``, ``background`` elsewhere.
A token counts as covered when ``coverage`` of its 32×32 pixels are inside
the mask; ``dilate`` grows the fine region by that many tokens so edges stay sharp.
"""
fine = _dilate(_tokens(mask, latent_t, token_h, token_w) > coverage, dilate)
mid = None
if secondary is not None:
mid = _dilate(_tokens(secondary, latent_t, token_h, token_w) > coverage, dilate)
return layout_from_desired_any(desired_from_levels(fine, mid, background))
def boxes_to_mask(boxes: list[tuple[float, float, float, float]], height: int, width: int,
frames: int = 1) -> torch.Tensor:
"""Normalised ``(x0, y0, x1, y1)`` boxes -> ``(frames, height, width)`` mask (same boxes every frame)."""
mask = torch.zeros(frames, height, width)
for x0, y0, x1, y1 in boxes:
a, b = sorted((x0, x1))
c, d = sorted((y0, y1))
mask[:, int(c * height):max(int(d * height), int(c * height) + 1),
int(a * width):max(int(b * width), int(a * width) + 1)] = 1.0
return mask
# lot/layout.py layout_from_blur. A cell may join extent b only when its circle of
# confusion is at least this radius. Coarser extents need a larger defocus.
_BLUR_SUPPORT = ((8, 3.0), (4, 1.0), (2, 0.2))
def _desired_from_radius(radius: torch.Tensor, background: int) -> torch.Tensor:
"""Blur radius -> coarsest extent each cell may join. In-focus cells stay at 1.
``background`` is the coarsest square this caller can emit. H3's video heads stop at 4,
so a radius that would be an 8×8 square stays at 4.
"""
desired = torch.ones(radius.shape, dtype=torch.long)
for extent, support in reversed(_BLUR_SUPPORT):
if extent > background:
continue
desired = torch.where(radius >= support, torch.full_like(desired, extent), desired)
return desired
def depth_layout(depth: torch.Tensor, latent_t: int, token_h: int, token_w: int, *,
focus: float = 0.75, band: float = 0.15, background: int = 4,
r_target: float = 4.0) -> LotLayout:
"""Derived depth-of-field layout. Fine tokens on the focal plane, coarse squares on the defocus.
The radius is the circle of confusion (eqs. 21–22): ``|1/z - 1/z_focus|``, scaled so the
95th percentile of each latent frame equals ``r_target``. A square is kept only when every
cell inside it is blurred enough for that extent. ``depth`` is non-negative and on the same
scale as ``focus``. A Depth Anything map (brighter = nearer, typically in [0, 1]) is legal
when ``focus`` is read from that same map. ``band`` stays in the signature for existing
callers; the radius replaces that fixed band.
"""
if background not in EXTENTS:
raise ValueError(f"background extent must be one of {EXTENTS}")
if focus <= 0 or r_target < 0 or band <= 0:
raise ValueError("focus must be positive, on the depth map's scale; r_target and band must be positive")
tokens = _tokens(depth, latent_t, token_h, token_w, reduce="mean")
if not torch.isfinite(tokens).all() or float(tokens.min()) < 0:
raise ValueError("depth must be finite and non-negative")
desired = torch.stack([_desired_from_radius(blur_radius(frame, float(focus), r_target), background)
for frame in tokens])
return layout_from_desired_any(desired)
def texture_scores(image: torch.Tensor, token_h: int, token_w: int) -> torch.Tensor:
"""Per-token texture: std of luminance inside each token's pixels, ``(F, token_h, token_w)``.
``image`` is ``(F, H, W, C)`` or ``(F, H, W)`` in [0, 1].
"""
if image.ndim == 4:
image = (image[..., :3].float() * torch.tensor([0.299, 0.587, 0.114])).sum(-1)
lum = image.float()
mean = area_average(lum, token_h, token_w)
sq = area_average(lum * lum, token_h, token_w)
return (sq - mean * mean).clamp(min=0).sqrt()
def score_layout(score: torch.Tensor, *, fine_q: float = 0.7, mid_q: float = 0.4,
background: int = 4) -> LotLayout:
"""``(T, H, W)`` importance scores -> layout by per-frame quantiles: top ``1-fine_q`` 1×1, next band 2×2."""
flat = score.flatten(1)
hi = flat.quantile(fine_q, dim=1)[:, None, None]
lo = flat.quantile(mid_q, dim=1)[:, None, None]
# Strictly above the low quantile: perfectly flat tokens (ties at the bottom) may go coarse.
mid = score > lo
return layout_from_desired_any(desired_from_levels((score >= hi) & mid, mid, background))
def texture_layout(image: torch.Tensor, latent_t: int, token_h: int, token_w: int, *,
fine_q: float = 0.7, mid_q: float = 0.4, background: int = 4) -> LotLayout:
"""Texture variance (the paper's VRS source): fine where the image is busy, coarse where it is flat."""
score = to_latent_frames(texture_scores(image, token_h, token_w), latent_t)
return score_layout(score, fine_q=fine_q, mid_q=mid_q, background=background)
def compression(layout: LotLayout) -> float:
return layout.dense_count / max(layout.count, 1)
__all__ = [
"EXTENTS", "TokenRect", "boxes_to_mask", "compression", "depth_layout", "desired_from_levels",
"latent_frame_spans", "layout_from_desired_any", "mask_layout", "score_layout", "texture_layout",
"texture_scores", "to_latent_frames",
]