Spaces:
Running on Zero
Running on Zero
File size: 9,385 Bytes
6febba8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 | """Content-aware LoT layouts: semantic mask, bounding boxes, depth of field, texture.
The LoT paper's supplementary (layout-adaptive generation) derives per-frame
layouts from these four sources with one trained model: fine tokens on the
primary action region, coarser ones on secondary objects and background. Each
source here becomes a per-token *desired extent* map (the coarsest square a
token may join: 1, 2 or 4), and ``layout_from_desired_any`` turns that into a
quadtree layout per latent frame, so the layout follows the content over time.
Pixel inputs are ``(F, H, W)`` (or ``(H, W)`` for a single frame) in [0, 1] at
pixel resolution; they are area-averaged onto the token lattice (one H3 token =
32×32 pixels) and mapped from pixel frames onto H3's latent frames
(``(1, 4, 4, 4, 4)`` pixel frames per latent frame, repeating).
"""
from __future__ import annotations
import torch
import torch.nn.functional as F
from h3_positions import FRAME_PER_TOKEN
from layout import LotLayout, TokenRect, area_average, blur_radius, layout_from_desired
EXTENTS = (1, 2, 4)
def latent_frame_spans(latent_t: int) -> list[tuple[int, int]]:
"""``[start, stop)`` pixel frames covered by each H3 latent frame."""
spans, start = [], 0
for k in range(latent_t):
n = FRAME_PER_TOKEN[k % 5]
spans.append((start, start + n))
start += n
return spans
def to_latent_frames(field: torch.Tensor, latent_t: int, reduce: str = "max") -> torch.Tensor:
"""``(F, h, w)`` per pixel frame -> ``(latent_t, h, w)`` per latent frame.
A single frame is broadcast. Any other frame count is stretched onto the
expected pixel-frame count first, so a mask from a shorter or longer clip
still lines up in time.
"""
if field.ndim == 2:
field = field[None]
spans = latent_frame_spans(latent_t)
total = spans[-1][1]
if field.shape[0] == 1:
return field.expand(latent_t, *field.shape[1:]).clone()
if field.shape[0] != total:
index = torch.linspace(0, field.shape[0] - 1, total).round().long()
field = field[index]
pool = (lambda x: x.amax(0)) if reduce == "max" else (lambda x: x.mean(0))
return torch.stack([pool(field[a:b]) for a, b in spans])
def layout_from_desired_any(desired: torch.Tensor) -> LotLayout:
"""Quadtree (4 → 2 → 1) for any ``(T, H, W)`` grid.
A 4×4 cell cut by the far edge keeps the clipped rectangle when its cells
allow it (4×2 on the 2-token strip of H3's 24×42, 2×4 on a 2-row strip), so
edges coarsen like the interior instead of falling back to 1×1. Every
clipped extent is in H3's bank (FINE + MID + COARSE).
"""
return layout_from_desired(desired.long().clamp(1, 4), root=4, edge_rects=True)
def desired_from_levels(fine: torch.Tensor, mid: torch.Tensor | None, background: int) -> torch.Tensor:
"""Boolean token maps -> desired extents: fine 1, mid 2, everything else ``background``."""
if background not in EXTENTS:
raise ValueError(f"background extent must be one of {EXTENTS}")
desired = torch.full(fine.shape, background, dtype=torch.long)
if mid is not None:
desired[mid & (desired > 2)] = 2
desired[fine] = 1
return desired
def _tokens(pixel: torch.Tensor, latent_t: int, token_h: int, token_w: int, reduce: str = "max") -> torch.Tensor:
pixel = pixel.float()
if pixel.ndim == 2:
pixel = pixel[None]
return to_latent_frames(area_average(pixel, token_h, token_w), latent_t, reduce)
def _dilate(field: torch.Tensor, tokens: int) -> torch.Tensor:
if tokens <= 0:
return field
k = 2 * tokens + 1
return F.max_pool2d(field.float()[:, None], k, stride=1, padding=tokens)[:, 0] > 0.5
def mask_layout(mask: torch.Tensor, latent_t: int, token_h: int, token_w: int, *,
secondary: torch.Tensor | None = None, background: int = 4,
coverage: float = 0.05, dilate: int = 1) -> LotLayout:
"""Semantic mask (or rasterised boxes): 1×1 on the mask, 2×2 on ``secondary``, ``background`` elsewhere.
A token counts as covered when ``coverage`` of its 32×32 pixels are inside
the mask; ``dilate`` grows the fine region by that many tokens so edges stay sharp.
"""
fine = _dilate(_tokens(mask, latent_t, token_h, token_w) > coverage, dilate)
mid = None
if secondary is not None:
mid = _dilate(_tokens(secondary, latent_t, token_h, token_w) > coverage, dilate)
return layout_from_desired_any(desired_from_levels(fine, mid, background))
def boxes_to_mask(boxes: list[tuple[float, float, float, float]], height: int, width: int,
frames: int = 1) -> torch.Tensor:
"""Normalised ``(x0, y0, x1, y1)`` boxes -> ``(frames, height, width)`` mask (same boxes every frame)."""
mask = torch.zeros(frames, height, width)
for x0, y0, x1, y1 in boxes:
a, b = sorted((x0, x1))
c, d = sorted((y0, y1))
mask[:, int(c * height):max(int(d * height), int(c * height) + 1),
int(a * width):max(int(b * width), int(a * width) + 1)] = 1.0
return mask
# lot/layout.py layout_from_blur. A cell may join extent b only when its circle of
# confusion is at least this radius. Coarser extents need a larger defocus.
_BLUR_SUPPORT = ((8, 3.0), (4, 1.0), (2, 0.2))
def _desired_from_radius(radius: torch.Tensor, background: int) -> torch.Tensor:
"""Blur radius -> coarsest extent each cell may join. In-focus cells stay at 1.
``background`` is the coarsest square this caller can emit. H3's video heads stop at 4,
so a radius that would be an 8×8 square stays at 4.
"""
desired = torch.ones(radius.shape, dtype=torch.long)
for extent, support in reversed(_BLUR_SUPPORT):
if extent > background:
continue
desired = torch.where(radius >= support, torch.full_like(desired, extent), desired)
return desired
def depth_layout(depth: torch.Tensor, latent_t: int, token_h: int, token_w: int, *,
focus: float = 0.75, band: float = 0.15, background: int = 4,
r_target: float = 4.0) -> LotLayout:
"""Derived depth-of-field layout. Fine tokens on the focal plane, coarse squares on the defocus.
The radius is the circle of confusion (eqs. 21–22): ``|1/z - 1/z_focus|``, scaled so the
95th percentile of each latent frame equals ``r_target``. A square is kept only when every
cell inside it is blurred enough for that extent. ``depth`` is non-negative and on the same
scale as ``focus``. A Depth Anything map (brighter = nearer, typically in [0, 1]) is legal
when ``focus`` is read from that same map. ``band`` stays in the signature for existing
callers; the radius replaces that fixed band.
"""
if background not in EXTENTS:
raise ValueError(f"background extent must be one of {EXTENTS}")
if focus <= 0 or r_target < 0 or band <= 0:
raise ValueError("focus must be positive, on the depth map's scale; r_target and band must be positive")
tokens = _tokens(depth, latent_t, token_h, token_w, reduce="mean")
if not torch.isfinite(tokens).all() or float(tokens.min()) < 0:
raise ValueError("depth must be finite and non-negative")
desired = torch.stack([_desired_from_radius(blur_radius(frame, float(focus), r_target), background)
for frame in tokens])
return layout_from_desired_any(desired)
def texture_scores(image: torch.Tensor, token_h: int, token_w: int) -> torch.Tensor:
"""Per-token texture: std of luminance inside each token's pixels, ``(F, token_h, token_w)``.
``image`` is ``(F, H, W, C)`` or ``(F, H, W)`` in [0, 1].
"""
if image.ndim == 4:
image = (image[..., :3].float() * torch.tensor([0.299, 0.587, 0.114])).sum(-1)
lum = image.float()
mean = area_average(lum, token_h, token_w)
sq = area_average(lum * lum, token_h, token_w)
return (sq - mean * mean).clamp(min=0).sqrt()
def score_layout(score: torch.Tensor, *, fine_q: float = 0.7, mid_q: float = 0.4,
background: int = 4) -> LotLayout:
"""``(T, H, W)`` importance scores -> layout by per-frame quantiles: top ``1-fine_q`` 1×1, next band 2×2."""
flat = score.flatten(1)
hi = flat.quantile(fine_q, dim=1)[:, None, None]
lo = flat.quantile(mid_q, dim=1)[:, None, None]
# Strictly above the low quantile: perfectly flat tokens (ties at the bottom) may go coarse.
mid = score > lo
return layout_from_desired_any(desired_from_levels((score >= hi) & mid, mid, background))
def texture_layout(image: torch.Tensor, latent_t: int, token_h: int, token_w: int, *,
fine_q: float = 0.7, mid_q: float = 0.4, background: int = 4) -> LotLayout:
"""Texture variance (the paper's VRS source): fine where the image is busy, coarse where it is flat."""
score = to_latent_frames(texture_scores(image, token_h, token_w), latent_t)
return score_layout(score, fine_q=fine_q, mid_q=mid_q, background=background)
def compression(layout: LotLayout) -> float:
return layout.dense_count / max(layout.count, 1)
__all__ = [
"EXTENTS", "TokenRect", "boxes_to_mask", "compression", "depth_layout", "desired_from_levels",
"latent_frame_spans", "layout_from_desired_any", "mask_layout", "score_layout", "texture_layout",
"texture_scores", "to_latent_frames",
]
|