File size: 9,385 Bytes
6febba8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
"""Content-aware LoT layouts: semantic mask, bounding boxes, depth of field, texture.

The LoT paper's supplementary (layout-adaptive generation) derives per-frame
layouts from these four sources with one trained model: fine tokens on the
primary action region, coarser ones on secondary objects and background. Each
source here becomes a per-token *desired extent* map (the coarsest square a
token may join: 1, 2 or 4), and ``layout_from_desired_any`` turns that into a
quadtree layout per latent frame, so the layout follows the content over time.

Pixel inputs are ``(F, H, W)`` (or ``(H, W)`` for a single frame) in [0, 1] at
pixel resolution; they are area-averaged onto the token lattice (one H3 token =
32×32 pixels) and mapped from pixel frames onto H3's latent frames
(``(1, 4, 4, 4, 4)`` pixel frames per latent frame, repeating).
"""

from __future__ import annotations

import torch
import torch.nn.functional as F

from h3_positions import FRAME_PER_TOKEN
from layout import LotLayout, TokenRect, area_average, blur_radius, layout_from_desired

EXTENTS = (1, 2, 4)


def latent_frame_spans(latent_t: int) -> list[tuple[int, int]]:
    """``[start, stop)`` pixel frames covered by each H3 latent frame."""
    spans, start = [], 0
    for k in range(latent_t):
        n = FRAME_PER_TOKEN[k % 5]
        spans.append((start, start + n))
        start += n
    return spans


def to_latent_frames(field: torch.Tensor, latent_t: int, reduce: str = "max") -> torch.Tensor:
    """``(F, h, w)`` per pixel frame -> ``(latent_t, h, w)`` per latent frame.

    A single frame is broadcast. Any other frame count is stretched onto the
    expected pixel-frame count first, so a mask from a shorter or longer clip
    still lines up in time.
    """
    if field.ndim == 2:
        field = field[None]
    spans = latent_frame_spans(latent_t)
    total = spans[-1][1]
    if field.shape[0] == 1:
        return field.expand(latent_t, *field.shape[1:]).clone()
    if field.shape[0] != total:
        index = torch.linspace(0, field.shape[0] - 1, total).round().long()
        field = field[index]
    pool = (lambda x: x.amax(0)) if reduce == "max" else (lambda x: x.mean(0))
    return torch.stack([pool(field[a:b]) for a, b in spans])


def layout_from_desired_any(desired: torch.Tensor) -> LotLayout:
    """Quadtree (4 → 2 → 1) for any ``(T, H, W)`` grid.

    A 4×4 cell cut by the far edge keeps the clipped rectangle when its cells
    allow it (4×2 on the 2-token strip of H3's 24×42, 2×4 on a 2-row strip), so
    edges coarsen like the interior instead of falling back to 1×1. Every
    clipped extent is in H3's bank (FINE + MID + COARSE).
    """
    return layout_from_desired(desired.long().clamp(1, 4), root=4, edge_rects=True)


def desired_from_levels(fine: torch.Tensor, mid: torch.Tensor | None, background: int) -> torch.Tensor:
    """Boolean token maps -> desired extents: fine 1, mid 2, everything else ``background``."""
    if background not in EXTENTS:
        raise ValueError(f"background extent must be one of {EXTENTS}")
    desired = torch.full(fine.shape, background, dtype=torch.long)
    if mid is not None:
        desired[mid & (desired > 2)] = 2
    desired[fine] = 1
    return desired


def _tokens(pixel: torch.Tensor, latent_t: int, token_h: int, token_w: int, reduce: str = "max") -> torch.Tensor:
    pixel = pixel.float()
    if pixel.ndim == 2:
        pixel = pixel[None]
    return to_latent_frames(area_average(pixel, token_h, token_w), latent_t, reduce)


def _dilate(field: torch.Tensor, tokens: int) -> torch.Tensor:
    if tokens <= 0:
        return field
    k = 2 * tokens + 1
    return F.max_pool2d(field.float()[:, None], k, stride=1, padding=tokens)[:, 0] > 0.5


def mask_layout(mask: torch.Tensor, latent_t: int, token_h: int, token_w: int, *,
                secondary: torch.Tensor | None = None, background: int = 4,
                coverage: float = 0.05, dilate: int = 1) -> LotLayout:
    """Semantic mask (or rasterised boxes): 1×1 on the mask, 2×2 on ``secondary``, ``background`` elsewhere.

    A token counts as covered when ``coverage`` of its 32×32 pixels are inside
    the mask; ``dilate`` grows the fine region by that many tokens so edges stay sharp.
    """
    fine = _dilate(_tokens(mask, latent_t, token_h, token_w) > coverage, dilate)
    mid = None
    if secondary is not None:
        mid = _dilate(_tokens(secondary, latent_t, token_h, token_w) > coverage, dilate)
    return layout_from_desired_any(desired_from_levels(fine, mid, background))


def boxes_to_mask(boxes: list[tuple[float, float, float, float]], height: int, width: int,
                  frames: int = 1) -> torch.Tensor:
    """Normalised ``(x0, y0, x1, y1)`` boxes -> ``(frames, height, width)`` mask (same boxes every frame)."""
    mask = torch.zeros(frames, height, width)
    for x0, y0, x1, y1 in boxes:
        a, b = sorted((x0, x1))
        c, d = sorted((y0, y1))
        mask[:, int(c * height):max(int(d * height), int(c * height) + 1),
             int(a * width):max(int(b * width), int(a * width) + 1)] = 1.0
    return mask


# lot/layout.py layout_from_blur. A cell may join extent b only when its circle of
# confusion is at least this radius. Coarser extents need a larger defocus.
_BLUR_SUPPORT = ((8, 3.0), (4, 1.0), (2, 0.2))


def _desired_from_radius(radius: torch.Tensor, background: int) -> torch.Tensor:
    """Blur radius -> coarsest extent each cell may join. In-focus cells stay at 1.

    ``background`` is the coarsest square this caller can emit. H3's video heads stop at 4,
    so a radius that would be an 8×8 square stays at 4.
    """
    desired = torch.ones(radius.shape, dtype=torch.long)
    for extent, support in reversed(_BLUR_SUPPORT):
        if extent > background:
            continue
        desired = torch.where(radius >= support, torch.full_like(desired, extent), desired)
    return desired


def depth_layout(depth: torch.Tensor, latent_t: int, token_h: int, token_w: int, *,
                 focus: float = 0.75, band: float = 0.15, background: int = 4,
                 r_target: float = 4.0) -> LotLayout:
    """Derived depth-of-field layout. Fine tokens on the focal plane, coarse squares on the defocus.

    The radius is the circle of confusion (eqs. 21–22): ``|1/z - 1/z_focus|``, scaled so the
    95th percentile of each latent frame equals ``r_target``. A square is kept only when every
    cell inside it is blurred enough for that extent. ``depth`` is non-negative and on the same
    scale as ``focus``. A Depth Anything map (brighter = nearer, typically in [0, 1]) is legal
    when ``focus`` is read from that same map. ``band`` stays in the signature for existing
    callers; the radius replaces that fixed band.
    """
    if background not in EXTENTS:
        raise ValueError(f"background extent must be one of {EXTENTS}")
    if focus <= 0 or r_target < 0 or band <= 0:
        raise ValueError("focus must be positive, on the depth map's scale; r_target and band must be positive")
    tokens = _tokens(depth, latent_t, token_h, token_w, reduce="mean")
    if not torch.isfinite(tokens).all() or float(tokens.min()) < 0:
        raise ValueError("depth must be finite and non-negative")
    desired = torch.stack([_desired_from_radius(blur_radius(frame, float(focus), r_target), background)
                           for frame in tokens])
    return layout_from_desired_any(desired)


def texture_scores(image: torch.Tensor, token_h: int, token_w: int) -> torch.Tensor:
    """Per-token texture: std of luminance inside each token's pixels, ``(F, token_h, token_w)``.

    ``image`` is ``(F, H, W, C)`` or ``(F, H, W)`` in [0, 1].
    """
    if image.ndim == 4:
        image = (image[..., :3].float() * torch.tensor([0.299, 0.587, 0.114])).sum(-1)
    lum = image.float()
    mean = area_average(lum, token_h, token_w)
    sq = area_average(lum * lum, token_h, token_w)
    return (sq - mean * mean).clamp(min=0).sqrt()


def score_layout(score: torch.Tensor, *, fine_q: float = 0.7, mid_q: float = 0.4,
                 background: int = 4) -> LotLayout:
    """``(T, H, W)`` importance scores -> layout by per-frame quantiles: top ``1-fine_q`` 1×1, next band 2×2."""
    flat = score.flatten(1)
    hi = flat.quantile(fine_q, dim=1)[:, None, None]
    lo = flat.quantile(mid_q, dim=1)[:, None, None]
    # Strictly above the low quantile: perfectly flat tokens (ties at the bottom) may go coarse.
    mid = score > lo
    return layout_from_desired_any(desired_from_levels((score >= hi) & mid, mid, background))


def texture_layout(image: torch.Tensor, latent_t: int, token_h: int, token_w: int, *,
                   fine_q: float = 0.7, mid_q: float = 0.4, background: int = 4) -> LotLayout:
    """Texture variance (the paper's VRS source): fine where the image is busy, coarse where it is flat."""
    score = to_latent_frames(texture_scores(image, token_h, token_w), latent_t)
    return score_layout(score, fine_q=fine_q, mid_q=mid_q, background=background)


def compression(layout: LotLayout) -> float:
    return layout.dense_count / max(layout.count, 1)


__all__ = [
    "EXTENTS", "TokenRect", "boxes_to_mask", "compression", "depth_layout", "desired_from_levels",
    "latent_frame_spans", "layout_from_desired_any", "mask_layout", "score_layout", "texture_layout",
    "texture_scores", "to_latent_frames",
]