"""Constrained decoding: a per-sequence structure state machine + token sampling. The state machine guarantees every generated media block is well formed: inside only visual codes are allowed and exactly G of them; inside a video, frames are G codes with consecutive timestamps; text spans can only contain text tokens. Purely structural tokens are *forced* (no sampling); content tokens are sampled from the model's distribution restricted to the legal set. """ from __future__ import annotations from dataclasses import dataclass import torch from ..tokenizers.multimodal import MAX_FRAMES, UnifiedVocab @dataclass class Sampling: temperature: float = 1.0 top_k: int = 0 top_p: float = 1.0 repetition_penalty: float = 1.0 # applied to previously generated *text* tokens only max_new_tokens: int = 256 seed: int = 0 cfg_scale: float = 1.0 # classifier-free guidance on visual tokens (1.0 = off) class Masks: """Boolean legal-token sets, precomputed once per vocabulary/device.""" def __init__(self, v: UnifiedVocab, device): V = v.size def mk(ids=(), ranges=()): m = torch.zeros(V, dtype=torch.bool, device=device) for lo, hi in ranges: m[lo:hi] = True if ids: m[list(ids)] = True return m self.text = mk([v.eos], [(0, v.text_size)]) self.text_media = mk([v.eos, v.img_start, v.vid_start], [(0, v.text_size)]) self.visual = mk(ranges=[(v.visual_offset, v.visual_offset + v.visual_size)]) self.timestamp = mk(ranges=[(v.timestamp(0), v.timestamp(0) + MAX_FRAMES)]) self.frame_or_end = mk([v.frm_start, v.vid_end]) class StructureFSM: """Tracks where one sequence is in the media grammar. mode: "text" free text until (media tags disallowed unless allow_media) "image" we are right after : G codes, , then if stop_after_media "video" we are inside self.state = "eos" if self.stop_after_media else "text" elif s == "eos": self.done = True @property def in_visual(self) -> bool: return self.state in ("img_codes", "frm_codes") def sample_next(logits: torch.Tensor, legal: torch.Tensor, sp: Sampling, gen: torch.Generator, history: torch.Tensor | None = None, text_size: int = 0) -> torch.Tensor: """logits [B,V] (float32), legal [B,V] bool -> sampled ids [B].""" logits = logits.float().clone() if sp.repetition_penalty != 1.0 and history is not None and history.numel(): prev = history.clamp(min=0) is_text = prev < text_size vals = logits.gather(1, prev) pen = torch.where(vals > 0, vals / sp.repetition_penalty, vals * sp.repetition_penalty) logits.scatter_(1, prev, torch.where(is_text, pen, vals)) logits = logits.masked_fill(~legal, float("-inf")) if sp.temperature <= 0: return logits.argmax(-1) logits = logits / sp.temperature if sp.top_k > 0: kth = logits.topk(min(sp.top_k, logits.size(-1)), dim=-1).values[:, -1:] logits = logits.masked_fill(logits < kth, float("-inf")) if sp.top_p < 1.0: srt, idx = logits.sort(dim=-1, descending=True) probs = srt.softmax(-1) drop = probs.cumsum(-1) - probs > sp.top_p # keep the smallest prefix with mass >= top_p logits = logits.masked_fill(drop.scatter(1, idx, drop), float("-inf")) return torch.multinomial(logits.softmax(-1), 1, generator=gen).squeeze(1)