TTS-2-Studio / models /cudagraph /depth_decoder_graph.py
owensong's picture
Launch complete Breeze TTS 2 ZeroGPU studio
ecfc07b verified
Raw History Blame Contribute Delete
40.2 kB
"""
CUDA graph capture for the Breeze depth decoder's decode loop,
using transformers StaticCache.
The depth decoder generates num_codebooks-1 codebook tokens autoregressively
(typically 15 for production models with 16 RVQ codebooks):
- Step 0: prefill with 2 tokens (backbone_hidden placeholder + first_cb_token),
get logits for codebook[1]
- Steps 1..num_codebooks-2: decode 1 token at a time using previous codebook token's embedding
Strategy:
- Use transformers StaticCache for KV cache management
- Use the depth decoder's inner model forward
- Unroll the full loop for deterministic shapes
- Capture the entire loop as a single CUDA graph
- CFG is baked in: batch=2 (cond + uncond), guidance applied per step
- All tensors pre-allocated β€” no torch.tensor() calls inside the graph
- Sampling params (temperature, top_k, top_p, do_sample, guidance_scale) are
per-sample tensor buffers [half, 1] or [half]: update via set_*() methods
at runtime without recapture. Accepts both scalar (broadcast) and tensor
(per-sample) values.
- top_k uses a fixed MAX_K workspace; top_k_buf can be set to any value <= _max_k.
- Multi-batch bucket support: pre-capture graphs for multiple batch sizes
(powers of 2), select the smallest fitting bucket at runtime. No recapture
needed when batch size changes within the pre-captured range.
"""
import logging
from dataclasses import dataclass
import torch
import torch._dynamo
import torch.nn.functional as F
from transformers import StaticCache
from transformers.masking_utils import create_causal_mask
from ..logits_process import mask_invalid_codec_token_logits
_log = logging.getLogger(__name__)
@dataclass
class _BucketState:
"""Stores all per-bucket-size state needed for CUDA graph replay."""
batch_size: int
half: int
graph: torch.cuda.CUDAGraph
static_cache: StaticCache
# I/O buffers
backbone_hidden_buf: torch.Tensor
first_cb_token_buf: torch.Tensor
output_tokens: torch.Tensor
_tok_buf: torch.Tensor
prefill_input_ids: torch.Tensor
# Attention masks
prefill_attn: torch.Tensor
decode_attn: list
# Sampling param buffers
temperature_buf: torch.Tensor
top_k_buf: torch.Tensor
top_p_buf: torch.Tensor
do_sample_buf: torch.Tensor
guidance_scale: torch.Tensor
# Debug buffers (optional)
debug_logits: torch.Tensor | None = None
_debug_head_input: torch.Tensor | None = None
_debug_probs: torch.Tensor | None = None
_debug_probs_slot: torch.Tensor | None = None
class DepthDecoderGraph:
"""
Captures the full depth decoder loop as a CUDA graph,
using the model's forward with transformers StaticCache.
For CFG: batch=2 (cond=batch[0], uncond=batch[1]).
Sampling params are per-sample tensor buffers β€” no recapture needed:
# Scalar (broadcast to all samples):
graph.set_temperature(0.5)
graph.set_guidance_scale(3.0)
# Per-sample (tensor):
graph.set_temperature(torch.tensor([0.5, 1.2]))
graph.set_guidance_scale(torch.tensor([3.0, 1.0]))
# Or pass directly in run():
graph.run(backbone_h, first_cb, temperature=torch.tensor([0.5, 1.2]))
Batch size changes trigger automatic recapture:
graph.ensure_batch_size(4)
"""
def __init__(
self,
depth_decoder,
config,
device="cuda:0",
dtype=torch.bfloat16,
do_sample=True,
top_k=50,
top_p=1.0,
temperature=0.9,
guidance_scale=3.0,
num_codebooks=None,
codec_codebook_size=None,
fast: bool = False,
debug: bool = False,
batch_size=2,
bucket_sizes: list[int] | None = None,
):
self.device = device
self.dtype = dtype
self.config = config
self.debug = debug
self.no_graph = (
False # runtime flag: True = skip graph replay (for layer-diff hooks)
)
self.num_layers = config.num_hidden_layers
self.hidden_size = config.hidden_size
self.backbone_hidden_size = config.backbone_hidden_size
self.vocab_size = config.vocab_size
self.codec_codebook_size = (
int(codec_codebook_size)
if codec_codebook_size is not None
else self.vocab_size
)
if not 0 < self.codec_codebook_size <= self.vocab_size:
raise ValueError(
f"codec_codebook_size must be in [1, {self.vocab_size}], "
f"got {self.codec_codebook_size}"
)
self.num_codebooks = (
num_codebooks if num_codebooks is not None else config.num_codebooks
)
self.num_decode_codebooks = self.num_codebooks - 1
self.max_seq = 2 + self.num_decode_codebooks
self.batch_size = batch_size
self.half = self._real_batch_size(self.batch_size)
# Multi-batch bucket support
self.bucket_sizes = sorted(bucket_sizes) if bucket_sizes else [batch_size]
self._bucket_graphs: dict[int, _BucketState] = {}
# Extract model components (references, not copies)
self.depth_model = depth_decoder.model # BreezeDepthDecoderModel
self.codebooks_head = depth_decoder.codebooks_head # BreezeCodebooksHead
self._orig_codebooks_head = (
depth_decoder.codebooks_head
) # unwrapped ref for fake forward hooks
self.embed_tokens = self.depth_model.embed_tokens
self.inputs_embeds_projector = self.depth_model.inputs_embeds_projector
self.backbone_hidden_state_projector = getattr(
self.depth_model, "backbone_hidden_state_projector", None
)
self.audio_embed_size = self.depth_model.audio_embed_size
# Cast codebooks_head weight to fp32 for numerical stability
# (module forward still fires hooks, and F.linear works with fp32 input)
self.codebooks_head.weight.data = self.codebooks_head.weight.data.float()
self.fast = bool(fast)
# The fast path uses one maintained compile configuration before
# manual CUDA Graph capture. Compile modes are intentionally not public.
if self.fast:
compile_mode = "default"
_limit = self.num_layers * 4 + 16
torch._dynamo.config.cache_size_limit = max(
torch._dynamo.config.cache_size_limit, _limit
)
torch._dynamo.config.recompile_limit = max(
torch._dynamo.config.recompile_limit, _limit
)
already_compiled = hasattr(self.depth_model.layers[0], "_orig_mod")
if already_compiled:
_log.info("Depth decoder layers already compiled, skipping.")
else:
_log.info("Compiling depth decoder with the fast configuration.")
for index, layer in enumerate(self.depth_model.layers):
self.depth_model.layers[index] = torch.compile(
layer, mode=compile_mode, fullgraph=True
)
self.depth_model.norm = torch.compile(
self.depth_model.norm, mode=compile_mode, fullgraph=True
)
if not hasattr(self.codebooks_head, "_orig_mod"):
self.codebooks_head = torch.compile(
self.codebooks_head, mode=compile_mode, fullgraph=True
)
# Sampling param defaults β€” saved for _alloc_buffers / _rebuild_for_batch
self._default_temperature = temperature
self._default_top_k = top_k
self._default_top_p = top_p
self._default_do_sample = do_sample
self._default_guidance_scale = guidance_scale
# Fixed MAX_K workspace for graph-safe top_k.
# top_k_buf can be set to any value in [1, _max_k] at runtime without recapture.
# Default 1024 so users can freely adjust top_k up to 1024.
self._max_k = min(1024, self.vocab_size)
self._topk_ranks = torch.arange(
self._max_k, device=device
) # pre-alloc, never recreated
# top_p: always keep the highest-prob token (graph-safe alternative to remove[0]=False)
self._keep_first = torch.zeros(self.vocab_size, dtype=torch.bool, device=device)
self._keep_first[0] = True
# Pre-allocate ALL cache_position tensors (no torch.tensor() inside graph)
self.prefill_cache_pos = torch.arange(2, device=device)
self.decode_cache_positions = [
torch.tensor([2 + i], device=device)
for i in range(self.num_decode_codebooks - 1)
]
self.head_prefill_pos = torch.tensor([1], device=device)
self.codebook_offsets = [
torch.tensor([i * self.vocab_size], dtype=torch.long, device=device)
for i in range(self.num_decode_codebooks)
]
# Transformers StaticCache
self.static_cache = StaticCache(
config=config, max_cache_len=self.max_seq, batch_size=self.batch_size
)
self._alloc_buffers()
self.graph = None
self.captured = False
self.prefill_attn = None
self.decode_attn = None
@staticmethod
def _real_batch_size(batch_size: int) -> int:
if batch_size <= 1:
return batch_size
return batch_size // 2
# ------------------------------------------------------------------
# Runtime setters β€” no recapture needed
# ------------------------------------------------------------------
def set_temperature(self, v):
"""v: scalar float (broadcast) or [half] / [half,1] tensor (per-sample)."""
if isinstance(v, (int, float)):
self.temperature_buf.fill_(v)
else:
self.temperature_buf.copy_(v.view(self.half, 1))
def set_top_k(self, v):
"""v: scalar int (broadcast) or [half] / [half,1] tensor (per-sample)."""
if isinstance(v, (int, float)):
assert v <= self._max_k, (
f"top_k={v} exceeds _max_k={self._max_k}; rebuild graph to increase MAX_K"
)
self.top_k_buf.fill_(v)
else:
assert v.max().item() <= self._max_k, (
f"top_k max={v.max().item()} exceeds _max_k={self._max_k}"
)
self.top_k_buf.copy_(v.view(self.half, 1))
def set_top_p(self, v):
"""v: scalar float (broadcast) or [half] / [half,1] tensor (per-sample)."""
if isinstance(v, (int, float)):
self.top_p_buf.fill_(v)
else:
self.top_p_buf.copy_(v.view(self.half, 1))
def set_do_sample(self, v):
"""v: scalar bool (broadcast) or [half] long tensor (per-sample, 0/1)."""
if isinstance(v, (bool, int, float)):
self.do_sample_buf.fill_(1 if v else 0)
else:
self.do_sample_buf.copy_(v.view(self.half))
def set_guidance_scale(self, v):
"""v: scalar float (broadcast) or [half] / [half,1] tensor (per-sample)."""
if isinstance(v, (int, float)):
self.guidance_scale.fill_(v)
else:
self.guidance_scale.copy_(v.view(self.half, 1))
# ------------------------------------------------------------------
# Buffer allocation (called from __init__ and _rebuild_for_batch)
# ------------------------------------------------------------------
def _alloc_buffers(self):
"""Allocate / re-allocate I/O buffers for current batch_size.
Also (re-)creates per-sample sampling param buffers whose first
dimension is ``self.half``. On first call the ``_default_*``
values are used; on subsequent calls (batch-size change) the
current first-sample value is preserved as the uniform default.
"""
self.backbone_hidden_buf = torch.zeros(
self.batch_size,
self.backbone_hidden_size,
dtype=self.dtype,
device=self.device,
)
self.first_cb_token_buf = torch.zeros(
self.batch_size, dtype=torch.long, device=self.device
)
self.output_tokens = torch.zeros(
self.batch_size,
self.num_decode_codebooks,
dtype=torch.long,
device=self.device,
)
self._tok_buf = torch.zeros(
self.batch_size, dtype=torch.long, device=self.device
)
self.prefill_input_ids = torch.zeros(
self.batch_size, 2, dtype=torch.long, device=self.device
)
# --- Per-sample sampling param buffers [half, 1] or [half] ---
# Preserve current value when resizing, fall back to defaults on first init.
_temp = (
float(self.temperature_buf[0, 0])
if hasattr(self, "temperature_buf") and self.temperature_buf.numel()
else self._default_temperature
)
_top_k = (
int(self.top_k_buf[0, 0])
if hasattr(self, "top_k_buf") and self.top_k_buf.numel()
else self._default_top_k
)
_top_p = (
float(self.top_p_buf[0, 0])
if hasattr(self, "top_p_buf") and self.top_p_buf.numel()
else self._default_top_p
)
_do_sample = (
int(self.do_sample_buf[0])
if hasattr(self, "do_sample_buf") and self.do_sample_buf.numel()
else (1 if self._default_do_sample else 0)
)
_gs = (
float(self.guidance_scale[0, 0])
if hasattr(self, "guidance_scale") and self.guidance_scale.numel()
else self._default_guidance_scale
)
self.temperature_buf = torch.full(
(self.half, 1), _temp, dtype=torch.float32, device=self.device
)
self.top_k_buf = torch.full(
(self.half, 1), _top_k, dtype=torch.long, device=self.device
)
self.top_p_buf = torch.full(
(self.half, 1), _top_p, dtype=torch.float32, device=self.device
)
self.do_sample_buf = torch.full(
(self.half,), _do_sample, dtype=torch.long, device=self.device
)
self.guidance_scale = torch.full(
(self.half, 1), _gs, dtype=torch.float32, device=self.device
)
# Debug: per-codebook raw logits [num_decode_codebooks, batch_size, vocab_size]
if self.debug:
self.debug_logits = torch.zeros(
self.num_decode_codebooks,
self.batch_size,
self.vocab_size,
dtype=torch.float32,
device=self.device,
)
# Debug: hidden states before each codebooks_head call (for fake forward hooks)
self._debug_head_input = torch.zeros(
self.num_decode_codebooks,
self.batch_size,
1,
self.hidden_size,
dtype=torch.float32,
device=self.device,
)
# Debug: probs before torch.multinomial (for fake multinomial hooks)
self._debug_probs = torch.zeros(
self.num_decode_codebooks,
self.half,
self.vocab_size,
dtype=torch.float32,
device=self.device,
)
# Single-slot buffer written by _cfg_sample, copied per-cb in _full_loop
self._debug_probs_slot = torch.zeros(
self.half,
self.vocab_size,
dtype=torch.float32,
device=self.device,
)
else:
self.debug_logits = None
self._debug_head_input = None
self._debug_probs = None
self._debug_probs_slot = None
# ------------------------------------------------------------------
# Batch size management β€” auto-recapture on change
# ------------------------------------------------------------------
@torch.inference_mode()
def ensure_batch_size(self, batch_size: int):
"""No-op if batch_size matches current captured graph; else rebuild and recapture."""
if batch_size == self.batch_size:
return
self._rebuild_for_batch(batch_size)
@torch.inference_mode()
def _rebuild_for_batch(self, batch_size: int):
self.batch_size = batch_size
self.half = self._real_batch_size(self.batch_size)
self.static_cache = StaticCache(
config=self.config, max_cache_len=self.max_seq, batch_size=batch_size
)
self._alloc_buffers()
self.captured = False
self.graph = None
self.prefill_attn = None
self.decode_attn = None
if self.no_graph:
self.prepare_eager()
else:
self.capture()
@torch.inference_mode()
def prepare_eager(self):
"""Initialize the depth loop for native eager execution only."""
self.no_graph = True
self._init_cache_layers()
self._build_attention_masks()
self.captured = False
return self
# ------------------------------------------------------------------
# Internal helpers
# ------------------------------------------------------------------
def _init_cache_layers(self):
"""Force lazy initialization of StaticCache layers before graph capture."""
config = self.depth_model.config
num_kv_heads = getattr(
config, "num_key_value_heads", config.num_attention_heads
)
head_dim = getattr(
config, "head_dim", config.hidden_size // config.num_attention_heads
)
dummy_k = torch.zeros(
self.batch_size,
num_kv_heads,
1,
head_dim,
dtype=self.dtype,
device=self.device,
)
for layer in self.static_cache.layers:
if not layer.is_initialized:
layer.lazy_initialization(dummy_k)
def _make_attn_mask(self, input_embeds, cache_position):
return create_causal_mask(
config=self.depth_model.config,
input_embeds=input_embeds,
attention_mask=None,
cache_position=cache_position,
past_key_values=self.static_cache,
)
def _build_attention_masks(self):
dummy_prefill = torch.zeros(
self.batch_size, 2, self.hidden_size, dtype=self.dtype, device=self.device
)
dummy_decode = torch.zeros(
self.batch_size, 1, self.hidden_size, dtype=self.dtype, device=self.device
)
self.prefill_attn = self._make_attn_mask(dummy_prefill, self.prefill_cache_pos)
self.decode_attn = []
for pos in self.decode_cache_positions:
self.decode_attn.append(self._make_attn_mask(dummy_decode, pos))
def _cfg_sample(self, logits):
"""Graph-safe CFG + sampling matching HF transformers order.
HF order: temperature -> top_k -> top_p -> softmax -> multinomial.
All filtering on raw logits (not log-space) for parity with
TemperatureLogitsWarper / TopKLogitsWarper / TopPLogitsWarper.
Writes sampled tokens into self._tok_buf [batch_size].
For batch_size=2*N: processes N samples, writes to both halves (cond+uncond).
"""
# logits: [batch_size, 1, vocab]
if self.batch_size >= 2:
cond_logits = logits[: self.half, 0, :]
uncond_logits = logits[self.half :, 0, :]
cfg = uncond_logits + self.guidance_scale * (cond_logits - uncond_logits)
else:
cfg = logits[: self.half, 0, :]
mask_invalid_codec_token_logits(
cfg,
codebook_size=self.codec_codebook_size,
token_vocab_size=self.vocab_size,
)
# temperature scaling (on raw logits, same as TemperatureLogitsWarper)
scaled = cfg / self.temperature_buf # [half, vocab]
# top_k on raw logits (graph-safe: fixed _max_k workspace)
effective_k = torch.where(
self.top_k_buf > 0,
self.top_k_buf,
torch.full_like(self.top_k_buf, self._max_k),
)
topk_vals, topk_idx = torch.topk(scaled, self._max_k) # [half, _max_k]
topk_mask = self._topk_ranks < effective_k # [_max_k] broadcasts
topk_vals = torch.where(
topk_mask, topk_vals, torch.full_like(topk_vals, float("-inf"))
)
scaled = torch.full_like(scaled, float("-inf")).scatter_(1, topk_idx, topk_vals)
# top_p on raw logits (HF-style: softmax -> cumsum -> mask -> apply to raw logits)
sorted_logits, sorted_idx = torch.sort(scaled, descending=True) # [half, vocab]
cumprobs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
remove = cumprobs > self.top_p_buf
remove = F.pad(
remove[..., :-1], (1, 0), value=False
) # shift right, first always kept
sorted_logits = torch.where(
remove, torch.full_like(sorted_logits, float("-inf")), sorted_logits
)
scaled = torch.full_like(scaled, float("-inf")).scatter_(
1, sorted_idx, sorted_logits
)
# softmax -> multinomial (same as HF _sample)
probs = F.softmax(scaled, dim=-1)
# Debug: save probs before multinomial for fake forward hooks
if self._debug_probs_slot is not None:
self._debug_probs_slot.copy_(probs)
# do_sample: compute both paths, select via torch.where (no Python branch in graph)
greedy_toks = torch.argmax(cfg, dim=-1) # [half]
sampled_toks = torch.multinomial(probs, 1).squeeze(-1) # [half]
toks = torch.where(
self.do_sample_buf.bool(), sampled_toks, greedy_toks
) # [half] per-sample
# Write to the active slots; duplicate for paired cond/uncond mode.
self._tok_buf[: self.half] = toks
if self.batch_size >= 2:
self._tok_buf[self.half :] = toks
def _full_loop(self):
"""The full depth decoder loop on static buffers. Graph-safe."""
# Build [B, 2] input_ids: pos0=0 (placeholder), pos1=first_cb_token
self.prefill_input_ids[:, 0] = 0
self.prefill_input_ids[:, 1] = self.first_cb_token_buf
# Embed full 2-token sequence, then overwrite pos0 with backbone hidden
# (matches parity baseline: embed placeholder, then replace with backbone_h)
prefill_embeds = self.embed_tokens(
self.prefill_input_ids
) # [B, 2, audio_embed_size]
# Project backbone hidden to audio_embed_size if dimensions differ
backbone_h = self.backbone_hidden_buf # [B, backbone_hidden_size]
if self.backbone_hidden_state_projector is not None:
backbone_h = self.backbone_hidden_state_projector(
backbone_h
) # [B, audio_embed_size]
prefill_embeds[:, 0] = backbone_h # overwrite pos0
prefill_embeds = self.inputs_embeds_projector(prefill_embeds) # [B, 2, hidden]
hidden_states = prefill_embeds
position_ids = self.prefill_cache_pos.unsqueeze(0) # [1, 2]
position_embeddings = self.depth_model.rotary_emb(hidden_states, position_ids)
for decoder_layer in self.depth_model.layers:
hidden_states = decoder_layer(
hidden_states,
attention_mask=self.prefill_attn,
position_ids=position_ids,
past_key_values=self.static_cache,
use_cache=True,
cache_position=self.prefill_cache_pos,
position_embeddings=position_embeddings,
)
hidden_states = self.depth_model.norm(hidden_states)
# First codebook logits from position 1 (fp32: weight already cast, input cast here)
if self._debug_head_input is not None:
self._debug_head_input[0].copy_(hidden_states[:, 1:, :].float())
first_logits = self.codebooks_head(
hidden_states[:, 1:, :].float(),
cache_position=self.head_prefill_pos,
) # [B, 1, vocab] fp32
if self.debug_logits is not None:
self.debug_logits[0].copy_(first_logits[:, 0, :])
self._cfg_sample(first_logits)
if self._debug_probs is not None:
self._debug_probs[0].copy_(self._debug_probs_slot)
self._tok_buf.clamp_(0, self.vocab_size - 1)
self.output_tokens[:, 0] = self._tok_buf
# Remaining codebooks: decode one at a time
for cb_idx in range(1, self.num_decode_codebooks):
offset_tok = self._tok_buf + self.codebook_offsets[cb_idx]
emb = self.embed_tokens(
offset_tok.unsqueeze(1).clamp_(
0, self.num_codebooks * self.vocab_size - 1
)
) # [B, 1, backbone_H]
emb = self.inputs_embeds_projector(emb) # [B, 1, hidden]
cache_pos = self.decode_cache_positions[cb_idx - 1]
pos_ids = cache_pos.unsqueeze(0)
pos_emb = self.depth_model.rotary_emb(emb, pos_ids)
hidden_states = emb
for decoder_layer in self.depth_model.layers:
hidden_states = decoder_layer(
hidden_states,
attention_mask=self.decode_attn[cb_idx - 1],
position_ids=pos_ids,
past_key_values=self.static_cache,
use_cache=True,
cache_position=cache_pos,
position_embeddings=pos_emb,
)
hidden_states = self.depth_model.norm(hidden_states)
if self._debug_head_input is not None:
self._debug_head_input[cb_idx].copy_(hidden_states.float())
logits = self.codebooks_head(
hidden_states.float(),
cache_position=cache_pos,
) # [B, 1, vocab] fp32
if self.debug_logits is not None:
self.debug_logits[cb_idx].copy_(logits[:, 0, :])
self._cfg_sample(logits)
if self._debug_probs is not None:
self._debug_probs[cb_idx].copy_(self._debug_probs_slot)
self._tok_buf.clamp_(0, self.vocab_size - 1)
self.output_tokens[:, cb_idx] = self._tok_buf
# ------------------------------------------------------------------
# Snapshot / restore helpers for multi-bucket capture
# ------------------------------------------------------------------
def _snapshot_state(self) -> _BucketState:
"""Snapshot current self.* buffers into a _BucketState."""
return _BucketState(
batch_size=self.batch_size,
half=self.half,
graph=self.graph,
static_cache=self.static_cache,
backbone_hidden_buf=self.backbone_hidden_buf,
first_cb_token_buf=self.first_cb_token_buf,
output_tokens=self.output_tokens,
_tok_buf=self._tok_buf,
prefill_input_ids=self.prefill_input_ids,
prefill_attn=self.prefill_attn,
decode_attn=self.decode_attn,
temperature_buf=self.temperature_buf,
top_k_buf=self.top_k_buf,
top_p_buf=self.top_p_buf,
do_sample_buf=self.do_sample_buf,
guidance_scale=self.guidance_scale,
debug_logits=self.debug_logits,
_debug_head_input=self._debug_head_input,
_debug_probs=self._debug_probs,
_debug_probs_slot=self._debug_probs_slot,
)
def _swap_to_bucket(self, state: _BucketState):
"""Point self.* at the given bucket's buffers (for graph replay)."""
self.batch_size = state.batch_size
self.half = state.half
self.graph = state.graph
self.static_cache = state.static_cache
self.backbone_hidden_buf = state.backbone_hidden_buf
self.first_cb_token_buf = state.first_cb_token_buf
self.output_tokens = state.output_tokens
self._tok_buf = state._tok_buf
self.prefill_input_ids = state.prefill_input_ids
self.prefill_attn = state.prefill_attn
self.decode_attn = state.decode_attn
self.temperature_buf = state.temperature_buf
self.top_k_buf = state.top_k_buf
self.top_p_buf = state.top_p_buf
self.do_sample_buf = state.do_sample_buf
self.guidance_scale = state.guidance_scale
self.debug_logits = state.debug_logits
self._debug_head_input = state._debug_head_input
self._debug_probs = state._debug_probs
self._debug_probs_slot = state._debug_probs_slot
# ------------------------------------------------------------------
# Bucket selection
# ------------------------------------------------------------------
def _select_bucket(self, actual_batch: int) -> int | None:
"""Return smallest bucket_size >= actual_batch, or None if none fits."""
for bsz in self.bucket_sizes:
if bsz >= actual_batch:
return bsz
return None
@staticmethod
def _copy_cfg_padded(dst, src, actual_batch: int, bucket_half: int):
"""Copy [cond..., uncond...] rows into a padded CFG bucket.
``_cfg_sample`` slices the static bucket as
``[:bucket_half]`` for conditional rows and ``[bucket_half:]`` for
unconditional rows. When an actual batch is padded to a larger bucket,
real uncond rows must start at ``bucket_half`` rather than immediately
after the real cond rows.
"""
actual_half = actual_batch // 2
bucket_batch = bucket_half * 2
dst[:actual_half].copy_(src[:actual_half])
dst[bucket_half : bucket_half + actual_half].copy_(
src[actual_half:actual_batch]
)
if actual_half < bucket_half:
dst[actual_half:bucket_half].zero_()
dst[bucket_half + actual_half : bucket_batch].zero_()
# ------------------------------------------------------------------
# Padded param setter (for bucket padding)
# ------------------------------------------------------------------
def _set_param_padded(self, buf, value, actual_n, bucket_n, is_1d=False):
"""Set a sampling param buffer with padding for bucket size mismatch.
Args:
buf: target buffer, shape [bucket_n, 1] or [bucket_n]
value: scalar or tensor
actual_n: number of actual samples
bucket_n: bucket's half size
is_1d: True if buf is 1D (e.g. do_sample_buf)
"""
if isinstance(value, (int, float, bool)):
buf.fill_(int(value) if is_1d else value)
else:
t = value.view(actual_n) if is_1d else value.view(actual_n, -1)
if is_1d:
buf[:actual_n].copy_(t)
if actual_n < bucket_n:
buf[actual_n:].fill_(t[0].item())
else:
buf[:actual_n].copy_(t[:, : buf.shape[-1]])
if actual_n < bucket_n:
buf[actual_n:].fill_(t[0, 0].item())
# ------------------------------------------------------------------
# Capture: single bucket + multi-bucket entry point
# ------------------------------------------------------------------
@torch.inference_mode()
def _capture_single_bucket(self, bsz: int, num_warmup: int):
"""Capture a CUDA graph for a single batch size."""
# Set up state for this bucket size
self.batch_size = bsz
self.half = self._real_batch_size(bsz)
self.static_cache = StaticCache(
config=self.config, max_cache_len=self.max_seq, batch_size=bsz
)
self._alloc_buffers()
self._init_cache_layers()
self._build_attention_masks()
# Warmup
for _ in range(num_warmup):
self.static_cache.reset()
self._full_loop()
torch.cuda.synchronize(device=self.device)
# Capture
s = torch.cuda.Stream(device=self.device)
s.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(s):
self.graph = torch.cuda.CUDAGraph()
self.static_cache.reset()
self._full_loop()
torch.cuda.synchronize(device=self.device)
self.static_cache.reset()
with torch.cuda.graph(self.graph):
self._full_loop()
torch.cuda.current_stream().wait_stream(s)
torch.cuda.synchronize(device=self.device)
# Save state for this bucket
self._bucket_graphs[bsz] = self._snapshot_state()
print(f" Depth decoder CUDA graph captured for bucket_size={bsz}")
@torch.inference_mode()
def capture(self, num_warmup=3):
"""Warmup and capture CUDA graphs for all bucket sizes."""
if self.fast:
num_warmup = max(num_warmup, 8)
print(
f"Capturing depth decoder graphs for buckets={self.bucket_sizes} "
f"({num_warmup} warmup runs, fast={self.fast})..."
)
for bsz in self.bucket_sizes:
self._capture_single_bucket(bsz, num_warmup)
# Leave self.* pointing at the smallest bucket (default)
self._swap_to_bucket(self._bucket_graphs[self.bucket_sizes[0]])
self.captured = True
print(
f"Depth decoder CUDA graphs captured for all {len(self.bucket_sizes)} buckets!"
)
return self
# ------------------------------------------------------------------
# Eager fallback (for batch sizes exceeding all buckets)
# ------------------------------------------------------------------
@torch.inference_mode()
def _run_eager(
self,
backbone_hidden,
first_cb_token,
temperature=None,
top_k=None,
top_p=None,
do_sample=None,
guidance_scale=None,
):
"""Fallback: ensure_batch_size + _full_loop (no CUDA graph)."""
actual_batch = backbone_hidden.shape[0]
self.ensure_batch_size(actual_batch)
self.backbone_hidden_buf.copy_(backbone_hidden)
self.first_cb_token_buf.copy_(first_cb_token)
if temperature is not None:
self.set_temperature(temperature)
if top_k is not None:
self.set_top_k(top_k)
if top_p is not None:
self.set_top_p(top_p)
if do_sample is not None:
self.set_do_sample(do_sample)
if guidance_scale is not None:
self.set_guidance_scale(guidance_scale)
self.static_cache.reset()
self._full_loop()
return self.output_tokens[: self.half].clone()
# ------------------------------------------------------------------
# Run: bucket selection + graph replay
# ------------------------------------------------------------------
@torch.inference_mode()
def run(
self,
backbone_hidden,
first_cb_token,
temperature=None,
top_k=None,
top_p=None,
do_sample=None,
guidance_scale=None,
):
"""
Run the captured graph with automatic bucket selection.
backbone_hidden: [batch_size, backbone_hidden_size]
first_cb_token: [batch_size] long tensor
Optional per-sample sampling params (applied before graph replay):
temperature: float or [half] / [half,1] tensor
top_k: int or [half] / [half,1] tensor
top_p: float or [half] / [half,1] tensor
do_sample: bool or [half] long tensor (0/1)
guidance_scale: float or [half] / [half,1] tensor
Returns: [actual_half, num_decode_codebooks] long tensor of codebook tokens
"""
actual_batch = backbone_hidden.shape[0]
actual_half = self._real_batch_size(actual_batch)
# Safety guard: single CFG expects paired cond+uncond rows. no-CFG uses
# batch=1 and is valid.
if actual_batch != 1 and actual_batch % 2 != 0:
_log.warning(
"DepthDecoderGraph.run: invalid CFG actual_batch=%d, "
"returning zeros to avoid CUDA graph mismatch",
actual_batch,
)
return torch.zeros(
1,
self.num_decode_codebooks,
dtype=torch.long,
device=self.device,
)
# Eager fallback: no_graph flag or no fitting bucket
bucket_bsz = self._select_bucket(actual_batch)
if self.no_graph or bucket_bsz is None:
return self._run_eager(
backbone_hidden,
first_cb_token,
temperature,
top_k,
top_p,
do_sample,
guidance_scale,
)
# Swap to the selected bucket's state
state = self._bucket_graphs[bucket_bsz]
self._swap_to_bucket(state)
bucket_half = self.half
if actual_batch == 1:
self.backbone_hidden_buf[:1].copy_(backbone_hidden[:1])
self.first_cb_token_buf[:1].copy_(first_cb_token[:1])
else:
# Preserve the CFG layout expected by _cfg_sample:
# [cond real][cond pad][uncond real][uncond pad].
self._copy_cfg_padded(
self.backbone_hidden_buf, backbone_hidden, actual_batch, bucket_half
)
self._copy_cfg_padded(
self.first_cb_token_buf, first_cb_token, actual_batch, bucket_half
)
# Set sampling params with padding
if temperature is not None:
self._set_param_padded(
self.temperature_buf, temperature, actual_half, bucket_half
)
if top_k is not None:
self._set_param_padded(self.top_k_buf, top_k, actual_half, bucket_half)
if top_p is not None:
self._set_param_padded(self.top_p_buf, top_p, actual_half, bucket_half)
if do_sample is not None:
self._set_param_padded(
self.do_sample_buf, do_sample, actual_half, bucket_half, is_1d=True
)
if guidance_scale is not None:
self._set_param_padded(
self.guidance_scale, guidance_scale, actual_half, bucket_half
)
# Replay graph
self.static_cache.reset()
state.graph.replay()
if self.debug and self._debug_head_input is not None:
head = self._orig_codebooks_head
for cb_idx in range(self.num_decode_codebooks):
if cb_idx == 0:
head(
self._debug_head_input[0], cache_position=self.head_prefill_pos
)
else:
head(
self._debug_head_input[cb_idx],
cache_position=self.decode_cache_positions[cb_idx - 1],
)
torch.multinomial(
self._debug_probs[cb_idx], 1, _hook_only_for_debug=True
)
# Slice output to actual_half (discard padding)
return self.output_tokens[:actual_half].clone()