agent.voice / models /cudagraph /backbone_prefill_graph.py
multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
878a616 verified
Raw History Blame Contribute Delete
9.5 kB
"""Static-shape CUDA Graph buckets for batch-1 backbone prefill."""
from __future__ import annotations
import threading
from dataclasses import dataclass
import torch
@dataclass
class BackbonePrefillOutput:
hidden_states: torch.Tensor
logits: torch.Tensor
attention_mask: torch.Tensor
prefill_len: int
@dataclass
class _PrefillRecord:
graph: torch.cuda.CUDAGraph
stream: torch.cuda.Stream
inputs_embeds: torch.Tensor
attention_mask: torch.Tensor
position_ids: torch.Tensor
causal_mask: torch.Tensor
cache_position: torch.Tensor
hidden_states: torch.Tensor
logits: torch.Tensor
class BackbonePrefillGraphCache:
"""Capture backbone prefill + lm_head while writing the decode StaticCache."""
def __init__(self, backbone_graph, *, token_granularity: int = 32) -> None:
if token_granularity <= 0:
raise ValueError("token_granularity must be > 0")
self.backbone_graph = backbone_graph
self.model = backbone_graph.model
self.lm_head = backbone_graph.lm_head
self.device = torch.device(backbone_graph.device)
self.dtype = backbone_graph.dtype
self.token_granularity = int(token_granularity)
self._records: dict[tuple[int, int], _PrefillRecord] = {}
self._graph_pool = torch.cuda.graph_pool_handle()
self._lock = threading.RLock()
self.captures = 0
self.replays = 0
self._frozen = False
def _bucket(self, length: int) -> int:
return (
(int(length) + self.token_granularity - 1) // self.token_granularity
) * self.token_granularity
@staticmethod
def _copy_inputs(
inputs_embeds: torch.Tensor,
attention_mask: torch.Tensor,
static_embeds: torch.Tensor,
static_mask: torch.Tensor,
static_positions: torch.Tensor,
) -> None:
seq_len = int(inputs_embeds.shape[1])
pad_len = int(static_embeds.shape[1]) - seq_len
static_embeds.zero_()
static_mask.zero_()
static_positions.fill_(1)
static_embeds[:, pad_len:].copy_(inputs_embeds)
static_mask[:, pad_len:].copy_(attention_mask)
positions = static_mask.long().cumsum(-1) - 1
positions.masked_fill_(static_mask == 0, 1)
static_positions.copy_(positions)
@staticmethod
def _update_causal_mask(
attention_mask: torch.Tensor, causal_mask: torch.Tensor
) -> None:
batch_size, query_len = attention_mask.shape
key_len = causal_mask.shape[-1]
query_idx = torch.arange(query_len, device=attention_mask.device).view(
1, query_len, 1
)
key_idx = torch.arange(key_len, device=attention_mask.device).view(
1, 1, key_len
)
valid_keys = torch.zeros(
(batch_size, key_len), dtype=torch.bool, device=attention_mask.device
)
valid_keys[:, :query_len].copy_(attention_mask.to(torch.bool))
allowed = (key_idx <= query_idx) & valid_keys[:, None, :]
causal_mask.fill_(torch.finfo(causal_mask.dtype).min)
causal_mask[:, 0].masked_fill_(allowed, 0.0)
def _forward(
self,
inputs_embeds: torch.Tensor,
attention_mask: torch.Tensor,
position_ids: torch.Tensor,
cache_position: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
output = self.model(
inputs_embeds=inputs_embeds,
attention_mask=attention_mask,
position_ids=position_ids,
past_key_values=self.backbone_graph.static_cache,
cache_position=cache_position,
use_cache=True,
)
hidden_states = output.last_hidden_state
logits = self.lm_head(hidden_states[:, -1, :].float()).float()
return hidden_states, logits
@torch.inference_mode()
def __call__(
self,
inputs_embeds: torch.Tensor,
attention_mask: torch.Tensor,
) -> BackbonePrefillOutput:
batch_size, seq_len, hidden_size = inputs_embeds.shape
if batch_size != self.backbone_graph.batch_size:
raise ValueError(
f"prefill batch={batch_size} does not match decode graph "
f"batch={self.backbone_graph.batch_size}"
)
bucket_len = self._bucket(int(seq_len))
if bucket_len > self.backbone_graph.max_seq_len:
raise RuntimeError(
f"prefill bucket {bucket_len} exceeds max_seq_len "
f"{self.backbone_graph.max_seq_len}"
)
key = (int(batch_size), bucket_len)
with self._lock:
record = self._records.get(key)
if record is None:
if self._frozen:
raise RuntimeError(
f"backbone prefill CUDA graph {key} was not declared in the warmup profile"
)
static_embeds = torch.zeros(
(batch_size, bucket_len, hidden_size),
dtype=inputs_embeds.dtype,
device=inputs_embeds.device,
)
static_mask = torch.zeros(
(batch_size, bucket_len),
dtype=attention_mask.dtype,
device=attention_mask.device,
)
static_positions = torch.ones_like(static_mask, dtype=torch.long)
static_causal_mask = torch.empty(
(batch_size, 1, bucket_len, self.backbone_graph.max_seq_len),
dtype=inputs_embeds.dtype,
device=inputs_embeds.device,
)
cache_position = torch.arange(
bucket_len, device=inputs_embeds.device, dtype=torch.long
)
self._copy_inputs(
inputs_embeds,
attention_mask,
static_embeds,
static_mask,
static_positions,
)
self._update_causal_mask(static_mask, static_causal_mask)
capture_stream = torch.cuda.Stream(device=inputs_embeds.device)
capture_stream.wait_stream(
torch.cuda.current_stream(inputs_embeds.device)
)
with torch.cuda.stream(capture_stream):
for _ in range(3):
hidden_states, logits = self._forward(
static_embeds,
static_causal_mask,
static_positions,
cache_position,
)
capture_stream.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(
graph,
stream=capture_stream,
pool=self._graph_pool,
capture_error_mode="thread_local",
):
hidden_states, logits = self._forward(
static_embeds,
static_causal_mask,
static_positions,
cache_position,
)
record = _PrefillRecord(
graph=graph,
stream=capture_stream,
inputs_embeds=static_embeds,
attention_mask=static_mask,
position_ids=static_positions,
causal_mask=static_causal_mask,
cache_position=cache_position,
hidden_states=hidden_states,
logits=logits,
)
self._records[key] = record
self.captures += 1
self._copy_inputs(
inputs_embeds,
attention_mask,
record.inputs_embeds,
record.attention_mask,
record.position_ids,
)
self._update_causal_mask(record.attention_mask, record.causal_mask)
current_stream = torch.cuda.current_stream(inputs_embeds.device)
record.stream.wait_stream(current_stream)
record.graph.replay()
current_stream.wait_stream(record.stream)
self.backbone_graph.finish_direct_prefill(bucket_len)
self.replays += 1
return BackbonePrefillOutput(
hidden_states=record.hidden_states,
logits=record.logits,
attention_mask=record.attention_mask,
prefill_len=bucket_len,
)
@property
def graph_keys(self) -> tuple[tuple[int, int], ...]:
return tuple(sorted(self._records))
@torch.inference_mode()
def warmup_graph(self, *, branch_batch_size: int, sequence_length: int) -> None:
if self._frozen:
raise RuntimeError("backbone prefill CUDA graph cache is already frozen")
inputs_embeds = torch.zeros(
branch_batch_size,
sequence_length,
int(self.backbone_graph.hidden_size),
dtype=self.dtype,
device=self.device,
)
attention_mask = torch.ones(
branch_batch_size,
sequence_length,
dtype=torch.long,
device=self.device,
)
self(inputs_embeds, attention_mask)
def freeze(self) -> None:
self._frozen = True