Spaces:
Running on Zero
Running on Zero
Download models/cudagraph/backbone_prefill_graph.py from Quantumbraid/agent.voice: direct link, hf CLI and curl.
- Browser
- Download file 9.5 kB
-
https://huggingface.co/spaces/Quantumbraid/agent.voice/resolve/main/models/cudagraph/backbone_prefill_graph.py
- Command line
-
hf download hf://spaces/Quantumbraid/agent.voice/models/cudagraph/backbone_prefill_graph.py
-
curl -L -o backbone_prefill_graph.py https://huggingface.co/spaces/Quantumbraid/agent.voice/resolve/main/models/cudagraph/backbone_prefill_graph.py
9.5 kB
| """Static-shape CUDA Graph buckets for batch-1 backbone prefill.""" | |
| from __future__ import annotations | |
| import threading | |
| from dataclasses import dataclass | |
| import torch | |
| class BackbonePrefillOutput: | |
| hidden_states: torch.Tensor | |
| logits: torch.Tensor | |
| attention_mask: torch.Tensor | |
| prefill_len: int | |
| class _PrefillRecord: | |
| graph: torch.cuda.CUDAGraph | |
| stream: torch.cuda.Stream | |
| inputs_embeds: torch.Tensor | |
| attention_mask: torch.Tensor | |
| position_ids: torch.Tensor | |
| causal_mask: torch.Tensor | |
| cache_position: torch.Tensor | |
| hidden_states: torch.Tensor | |
| logits: torch.Tensor | |
| class BackbonePrefillGraphCache: | |
| """Capture backbone prefill + lm_head while writing the decode StaticCache.""" | |
| def __init__(self, backbone_graph, *, token_granularity: int = 32) -> None: | |
| if token_granularity <= 0: | |
| raise ValueError("token_granularity must be > 0") | |
| self.backbone_graph = backbone_graph | |
| self.model = backbone_graph.model | |
| self.lm_head = backbone_graph.lm_head | |
| self.device = torch.device(backbone_graph.device) | |
| self.dtype = backbone_graph.dtype | |
| self.token_granularity = int(token_granularity) | |
| self._records: dict[tuple[int, int], _PrefillRecord] = {} | |
| self._graph_pool = torch.cuda.graph_pool_handle() | |
| self._lock = threading.RLock() | |
| self.captures = 0 | |
| self.replays = 0 | |
| self._frozen = False | |
| def _bucket(self, length: int) -> int: | |
| return ( | |
| (int(length) + self.token_granularity - 1) // self.token_granularity | |
| ) * self.token_granularity | |
| def _copy_inputs( | |
| inputs_embeds: torch.Tensor, | |
| attention_mask: torch.Tensor, | |
| static_embeds: torch.Tensor, | |
| static_mask: torch.Tensor, | |
| static_positions: torch.Tensor, | |
| ) -> None: | |
| seq_len = int(inputs_embeds.shape[1]) | |
| pad_len = int(static_embeds.shape[1]) - seq_len | |
| static_embeds.zero_() | |
| static_mask.zero_() | |
| static_positions.fill_(1) | |
| static_embeds[:, pad_len:].copy_(inputs_embeds) | |
| static_mask[:, pad_len:].copy_(attention_mask) | |
| positions = static_mask.long().cumsum(-1) - 1 | |
| positions.masked_fill_(static_mask == 0, 1) | |
| static_positions.copy_(positions) | |
| def _update_causal_mask( | |
| attention_mask: torch.Tensor, causal_mask: torch.Tensor | |
| ) -> None: | |
| batch_size, query_len = attention_mask.shape | |
| key_len = causal_mask.shape[-1] | |
| query_idx = torch.arange(query_len, device=attention_mask.device).view( | |
| 1, query_len, 1 | |
| ) | |
| key_idx = torch.arange(key_len, device=attention_mask.device).view( | |
| 1, 1, key_len | |
| ) | |
| valid_keys = torch.zeros( | |
| (batch_size, key_len), dtype=torch.bool, device=attention_mask.device | |
| ) | |
| valid_keys[:, :query_len].copy_(attention_mask.to(torch.bool)) | |
| allowed = (key_idx <= query_idx) & valid_keys[:, None, :] | |
| causal_mask.fill_(torch.finfo(causal_mask.dtype).min) | |
| causal_mask[:, 0].masked_fill_(allowed, 0.0) | |
| def _forward( | |
| self, | |
| inputs_embeds: torch.Tensor, | |
| attention_mask: torch.Tensor, | |
| position_ids: torch.Tensor, | |
| cache_position: torch.Tensor, | |
| ) -> tuple[torch.Tensor, torch.Tensor]: | |
| output = self.model( | |
| inputs_embeds=inputs_embeds, | |
| attention_mask=attention_mask, | |
| position_ids=position_ids, | |
| past_key_values=self.backbone_graph.static_cache, | |
| cache_position=cache_position, | |
| use_cache=True, | |
| ) | |
| hidden_states = output.last_hidden_state | |
| logits = self.lm_head(hidden_states[:, -1, :].float()).float() | |
| return hidden_states, logits | |
| def __call__( | |
| self, | |
| inputs_embeds: torch.Tensor, | |
| attention_mask: torch.Tensor, | |
| ) -> BackbonePrefillOutput: | |
| batch_size, seq_len, hidden_size = inputs_embeds.shape | |
| if batch_size != self.backbone_graph.batch_size: | |
| raise ValueError( | |
| f"prefill batch={batch_size} does not match decode graph " | |
| f"batch={self.backbone_graph.batch_size}" | |
| ) | |
| bucket_len = self._bucket(int(seq_len)) | |
| if bucket_len > self.backbone_graph.max_seq_len: | |
| raise RuntimeError( | |
| f"prefill bucket {bucket_len} exceeds max_seq_len " | |
| f"{self.backbone_graph.max_seq_len}" | |
| ) | |
| key = (int(batch_size), bucket_len) | |
| with self._lock: | |
| record = self._records.get(key) | |
| if record is None: | |
| if self._frozen: | |
| raise RuntimeError( | |
| f"backbone prefill CUDA graph {key} was not declared in the warmup profile" | |
| ) | |
| static_embeds = torch.zeros( | |
| (batch_size, bucket_len, hidden_size), | |
| dtype=inputs_embeds.dtype, | |
| device=inputs_embeds.device, | |
| ) | |
| static_mask = torch.zeros( | |
| (batch_size, bucket_len), | |
| dtype=attention_mask.dtype, | |
| device=attention_mask.device, | |
| ) | |
| static_positions = torch.ones_like(static_mask, dtype=torch.long) | |
| static_causal_mask = torch.empty( | |
| (batch_size, 1, bucket_len, self.backbone_graph.max_seq_len), | |
| dtype=inputs_embeds.dtype, | |
| device=inputs_embeds.device, | |
| ) | |
| cache_position = torch.arange( | |
| bucket_len, device=inputs_embeds.device, dtype=torch.long | |
| ) | |
| self._copy_inputs( | |
| inputs_embeds, | |
| attention_mask, | |
| static_embeds, | |
| static_mask, | |
| static_positions, | |
| ) | |
| self._update_causal_mask(static_mask, static_causal_mask) | |
| capture_stream = torch.cuda.Stream(device=inputs_embeds.device) | |
| capture_stream.wait_stream( | |
| torch.cuda.current_stream(inputs_embeds.device) | |
| ) | |
| with torch.cuda.stream(capture_stream): | |
| for _ in range(3): | |
| hidden_states, logits = self._forward( | |
| static_embeds, | |
| static_causal_mask, | |
| static_positions, | |
| cache_position, | |
| ) | |
| capture_stream.synchronize() | |
| graph = torch.cuda.CUDAGraph() | |
| with torch.cuda.graph( | |
| graph, | |
| stream=capture_stream, | |
| pool=self._graph_pool, | |
| capture_error_mode="thread_local", | |
| ): | |
| hidden_states, logits = self._forward( | |
| static_embeds, | |
| static_causal_mask, | |
| static_positions, | |
| cache_position, | |
| ) | |
| record = _PrefillRecord( | |
| graph=graph, | |
| stream=capture_stream, | |
| inputs_embeds=static_embeds, | |
| attention_mask=static_mask, | |
| position_ids=static_positions, | |
| causal_mask=static_causal_mask, | |
| cache_position=cache_position, | |
| hidden_states=hidden_states, | |
| logits=logits, | |
| ) | |
| self._records[key] = record | |
| self.captures += 1 | |
| self._copy_inputs( | |
| inputs_embeds, | |
| attention_mask, | |
| record.inputs_embeds, | |
| record.attention_mask, | |
| record.position_ids, | |
| ) | |
| self._update_causal_mask(record.attention_mask, record.causal_mask) | |
| current_stream = torch.cuda.current_stream(inputs_embeds.device) | |
| record.stream.wait_stream(current_stream) | |
| record.graph.replay() | |
| current_stream.wait_stream(record.stream) | |
| self.backbone_graph.finish_direct_prefill(bucket_len) | |
| self.replays += 1 | |
| return BackbonePrefillOutput( | |
| hidden_states=record.hidden_states, | |
| logits=record.logits, | |
| attention_mask=record.attention_mask, | |
| prefill_len=bucket_len, | |
| ) | |
| def graph_keys(self) -> tuple[tuple[int, int], ...]: | |
| return tuple(sorted(self._records)) | |
| def warmup_graph(self, *, branch_batch_size: int, sequence_length: int) -> None: | |
| if self._frozen: | |
| raise RuntimeError("backbone prefill CUDA graph cache is already frozen") | |
| inputs_embeds = torch.zeros( | |
| branch_batch_size, | |
| sequence_length, | |
| int(self.backbone_graph.hidden_size), | |
| dtype=self.dtype, | |
| device=self.device, | |
| ) | |
| attention_mask = torch.ones( | |
| branch_batch_size, | |
| sequence_length, | |
| dtype=torch.long, | |
| device=self.device, | |
| ) | |
| self(inputs_embeds, attention_mask) | |
| def freeze(self) -> None: | |
| self._frozen = True | |