Spaces:
Running on Zero
Running on Zero
Download p_image_oss_fast.py from PrunaAI/P-Image-OSS: direct link, hf CLI and curl.
- Browser
- Download file 43.5 kB
-
https://huggingface.co/spaces/PrunaAI/P-Image-OSS/resolve/main/p_image_oss_fast.py
- Command line
-
hf download hf://spaces/PrunaAI/P-Image-OSS/p_image_oss_fast.py
-
curl -L -o p_image_oss_fast.py https://huggingface.co/spaces/PrunaAI/P-Image-OSS/resolve/main/p_image_oss_fast.py
43.5 kB
| """Kernel-level speedups for p-image-oss (the diffusers fork) on an RTX PRO 6000 Blackwell (sm_120). No torch.compile. | |
| * `install_adaln_tables`: every block's AdaLN projection (13B of the transformer's 33B parameters) depends only on | |
| the timestep, and the pipeline only ever uses the timesteps of its fixed sigma grid. Their outputs are computed once | |
| per transformer call signature and the projection weights are dropped. Bitwise identical to the projections. | |
| * `fuse_qkv`: one GEMM for q, k and v. | |
| * `quantize_fp8`: W8A8 linears, per-output-channel weight scales and per-row dynamic activation scales, through | |
| torch._scaled_mm (cuBLASLt FP8). Activations are quantized by one Triton kernel per GEMM input. | |
| """ | |
| from __future__ import annotations | |
| import inspect | |
| import time | |
| import torch | |
| import triton | |
| import triton.language as tl | |
| from torch import nn | |
| FP8 = torch.float8_e4m3fn | |
| FP8_MAX = 448.0 | |
| def _free_cache() -> None: | |
| try: | |
| if torch.cuda.is_available() and torch.cuda.is_initialized(): | |
| torch.cuda.empty_cache() | |
| except Exception: # noqa: BLE001 - ZeroGPU emulates CUDA outside GPU calls | |
| pass | |
| # AdaLN tables ------------------------------------------------------------------------------------------------------- | |
| class AdaLNTable(nn.Module): | |
| """Replaces `MiniMaxH3AdaLayerNormModulation`: returns its precomputed output for the current call's timesteps.""" | |
| def __init__(self, tables: list[torch.Tensor], hidden_size: int, lookup: dict): | |
| super().__init__() | |
| for index, table in enumerate(tables): | |
| self.register_buffer(f"table_{index}", table, persistent=False) | |
| self.hidden_size = hidden_size | |
| self.lookup = lookup | |
| def forward(self, temb: torch.Tensor) -> tuple[torch.Tensor, ...]: | |
| table = getattr(self, f"table_{self.lookup['index']}") | |
| return table.view(-1, 6 * self.hidden_size).chunk(6, dim=-1) | |
| def install_adaln_tables(transformer, calls: list[torch.Tensor]) -> int: | |
| """Precompute every block's AdaLN output for each distinct `timestep` argument in `calls` (as the pipeline passes | |
| them, one tensor per transformer call) and drop the projection weights. Each table is computed with the call's own | |
| shapes, exactly as the forward would, so the outputs are bitwise identical.""" | |
| device = next(transformer.parameters()).device | |
| keys: dict[tuple, int] = {} | |
| tembs = [] | |
| for timestep in calls: | |
| key = tuple(timestep.tolist()) | |
| if key not in keys: | |
| keys[key] = len(keys) | |
| temb = transformer.time_proj(timestep.to(device)) | |
| tembs.append(transformer.time_embedder(temb.to(transformer.time_embedder.linear_1.weight.dtype))) | |
| lookup = {"keys": keys, "index": None} | |
| freed = 0 | |
| for block in transformer.transformer_blocks: | |
| proj = block.adaln_proj | |
| dtype = proj.linear.weight.dtype | |
| tables = [proj.linear(nn.functional.silu(temb).to(dtype)).contiguous() for temb in tembs] | |
| freed += sum(p.numel() for p in proj.parameters()) | |
| block.adaln_proj = AdaLNTable(tables, proj.hidden_size, lookup) | |
| del proj | |
| def set_index(module, args, kwargs): | |
| key = tuple(kwargs["timestep"].tolist()) | |
| if key not in keys: | |
| raise ValueError(f"No AdaLN table for timesteps {key}; known: {list(keys)}") | |
| lookup["index"] = keys[key] | |
| transformer.register_forward_pre_hook(set_index, with_kwargs=True) | |
| _free_cache() | |
| return freed | |
| # QKV fusion --------------------------------------------------------------------------------------------------------- | |
| def fuse_qkv(transformer) -> None: | |
| for block in transformer.transformer_blocks: | |
| attn = block.attn | |
| weight = torch.cat([attn.to_q.weight, attn.to_k.weight, attn.to_v.weight]) | |
| attn.to_qkv = nn.Linear(weight.shape[1], weight.shape[0], bias=False, device=weight.device, dtype=weight.dtype) | |
| attn.to_qkv.weight.copy_(weight) | |
| del attn.to_q, attn.to_k, attn.to_v | |
| attn.fused_projections = True | |
| _free_cache() | |
| # FP8 W8A8 ----------------------------------------------------------------------------------------------------------- | |
| def _rowwise_fp8_kernel(x_ptr, q_ptr, s_ptr, K, stride_x, stride_q, BLOCK: tl.constexpr): | |
| row = tl.program_id(0) | |
| offs = tl.arange(0, BLOCK) | |
| mask = offs < K | |
| x = tl.load(x_ptr + row * stride_x + offs, mask=mask, other=0.0).to(tl.float32) | |
| amax = tl.maximum(tl.max(tl.abs(x), axis=0), 1e-12) | |
| tl.store(q_ptr + row * stride_q + offs, (x * (448.0 / amax)).to(tl.float8e4nv), mask=mask) | |
| tl.store(s_ptr + row, amax / 448.0) | |
| def rowwise_fp8(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: | |
| """(M, K) bf16 -> (M, K) e4m3 and (M, 1) float32 scales with x ~= q * scale.""" | |
| M, K = x.shape | |
| q = torch.empty((M, K), device=x.device, dtype=FP8) | |
| scale = torch.empty((M, 1), device=x.device, dtype=torch.float32) | |
| block = triton.next_power_of_2(K) | |
| _rowwise_fp8_kernel[(M,)](x, q, scale, K, x.stride(0), q.stride(0), BLOCK=block, num_warps=8 if block <= 8192 else 16) | |
| return q, scale | |
| _W8A16_CONFIGS = [triton.Config({"BLOCK_N": n, "BLOCK_K": k}, num_warps=w, num_stages=4) | |
| for n in (16, 32, 64) for k in (128, 256) for w in (4, 8)] | |
| # Tuned configs persist in Triton's cache dir, so a new process (ZeroGPU workers) skips the autotune. | |
| _AUTOTUNE_CACHE = {"cache_results": True} if "cache_results" in inspect.signature(triton.autotune).parameters else {} | |
| def _w8a16_kernel(x_ptr, w_ptr, s_ptr, b_ptr, y_ptr, M, N, K, stride_x, stride_w, stride_y, | |
| HAS_BIAS: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr): | |
| offs_n = tl.program_id(0) * BLOCK_N + tl.arange(0, BLOCK_N) | |
| offs_m = tl.arange(0, 16) | |
| acc = tl.zeros((16, BLOCK_N), dtype=tl.float32) | |
| for k0 in range(0, K, BLOCK_K): | |
| offs_k = k0 + tl.arange(0, BLOCK_K) | |
| w = tl.load(w_ptr + offs_n[:, None] * stride_w + offs_k[None, :], | |
| mask=(offs_n[:, None] < N) & (offs_k[None, :] < K), other=0.0).to(tl.bfloat16) | |
| x = tl.load(x_ptr + offs_m[:, None] * stride_x + offs_k[None, :], | |
| mask=(offs_m[:, None] < M) & (offs_k[None, :] < K), other=0.0) | |
| acc += tl.dot(x, tl.trans(w)) | |
| acc *= tl.load(s_ptr + offs_n, mask=offs_n < N, other=0.0)[None, :] | |
| if HAS_BIAS: | |
| acc += tl.load(b_ptr + offs_n, mask=offs_n < N, other=0.0).to(tl.float32)[None, :] | |
| tl.store(y_ptr + offs_m[:, None] * stride_y + offs_n[None, :], acc.to(y_ptr.dtype.element_ty), | |
| mask=(offs_m[:, None] < M) & (offs_n[None, :] < N)) | |
| def w8a16_gemv(x: torch.Tensor, weight: torch.Tensor, scale: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor: | |
| """y = x @ (weight * scale).T for at most 16 rows of x: FP8 weights dequantized in registers, bandwidth-bound.""" | |
| M, K = x.shape | |
| N = weight.shape[0] | |
| x = x.contiguous() | |
| y = torch.empty((M, N), device=x.device, dtype=x.dtype) | |
| grid = lambda meta: (triton.cdiv(N, meta["BLOCK_N"]),) | |
| _w8a16_kernel[grid](x, weight, scale, bias if bias is not None else scale, y, M, N, K, x.stride(0), | |
| weight.stride(0), y.stride(0), HAS_BIAS=bias is not None) | |
| return y | |
| class FP8Linear(nn.Module): | |
| def __init__(self, linear: nn.Linear, fast_accum: bool = False): | |
| super().__init__() | |
| weight = linear.weight.detach() | |
| scale = weight.float().abs().amax(dim=1, keepdim=True).clamp_min(1e-12) / FP8_MAX | |
| self.register_buffer("weight", (weight.float() / scale).to(FP8).contiguous()) # (N, K) | |
| self.register_buffer("weight_scale", scale.t().contiguous()) # (1, N) | |
| self.bias = linear.bias | |
| self.in_features, self.out_features = linear.in_features, linear.out_features | |
| self.out_dtype = weight.dtype | |
| self.fast_accum = fast_accum | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| shape = x.shape | |
| rows = x.reshape(-1, shape[-1]) | |
| if rows.shape[0] <= 16: # decode: weight-only, the GEMM is a bandwidth-bound GEMV | |
| return w8a16_gemv(rows.to(self.out_dtype), self.weight, self.weight_scale.view(-1), | |
| self.bias).view(*shape[:-1], self.out_features) | |
| q, scale = rowwise_fp8(rows) | |
| out = torch._scaled_mm(q, self.weight.t(), scale_a=scale, scale_b=self.weight_scale, bias=self.bias, | |
| out_dtype=self.out_dtype, use_fast_accum=self.fast_accum) | |
| return out.view(*shape[:-1], self.out_features) | |
| def quantize_fp8(module: nn.Module, fast_accum: bool = False, skip: tuple[str, ...] = ()) -> int: | |
| """Replace every nn.Linear under `module` (names containing an entry of `skip` excluded) by an FP8Linear.""" | |
| count = 0 | |
| for name, child in list(module.named_modules()): | |
| for child_name, leaf in list(child.named_children()): | |
| full = f"{name}.{child_name}" if name else child_name | |
| if isinstance(leaf, nn.Linear) and not any(s in full for s in skip): | |
| setattr(child, child_name, FP8Linear(leaf, fast_accum)) | |
| del leaf | |
| count += 1 | |
| _free_cache() | |
| return count | |
| # CUDA-graph greedy decoding ----------------------------------------------------------------------------------------- | |
| class GraphedGreedyGenerate: | |
| """Greedy decoding of one sequence for a Qwen3_5ForConditionalGeneration, with the decode step replayed as a CUDA | |
| graph. A drop-in for `model.generate(**inputs, max_new_tokens=..., do_sample=False, eos_token_id=...)`. | |
| The prefill runs eagerly into a persistent StaticCache (its linear-attention states and KV tensors are updated in | |
| place, so their addresses never change). One decode step (embed, 32 layers, lm_head, argmax) is captured once over | |
| static buffers: the input token, its absolute position, the M-RoPE offset and the attention mask over the cache. | |
| Per token only the graph is replayed; the end token is checked every `check_every` tokens. | |
| """ | |
| def __init__(self, model, max_cache_len: int = 12288, max_new_tokens: int = 4096, check_every: int = 16): | |
| from transformers import StaticCache | |
| self.model, self.generate_eager = model, model.generate | |
| self.max_cache_len, self.check_every = max_cache_len, check_every | |
| device = model.device | |
| self.cache = StaticCache(config=model.config, max_cache_len=max_cache_len) | |
| self.tok = torch.zeros((1, 1), dtype=torch.long, device=device) | |
| self.pos = torch.zeros((1,), dtype=torch.long, device=device) | |
| self.offset = torch.zeros((4, 1, 1), dtype=torch.long, device=device) | |
| self.mask = torch.zeros((1, 1, 1, max_cache_len), dtype=torch.bool, device=device) | |
| self.out = torch.zeros((max_new_tokens,), dtype=torch.long, device=device) | |
| self.count = torch.zeros((1,), dtype=torch.long, device=device) | |
| self.graph = None | |
| def _step(self): | |
| text_model = self.model.model.language_model | |
| self.mask.index_fill_(3, self.pos, True) | |
| hidden = text_model( | |
| inputs_embeds=text_model.embed_tokens(self.tok), | |
| position_ids=self.pos.view(1, 1, 1) + self.offset, | |
| attention_mask={"full_attention": self.mask, "linear_attention": None}, | |
| past_key_values=self.cache, | |
| use_cache=True, | |
| ).last_hidden_state | |
| token = self.model.lm_head(hidden[:, -1]).argmax(-1) | |
| self.out.index_copy_(0, self.count, token) | |
| self.tok.copy_(token.view(1, 1)) | |
| self.pos.add_(1) | |
| self.count.add_(1) | |
| def _prefill(self, input_ids, kwargs): | |
| self.cache.reset() | |
| self.mask.zero_() | |
| self.model.model.rope_deltas = None | |
| logits = self.model(input_ids=input_ids, past_key_values=self.cache, use_cache=True, logits_to_keep=1, | |
| **kwargs).logits | |
| token = logits[:, -1].argmax(-1) | |
| rope_deltas = self.model.model.rope_deltas | |
| self.offset.zero_() | |
| if rope_deltas is not None: | |
| self.offset[1:] = rope_deltas.view(-1)[0] | |
| prompt_length = input_ids.shape[1] | |
| self.mask[..., :prompt_length] = True | |
| self.out[0] = token[0] | |
| self.count.fill_(1) | |
| self.tok.copy_(token.view(1, 1)) | |
| self.pos.fill_(prompt_length) | |
| def _capture(self): | |
| stream = torch.cuda.Stream() | |
| stream.wait_stream(torch.cuda.current_stream()) | |
| with torch.cuda.stream(stream): | |
| for _ in range(3): | |
| self._step() | |
| torch.cuda.current_stream().wait_stream(stream) | |
| self.graph = torch.cuda.CUDAGraph() | |
| with torch.cuda.graph(self.graph): | |
| self._step() | |
| def generate(self, input_ids, max_new_tokens: int, eos_token_id: int, **kwargs): | |
| kwargs.pop("do_sample", None) | |
| kwargs.pop("pad_token_id", None) | |
| if input_ids.shape[0] != 1 or input_ids.shape[1] + max_new_tokens > self.max_cache_len: | |
| return self.generate_eager(input_ids=input_ids, max_new_tokens=max_new_tokens, do_sample=False, | |
| eos_token_id=eos_token_id, **kwargs) | |
| if self.graph is None: | |
| self._prefill(input_ids, kwargs) | |
| self._capture() | |
| self._prefill(input_ids, kwargs) | |
| generated = 1 | |
| while generated < max_new_tokens and not bool((self.out[:generated] == eos_token_id).any()): | |
| for _ in range(min(self.check_every, max_new_tokens - generated)): | |
| self.graph.replay() | |
| generated = min(generated + self.check_every, max_new_tokens) | |
| tokens = self.out[:generated] | |
| ends = (tokens == eos_token_id).nonzero() | |
| if ends.numel(): | |
| tokens = tokens[: int(ends[0, 0]) + 1] | |
| return torch.cat([input_ids, tokens[None].to(input_ids.device)], dim=1) | |
| # MTP speculative decoding ------------------------------------------------------------------------------------------- | |
| class MTPHead(nn.Module): | |
| """Qwen3.5's multi-token-prediction head: (hidden state at i, embedding of token i+1) -> hidden for token i+2.""" | |
| def __init__(self, config): | |
| super().__init__() | |
| from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5DecoderLayer, Qwen3_5RMSNorm | |
| size, eps = config.hidden_size, config.rms_norm_eps | |
| self.config = config | |
| self.fc = nn.Linear(2 * size, size, bias=False) | |
| self.pre_fc_norm_embedding = Qwen3_5RMSNorm(size, eps) | |
| self.pre_fc_norm_hidden = Qwen3_5RMSNorm(size, eps) | |
| self.layers = nn.ModuleList([Qwen3_5DecoderLayer(config, 0)]) | |
| self.norm = Qwen3_5RMSNorm(size, eps) | |
| def forward(self, embeds, hidden, position_embeddings, text_positions, mask, cache): | |
| x = self.fc(torch.cat([self.pre_fc_norm_embedding(embeds), self.pre_fc_norm_hidden(hidden)], dim=-1)) | |
| x = self.layers[0](x, position_embeddings=position_embeddings, attention_mask=mask, position_ids=text_positions, | |
| past_key_values=cache, use_cache=True) | |
| return self.norm(x) | |
| MTP_REPO, MTP_REVISION = "Qwen/Qwen3.5-9B", "c202236235762e1c871ad0ccb60c8ee5ba337b9a" | |
| def read_hub_tensors(repo: str, revision: str, prefix: str) -> dict[str, torch.Tensor]: | |
| """The tensors named `prefix*` of a sharded safetensors repo, read by byte range (no full shard downloads).""" | |
| import json | |
| import struct | |
| from huggingface_hub import HfFileSystem, hf_hub_download | |
| dtypes = {"BF16": torch.bfloat16, "F16": torch.float16, "F32": torch.float32} | |
| index = json.load(open(hf_hub_download(repo, "model.safetensors.index.json", revision=revision))) | |
| fs, tensors = HfFileSystem(), {} | |
| for file in sorted({f for name, f in index["weight_map"].items() if name.startswith(prefix)}): | |
| with fs.open(f"{repo}/{file}", "rb", revision=revision) as handle: | |
| header_size = struct.unpack("<Q", handle.read(8))[0] | |
| header = json.loads(handle.read(header_size)) | |
| for name, meta in header.items(): | |
| if name.startswith(prefix): | |
| start, end = meta["data_offsets"] | |
| handle.seek(8 + header_size + start) | |
| data = bytearray(handle.read(end - start)) | |
| tensors[name] = torch.frombuffer(data, dtype=dtypes[meta["dtype"]]).reshape(meta["shape"]) | |
| return tensors | |
| def load_mtp_head(model, repo: str = MTP_REPO, revision: str = MTP_REVISION) -> MTPHead: | |
| """Qwen3.5-9B's MTP head (transformers drops it), used as the prompt upsampler's draft model.""" | |
| import copy | |
| config = copy.deepcopy(model.config.get_text_config()) | |
| config.num_hidden_layers, config.layer_types = 1, ["full_attention"] | |
| state = {name[len("mtp."):]: tensor for name, tensor in read_hub_tensors(repo, revision, "mtp.").items()} | |
| head = MTPHead(config) | |
| head.load_state_dict(state) | |
| return head.to(model.device, model.dtype).eval() | |
| GDN_VERIFY = {"torch_reference": False} | |
| def _gdn_verify(module, hidden_states, cache_params): | |
| """Qwen3_5GatedDeltaNet over a few cached tokens, one token at a time, keeping the conv and recurrent states after | |
| every token in `module._spec_snaps` so a rejected draft rolls back by index.""" | |
| from torch.nn import functional as F | |
| from transformers.models.qwen3_5 import modeling_qwen3_5 as qm | |
| batch, length, _ = hidden_states.shape | |
| layer = cache_params.layers[module.layer_idx] | |
| conv_state, recurrent_state = layer.conv_states[0], layer.recurrent_states[0] | |
| conv_snaps, recurrent_snaps = module._spec_snaps | |
| mixed = module.in_proj_qkv(hidden_states).transpose(1, 2) | |
| z = module.in_proj_z(hidden_states).reshape(batch, length, -1, module.head_v_dim) | |
| b, a = module.in_proj_b(hidden_states), module.in_proj_a(hidden_states) | |
| convolved = [] | |
| for j in range(length): | |
| convolved.append(qm.causal_conv1d_update(mixed[:, :, j : j + 1], conv_state, module.conv1d.weight.squeeze(1), | |
| module.conv1d.bias, module.activation)) | |
| conv_snaps[j].copy_(conv_state) | |
| mixed = torch.cat(convolved, dim=2).transpose(1, 2) | |
| query, key, value = torch.split(mixed, [module.key_dim, module.key_dim, module.value_dim], dim=-1) | |
| query = query.reshape(batch, length, -1, module.head_k_dim) | |
| key = key.reshape(batch, length, -1, module.head_k_dim) | |
| value = value.reshape(batch, length, -1, module.head_v_dim) | |
| beta = b.sigmoid() | |
| g = -module.A_log.float().exp() * F.softplus(a.float() + module.dt_bias) | |
| if module.num_v_heads // module.num_k_heads > 1: | |
| query = query.repeat_interleave(module.num_v_heads // module.num_k_heads, dim=2) | |
| key = key.repeat_interleave(module.num_v_heads // module.num_k_heads, dim=2) | |
| recurrence = qm.torch_recurrent_gated_delta_rule | |
| if GDN_VERIFY["torch_reference"]: | |
| recurrence = getattr(recurrence, "__wrapped__", recurrence) | |
| outputs = [] | |
| for j in range(length): | |
| step = [t[:, j : j + 1].contiguous() for t in (query, key, value, g, beta)] | |
| out, state = recurrence(step[0], step[1], step[2], g=step[3], beta=step[4], | |
| initial_state=recurrent_state.clone(), output_final_state=True, | |
| use_qk_l2norm_in_kernel=True) | |
| recurrent_state.copy_(state) | |
| recurrent_snaps[j].copy_(recurrent_state) | |
| outputs.append(out) | |
| core = module.norm(torch.cat(outputs, dim=1).reshape(-1, module.head_v_dim), z.reshape(-1, module.head_v_dim)) | |
| return module.out_proj(core.reshape(batch, length, -1)) | |
| def _install_gdn_verify(module) -> None: | |
| """Route `module`'s forward through `_gdn_verify` while `module._spec_verify` is set. Patched on the instance: | |
| `set_use_kernels` installs instance-level forwards, which a class-level patch would not reach.""" | |
| if getattr(module, "_spec_patched", False): | |
| return | |
| original = module.forward | |
| def forward(hidden_states, cache_params=None, attention_mask=None, **kwargs): | |
| if module._spec_verify: | |
| return _gdn_verify(module, hidden_states, cache_params) | |
| return original(hidden_states, cache_params=cache_params, attention_mask=attention_mask, **kwargs) | |
| module.forward, module._spec_verify, module._spec_patched = forward, False, True | |
| class MTPGreedyGenerate: | |
| """Greedy decoding of one sequence with one MTP draft token per step, each step one CUDA graph. Lossless: every | |
| emitted token is the main model's own greedy choice; an accepted draft adds a second token to the step. | |
| A step feeds `[t, draft]` to the main model (its Gated DeltaNet layers keep their states after each token, the | |
| full-attention layers write both K/V rows), takes the greedy predictions p0, p1, accepts the draft if it equals | |
| p0, rolls the states back to the accepted prefix and emits `draft, p1` or `p0`. The MTP head then runs on the two | |
| (hidden, next token) pairs of the step and drafts from the accepted one. The end token is checked every | |
| `check_every` steps. | |
| """ | |
| def __init__(self, model, mtp: MTPHead, max_cache_len: int = 12288, max_new_tokens: int = 4096, | |
| check_every: int = 8): | |
| from transformers import StaticCache | |
| self.model, self.mtp, self.generate_eager = model, mtp, model.generate | |
| self.text_model = model.model.language_model | |
| self.max_cache_len, self.check_every = max_cache_len, check_every | |
| device = model.device | |
| self.cache = StaticCache(config=model.config, max_cache_len=max_cache_len) | |
| self.mtp_cache = StaticCache(config=mtp.config, max_cache_len=max_cache_len) | |
| self.gdn = [layer.linear_attn for layer in self.text_model.layers if layer.block_type == "linear_attention"] | |
| for module in self.gdn: | |
| _install_gdn_verify(module) | |
| long = dict(dtype=torch.long, device=device) | |
| self.tok, self.draft, self.pos = (torch.zeros((1,), **long) for _ in range(3)) | |
| self.offset = torch.zeros((4, 1, 1), **long) | |
| self.keys = torch.arange(max_cache_len, **long) | |
| self.pair = torch.arange(2, **long) | |
| self.out = torch.zeros((max_new_tokens + 2 * check_every + 2,), **long) | |
| self.count, self.accepted = torch.zeros((1,), **long), torch.zeros((1,), **long) | |
| self.max_new_tokens = max_new_tokens | |
| self.graph, self.stats = None, {} | |
| def _set_verify(self, on: bool): | |
| for module in self.gdn: | |
| module._spec_verify = on | |
| def _static_lengths(self): | |
| from transformers.cache_utils import StaticLayer | |
| return [layer.cumulative_length for cache in (self.cache, self.mtp_cache) for layer in cache.layers | |
| if isinstance(layer, StaticLayer)] | |
| def _step(self): | |
| positions = self.pos + self.pair | |
| position_ids = positions.view(1, 1, 2) + self.offset | |
| mask = (self.keys.view(1, -1) <= positions.view(-1, 1)).view(1, 1, 2, -1) | |
| masks = {"full_attention": mask, "linear_attention": None} | |
| tokens = torch.cat([self.tok, self.draft]).view(1, 2) | |
| hidden = self.text_model(inputs_embeds=self.text_model.embed_tokens(tokens), position_ids=position_ids, | |
| attention_mask=masks, past_key_values=self.cache, use_cache=True).last_hidden_state | |
| preds = self.model.lm_head(hidden).argmax(-1).view(2) | |
| accept = (preds[0] == self.draft[0]).long().view(1) | |
| for module in self.gdn: | |
| layer = self.cache.layers[module.layer_idx] | |
| conv_snaps, recurrent_snaps = module._spec_snaps | |
| layer.conv_states[0].copy_(conv_snaps.index_select(0, accept)[0]) | |
| layer.recurrent_states[0].copy_(recurrent_snaps.index_select(0, accept)[0]) | |
| emitted = torch.stack([torch.where(accept.bool(), self.draft, preds[:1])[0], preds[1]]) | |
| self.out.index_copy_(0, self.count + self.pair, emitted) | |
| cos_sin = self.text_model.rotary_emb(hidden, position_ids[1:]) | |
| mtp_hidden = self.mtp(self.text_model.embed_tokens(emitted.view(1, 2)), hidden, cos_sin, position_ids[0], mask, | |
| self.mtp_cache) | |
| draft = self.model.lm_head(mtp_hidden.index_select(1, accept)).argmax(-1).view(1) | |
| new_length = (self.pos + 1 + accept).view(()) | |
| for length in self._static_lengths(): | |
| length.copy_(new_length) | |
| self.tok.copy_(torch.where(accept.bool(), preds[1:], preds[:1])) | |
| self.draft.copy_(draft) | |
| self.pos.add_(1 + accept) | |
| self.count.add_(1 + accept) | |
| self.accepted.add_(accept) | |
| def _prefill(self, input_ids, kwargs): | |
| self.cache.reset() | |
| self.mtp_cache.reset() | |
| self._set_verify(False) | |
| self.model.model.rope_deltas = None | |
| seen = {} | |
| hook = self.text_model.register_forward_pre_hook( | |
| lambda module, args, kw: seen.update(position_ids=kw.get("position_ids")), with_kwargs=True) | |
| hidden = self.model.model(input_ids=input_ids, past_key_values=self.cache, use_cache=True, | |
| **kwargs).last_hidden_state | |
| hook.remove() | |
| first = self.model.lm_head(hidden[:, -1]).argmax(-1) | |
| length = input_ids.shape[1] | |
| steps = torch.arange(length, device=input_ids.device) | |
| rope = seen.get("position_ids") | |
| rope = steps.view(1, 1, -1).expand(3, 1, -1) if rope is None else rope[-3:] | |
| mask = (self.keys.view(1, -1) <= steps.view(-1, 1)).view(1, 1, length, -1) | |
| next_ids = torch.cat([input_ids[:, 1:], first.view(1, 1)], dim=1) | |
| mtp_hidden = self.mtp(self.text_model.embed_tokens(next_ids), hidden, self.text_model.rotary_emb(hidden, rope), | |
| steps.view(1, -1), mask, self.mtp_cache) | |
| draft = self.model.lm_head(mtp_hidden[:, -1]).argmax(-1) | |
| rope_deltas = self.model.model.rope_deltas | |
| self.offset.zero_() | |
| if rope_deltas is not None: | |
| self.offset[1:] = rope_deltas.view(-1)[0] | |
| self.out[0] = first[0] | |
| self.count.fill_(1) | |
| self.accepted.zero_() | |
| self.tok.copy_(first) | |
| self.draft.copy_(draft) | |
| self.pos.fill_(length) | |
| self._set_verify(True) | |
| def _capture(self): | |
| for module in self.gdn: | |
| layer = self.cache.layers[module.layer_idx] | |
| module._spec_snaps = (torch.empty((2, *layer.conv_states[0].shape), dtype=layer.conv_states[0].dtype, | |
| device=layer.conv_states[0].device), | |
| torch.empty((2, *layer.recurrent_states[0].shape), | |
| dtype=layer.recurrent_states[0].dtype, | |
| device=layer.recurrent_states[0].device)) | |
| stream = torch.cuda.Stream() | |
| stream.wait_stream(torch.cuda.current_stream()) | |
| with torch.cuda.stream(stream): | |
| for _ in range(3): | |
| self._step() | |
| torch.cuda.current_stream().wait_stream(stream) | |
| self.graph = torch.cuda.CUDAGraph() | |
| with torch.cuda.graph(self.graph): | |
| self._step() | |
| def generate(self, input_ids, max_new_tokens: int, eos_token_id: int, **kwargs): | |
| kwargs.pop("do_sample", None) | |
| kwargs.pop("pad_token_id", None) | |
| margin = 2 * self.check_every + 2 | |
| self._set_verify(False) | |
| if (input_ids.shape[0] != 1 or max_new_tokens > self.max_new_tokens | |
| or input_ids.shape[1] + max_new_tokens + margin > self.max_cache_len): | |
| return self.generate_eager(input_ids=input_ids, max_new_tokens=max_new_tokens, do_sample=False, | |
| eos_token_id=eos_token_id, **kwargs) | |
| if self.graph is None: | |
| self._prefill(input_ids, kwargs) | |
| self._capture() | |
| self._prefill(input_ids, kwargs) | |
| steps, count = 0, 1 | |
| while count < max_new_tokens and not bool((self.out[:count] == eos_token_id).any()): | |
| for _ in range(self.check_every): | |
| self.graph.replay() | |
| steps += self.check_every | |
| count = int(self.count) | |
| tokens = self.out[: min(count, max_new_tokens)] | |
| ends = (tokens == eos_token_id).nonzero() | |
| if ends.numel(): | |
| tokens = tokens[: int(ends[0, 0]) + 1] | |
| self._set_verify(False) | |
| self.stats = {"steps": steps, "tokens": int(tokens.numel()), "accepted": int(self.accepted), | |
| "acceptance": round(int(self.accepted) / max(steps, 1), 3)} | |
| return torch.cat([input_ids, tokens[None].to(input_ids.device)], dim=1) | |
| # Decode attention --------------------------------------------------------------------------------------------------- | |
| def install_decode_attention(*configs) -> None: | |
| """Full attention for a few query rows over a long static cache as two grouped batched GEMVs (K/V read once per | |
| KV head, no repeat). SDPA's memory-efficient kernel parallelizes a single query poorly. Longer queries (prefill) | |
| keep SDPA.""" | |
| from transformers import AttentionInterface | |
| from transformers.integrations.sdpa_attention import sdpa_attention_forward | |
| def attention(module, query, key, value, attention_mask, dropout=0.0, scaling=None, **kwargs): | |
| batch, heads, length, dim = query.shape | |
| if length > 16 or attention_mask is None or attention_mask.dtype != torch.bool: | |
| return sdpa_attention_forward(module, query, key, value, attention_mask, dropout=dropout, | |
| scaling=scaling, **kwargs) | |
| kv_heads, keys = key.shape[1], key.shape[2] | |
| groups = heads // kv_heads | |
| q = query.reshape(batch, kv_heads, groups * length, dim) | |
| scores = torch.matmul(q, key.transpose(-1, -2)).float().view(batch, kv_heads, groups, length, keys) | |
| scores = scores * (scaling if scaling is not None else dim**-0.5) | |
| scores = scores.masked_fill(~attention_mask[..., :keys].view(1, 1, 1, length, keys), float("-inf")) | |
| probs = scores.softmax(-1).to(value.dtype).view(batch, kv_heads, groups * length, keys) | |
| out = torch.matmul(probs, value).view(batch, heads, length, -1) | |
| return out.transpose(1, 2).contiguous(), None | |
| AttentionInterface.register("p_image_oss_decode", attention) | |
| for config in configs: | |
| config._attn_implementation = "p_image_oss_decode" | |
| # Batched VAE tile decode -------------------------------------------------------------------------------------------- | |
| def batch_vae_tiles(vae, max_tiles: int = 96) -> None: | |
| """Decode the spatial tiles of `AutoencoderKLPH3Image` in batches of up to `max_tiles` instead of one decoder call | |
| per tile. The tile layout, the per-tile decoder math and the stitching are unchanged.""" | |
| def decode_clip(z: torch.Tensor) -> torch.Tensor: | |
| if not vae.use_tiling: | |
| return vae.decoder(vae.post_quant_conv(z)) | |
| ratio = vae.spatial_compression_ratio | |
| y_indices, y_lengths, y_overlaps = vae._split_tiles( | |
| z.shape[-2] * ratio, vae.tile_sample_min_height, vae.tile_sample_min_overlap_height) | |
| x_indices, x_lengths, x_overlaps = vae._split_tiles( | |
| z.shape[-1] * ratio, vae.tile_sample_min_width, vae.tile_sample_min_overlap_width) | |
| tiles = {} | |
| for i, (i_pos, i_len) in enumerate(zip(y_indices, y_lengths)): | |
| for j, (j_pos, j_len) in enumerate(zip(x_indices, x_lengths)): | |
| tiles[(i, j)] = z[..., i_pos // ratio: i_pos // ratio + i_len // ratio, | |
| j_pos // ratio: j_pos // ratio + j_len // ratio] | |
| decoded = {} | |
| by_shape: dict[tuple, list] = {} | |
| for key, tile in tiles.items(): | |
| by_shape.setdefault(tuple(tile.shape), []).append(key) | |
| for keys in by_shape.values(): | |
| for start in range(0, len(keys), max_tiles): | |
| chunk = keys[start: start + max_tiles] | |
| out = vae.decoder(vae.post_quant_conv(torch.cat([tiles[k] for k in chunk], dim=0))) | |
| for index, key in enumerate(chunk): | |
| decoded[key] = out[index: index + 1] | |
| rows = [[decoded[(i, j)] for j in range(len(x_indices))] for i in range(len(y_indices))] | |
| return vae._stitch_tiles(rows, y_overlaps, x_overlaps) | |
| vae._decode_clip = decode_clip | |
| # The Space's pipeline ------------------------------------------------------------------------------------------------ | |
| def load_slim_pipeline(path: str, adaln_calls: list, device: str = "cuda", log=print): | |
| """p-image-oss from a local snapshot, slimmed for a 48 GB slice. Components are loaded and slimmed one at a time | |
| (bf16 transformer peak ~66 GB host RAM) and moved to `device`. Returns the pipeline and the upsampler's MTP head.""" | |
| import gc | |
| from diffusers import ModularPipeline | |
| start = time.time() | |
| pipe = ModularPipeline.from_pretrained(path) | |
| pipe.load_components(names=["transformer"], dtype=torch.bfloat16) | |
| install_adaln_tables(pipe.transformer, [torch.tensor(call) for call in adaln_calls]) | |
| quantize_fp8(pipe.transformer.transformer_blocks) | |
| fuse_dit_blocks(pipe.transformer) | |
| pipe.transformer.to(device) | |
| gc.collect() | |
| log(f"[{time.time() - start:6.1f}s] transformer ready") | |
| pipe.load_components(names=["prompt_upsampler", "prompt_upsampler_processor"], dtype=torch.bfloat16) | |
| upsampler = pipe.prompt_upsampler | |
| quantize_fp8(upsampler.model.language_model) | |
| upsampler.lm_head = FP8Linear(upsampler.lm_head) | |
| mtp = load_mtp_head(upsampler) | |
| quantize_fp8(mtp) | |
| install_decode_attention(upsampler.model.language_model.config, mtp.config) | |
| upsampler.to(device) | |
| mtp.to(device) | |
| gc.collect() | |
| log(f"[{time.time() - start:6.1f}s] prompt upsampler ready") | |
| pipe.load_components(dtype=torch.bfloat16) | |
| quantize_fp8(pipe.text_encoder.model.language_model) | |
| quantize_fp8(pipe.vae.decoder.transformer_blocks) | |
| batch_vae_tiles(pipe.vae) | |
| pipe.to(device) | |
| gc.collect() | |
| log(f"[{time.time() - start:6.1f}s] pipeline ready") | |
| return pipe, mtp | |
| # Fused DiT block ---------------------------------------------------------------------------------------------------- | |
| # | |
| # One MiniMaxH3TransformerBlock in five row kernels around the FP8 GEMMs and attention. Every producer of a GEMM input | |
| # holds the whole row, so it also computes the row's FP8 scale and writes e4m3 directly (no separate quant pass): | |
| # norm1 + AdaLN modulation -> FP8 | QKV GEMM | q/k RMSNorm + RoPE | attention | out GEMM | |
| # gated residual + norm2 + modulation -> FP8 | FF-in GEMM | SwiGLU -> FP8 | FF-out GEMM | gated residual | |
| # Math runs in fp32 and rounds once per output (eager rounds to bf16 after every op). | |
| def _rms_mod_fp8_kernel(x_ptr, w_ptr, scale_ptr, shift_ptr, idx_ptr, q_ptr, s_ptr, n_cols, stride_x, stride_tab, eps, | |
| BLOCK: tl.constexpr): | |
| row = tl.program_id(0).to(tl.int64) | |
| cols = tl.arange(0, BLOCK) | |
| mask = cols < n_cols | |
| x = tl.load(x_ptr + row * stride_x + cols, mask=mask, other=0.0).to(tl.float32) | |
| w = tl.load(w_ptr + cols, mask=mask, other=0.0).to(tl.float32) | |
| tab = tl.load(idx_ptr + row).to(tl.int64) * stride_tab + cols | |
| sc = tl.load(scale_ptr + tab, mask=mask, other=0.0).to(tl.float32) | |
| sh = tl.load(shift_ptr + tab, mask=mask, other=0.0).to(tl.float32) | |
| y = x * tl.math.rsqrt(tl.sum(x * x, axis=0) / n_cols + eps) * w * (1.0 + sc) + sh | |
| amax = tl.maximum(tl.max(tl.where(mask, tl.abs(y), 0.0), axis=0), 1e-12) | |
| tl.store(q_ptr + row * n_cols + cols, (y * (448.0 / amax)).to(tl.float8e4nv), mask=mask) | |
| tl.store(s_ptr + row, amax / 448.0) | |
| def _res_gate_rms_mod_fp8_kernel(h_ptr, b_ptr, gate_ptr, w_ptr, scale_ptr, shift_ptr, idx_ptr, out_ptr, q_ptr, s_ptr, | |
| n_cols, stride_h, stride_b, stride_tab, eps, BLOCK: tl.constexpr): | |
| row = tl.program_id(0).to(tl.int64) | |
| cols = tl.arange(0, BLOCK) | |
| mask = cols < n_cols | |
| tab = tl.load(idx_ptr + row).to(tl.int64) * stride_tab + cols | |
| h = tl.load(h_ptr + row * stride_h + cols, mask=mask, other=0.0).to(tl.float32) | |
| b = tl.load(b_ptr + row * stride_b + cols, mask=mask, other=0.0).to(tl.float32) | |
| g = tl.load(gate_ptr + tab, mask=mask, other=0.0).to(tl.float32) | |
| h = (h + g * b).to(out_ptr.dtype.element_ty) | |
| tl.store(out_ptr + row * n_cols + cols, h, mask=mask) | |
| h = h.to(tl.float32) | |
| w = tl.load(w_ptr + cols, mask=mask, other=0.0).to(tl.float32) | |
| sc = tl.load(scale_ptr + tab, mask=mask, other=0.0).to(tl.float32) | |
| sh = tl.load(shift_ptr + tab, mask=mask, other=0.0).to(tl.float32) | |
| y = h * tl.math.rsqrt(tl.sum(h * h, axis=0) / n_cols + eps) * w * (1.0 + sc) + sh | |
| amax = tl.maximum(tl.max(tl.where(mask, tl.abs(y), 0.0), axis=0), 1e-12) | |
| tl.store(q_ptr + row * n_cols + cols, (y * (448.0 / amax)).to(tl.float8e4nv), mask=mask) | |
| tl.store(s_ptr + row, amax / 448.0) | |
| def _res_gate_kernel(h_ptr, b_ptr, gate_ptr, idx_ptr, out_ptr, n_cols, stride_h, stride_b, stride_tab, | |
| BLOCK: tl.constexpr): | |
| row = tl.program_id(0).to(tl.int64) | |
| cols = tl.arange(0, BLOCK) | |
| mask = cols < n_cols | |
| tab = tl.load(idx_ptr + row).to(tl.int64) * stride_tab + cols | |
| h = tl.load(h_ptr + row * stride_h + cols, mask=mask, other=0.0).to(tl.float32) | |
| b = tl.load(b_ptr + row * stride_b + cols, mask=mask, other=0.0).to(tl.float32) | |
| g = tl.load(gate_ptr + tab, mask=mask, other=0.0).to(tl.float32) | |
| tl.store(out_ptr + row * n_cols + cols, (h + g * b).to(out_ptr.dtype.element_ty), mask=mask) | |
| def _swiglu_fp8_kernel(x_ptr, q_ptr, s_ptr, n_cols, stride_x, BLOCK: tl.constexpr): | |
| row = tl.program_id(0).to(tl.int64) | |
| cols = tl.arange(0, BLOCK) | |
| mask = cols < n_cols | |
| v = tl.load(x_ptr + row * stride_x + cols, mask=mask, other=0.0).to(tl.float32) | |
| g = tl.load(x_ptr + row * stride_x + n_cols + cols, mask=mask, other=0.0).to(tl.float32) | |
| y = v * g * tl.sigmoid(g) | |
| amax = tl.maximum(tl.max(tl.where(mask, tl.abs(y), 0.0), axis=0), 1e-12) | |
| tl.store(q_ptr + row * n_cols + cols, (y * (448.0 / amax)).to(tl.float8e4nv), mask=mask) | |
| tl.store(s_ptr + row, amax / 448.0) | |
| def _qk_norm_rope_kernel(qkv_ptr, wq_ptr, wk_ptr, cos_ptr, sin_ptr, q_ptr, k_ptr, stride_qkv, stride_rope, heads, | |
| eps, D: tl.constexpr, ROT: tl.constexpr): | |
| row = tl.program_id(0).to(tl.int64) | |
| which = tl.program_id(1) # [0, heads) query heads, [heads, 2 * heads) key heads | |
| is_k = which >= heads | |
| head = which - heads * is_k.to(tl.int32) | |
| c = tl.arange(0, D) | |
| half = ROT // 2 | |
| partner = tl.where(c < half, c + half, tl.where(c < ROT, c - half, c)) | |
| base = qkv_ptr + row * stride_qkv + (is_k.to(tl.int64) * heads + head) * D | |
| x = tl.load(base + c).to(tl.float32) | |
| xp = tl.load(base + partner).to(tl.float32) | |
| w = tl.where(is_k, tl.load(wk_ptr + c), tl.load(wq_ptr + c)).to(tl.float32) | |
| wp = tl.where(is_k, tl.load(wk_ptr + partner), tl.load(wq_ptr + partner)).to(tl.float32) | |
| r = tl.math.rsqrt(tl.sum(x * x, axis=0) / D + eps) | |
| xn, xpn = x * r * w, xp * r * wp | |
| rot = c < ROT | |
| cos = tl.load(cos_ptr + row * stride_rope + c, mask=rot, other=1.0) | |
| sin = tl.load(sin_ptr + row * stride_rope + c, mask=rot, other=0.0) | |
| y = (xn * cos + tl.where(c < half, -xpn, xpn) * sin).to(q_ptr.dtype.element_ty) | |
| offset = (row * heads + head) * D + c | |
| tl.store(q_ptr + offset, y, mask=(c < D) & (which < heads)) | |
| tl.store(k_ptr + offset, y, mask=(c < D) & (which >= heads)) | |
| def _row_launch(kernel, rows, cols, *args, **kwargs): | |
| block = triton.next_power_of_2(cols) | |
| kernel[(rows,)](*args, BLOCK=block, num_warps=16 if block >= 16384 else 8, **kwargs) | |
| def _fp8_gemm(q, s, linear: FP8Linear): | |
| return torch._scaled_mm(q, linear.weight.t(), scale_a=s, scale_b=linear.weight_scale, bias=linear.bias, | |
| out_dtype=linear.out_dtype, use_fast_accum=linear.fast_accum) | |
| def _fused_block_forward(block, hidden_states, temb, adaln_indices, rotary_emb, attention_mask=None): | |
| from diffusers.models.attention_dispatch import dispatch_attention_fn | |
| shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = block.adaln_proj(temb) | |
| batch, rows, cols = hidden_states.shape | |
| h = hidden_states.reshape(rows, cols) | |
| idx, stride_tab, device = adaln_indices, shift_msa.stride(0), h.device | |
| attn = block.attn | |
| heads, dim = attn.heads, attn.head_dim | |
| q8 = torch.empty((rows, cols), device=device, dtype=FP8) | |
| s = torch.empty((rows, 1), device=device, dtype=torch.float32) | |
| _row_launch(_rms_mod_fp8_kernel, rows, cols, h, block.norm1.weight, scale_msa, shift_msa, idx, q8, s, cols, | |
| h.stride(0), stride_tab, block.norm1.eps) | |
| qkv = _fp8_gemm(q8, s, block._fused_qkv) | |
| q = torch.empty((rows, heads, dim), device=device, dtype=h.dtype) | |
| k = torch.empty_like(q) | |
| cos, sin = rotary_emb | |
| _qk_norm_rope_kernel[(rows, 2 * heads)](qkv, attn.norm_q.weight, attn.norm_k.weight, cos, sin, q, k, qkv.stride(0), | |
| cos.stride(0), heads, attn.norm_q.eps, D=dim, ROT=cos.shape[-1], | |
| num_warps=2) | |
| v = qkv[:, 2 * heads * dim:].view(rows, heads, dim) | |
| o = dispatch_attention_fn(q[None], k[None], v[None], attn_mask=None, dropout_p=0.0, is_causal=False, | |
| backend=attn.processor._attention_backend) | |
| attn_out = attn.to_out[0](o.reshape(rows, heads * dim)) | |
| h2 = torch.empty_like(h) | |
| _row_launch(_res_gate_rms_mod_fp8_kernel, rows, cols, h, attn_out, gate_msa, block.norm2.weight, scale_mlp, | |
| shift_mlp, idx, h2, q8, s, cols, h.stride(0), attn_out.stride(0), stride_tab, block.norm2.eps) | |
| proj = _fp8_gemm(q8, s, block.ff.net[0].proj) | |
| inner = proj.shape[1] // 2 | |
| f8 = torch.empty((rows, inner), device=device, dtype=FP8) | |
| _row_launch(_swiglu_fp8_kernel, rows, inner, proj, f8, s, inner, proj.stride(0)) | |
| ff_out = _fp8_gemm(f8, s, block.ff.net[2]) | |
| out = torch.empty_like(h) | |
| _row_launch(_res_gate_kernel, rows, cols, h2, ff_out, gate_mlp, idx, out, cols, h2.stride(0), ff_out.stride(0), | |
| stride_tab) | |
| return out.view(batch, rows, cols) | |
| def fuse_dit_blocks(transformer) -> None: | |
| """Fused forwards for every transformer block. Needs `install_adaln_tables` and `quantize_fp8` first (and q, k, v | |
| as separate FP8Linear, which are concatenated into one QKV GEMM here).""" | |
| import types | |
| for block in transformer.transformer_blocks: | |
| attn = block.attn | |
| parts = [attn.to_q, attn.to_k, attn.to_v] | |
| fused = FP8Linear.__new__(FP8Linear) | |
| nn.Module.__init__(fused) | |
| fused.register_buffer("weight", torch.cat([p.weight for p in parts])) | |
| fused.register_buffer("weight_scale", torch.cat([p.weight_scale for p in parts], dim=1)) | |
| fused.bias, fused.out_dtype, fused.fast_accum = None, parts[0].out_dtype, parts[0].fast_accum | |
| fused.in_features, fused.out_features = parts[0].in_features, sum(p.out_features for p in parts) | |
| block._fused_qkv = fused | |
| del attn.to_q, attn.to_k, attn.to_v | |
| block.forward = types.MethodType(_fused_block_forward, block) | |
| _free_cache() | |