areneau's picture
Update Clairic Transformers runtime
24191a8 verified
Raw History Blame Contribute Delete
1.65 kB
from typing import Any
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
# Inference-only adaptation; see LICENSES and NOTICE.
from typing import Optional
import torch
from .chunk_delta_h import chunk_gated_delta_rule_fwd_h
from .chunk_fwd import chunk_gated_delta_rule_fwd_intra
from .chunk_o import chunk_fwd_o
from .cumsum import chunk_local_cumsum_scalar as chunk_local_cumsum
CHUNK_SIZE = 64
SUPPRESS_LEVEL = 0
def chunk_gated_delta_rule_fwd(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
scale: float,
initial_state: torch.Tensor,
initial_state_indices: torch.Tensor,
cu_seqlens: Optional[torch.LongTensor] = None,
chunk_indices: torch.LongTensor | None = None,
) -> Any:
g = chunk_local_cumsum(
g, chunk_size=CHUNK_SIZE, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices
)
# fused kkt + solve_tril + recompute_w_u
w, u, A = chunk_gated_delta_rule_fwd_intra(
k=k,
v=v,
g=g,
beta=beta,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
)
h, v_new = chunk_gated_delta_rule_fwd_h(
k=k,
w=w,
u=u,
g=g,
initial_state=initial_state,
initial_state_indices=initial_state_indices,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
)
o = chunk_fwd_o(
q=q,
k=k,
v=v_new,
h=h,
g=g,
scale=scale,
cu_seqlens=cu_seqlens,
)
if SUPPRESS_LEVEL < 3:
return g, o, A, None, h, None
elif SUPPRESS_LEVEL >= 3:
return g, o, A, w, h, v_new