OpenOneRec's picture
Release KSA-1.9B-base weights and inference runtime
6c91dd5 verified
Raw History Blame Contribute Delete
10.5 kB
import cutlass
from cutlass import cute
from flash_attn.cute.seqlen_info import SeqlenInfoQK
from flash_attn.cute.utils import scalar_to_ssa, ssa_to_scalar
@cute.jit
def summary_block_sparse_attn_mask_func(
batch_idx_ssa: cute.TensorSSA,
head_idx_ssa: cute.TensorSSA,
q_idx_ssa: cute.TensorSSA,
kv_idx_ssa: cute.TensorSSA,
seqlen_info: SeqlenInfoQK,
aux_tensors,
block_size_tokens: cutlass.Constexpr[int],
local_window_tokens: cutlass.Constexpr[int],
local_alignment_tokens: cutlass.Constexpr[int] = 0,
):
"""Per-query raw-token mask selected through summary-block indices.
``aux_tensors[0]`` is a packed uint8 tensor with layout
``[batch, query, ceil(raw_block_count / 8)]``. Bit ``j`` is set exactly
when the query selected summary/raw block ``j``. Packing the dynamic
``[B,Q,K]`` indices before launching attention avoids materialising the
quadratic ``[B,Q,raw_block_count]`` boolean route.
The CuTe scheduler still works in 128-token tiles. A scheduled tile is
only a conservative union; this function is evaluated for every score and
preserves exact per-query, per-``block_size_tokens`` semantics.
"""
del head_idx_ssa, seqlen_info
raw_block_idx_ssa = kv_idx_ssa // block_size_tokens
route_byte_idx_ssa = raw_block_idx_ssa // 8
route_bit_idx_ssa = raw_block_idx_ssa % 8
packed_route = aux_tensors[0]
route_byte = packed_route[
ssa_to_scalar(batch_idx_ssa),
ssa_to_scalar(q_idx_ssa),
ssa_to_scalar(route_byte_idx_ssa),
]
route_bit = 1 << ssa_to_scalar(route_bit_idx_ssa)
selected = (route_byte & route_bit) != 0
selected_ssa = scalar_to_ssa(selected, cutlass.Boolean)
causal = kv_idx_ssa <= q_idx_ssa
raw_local_start = q_idx_ssa - local_window_tokens + 1
clamped_local_start = cute.where(
raw_local_start > 0,
raw_local_start,
cute.full_like(raw_local_start, 0),
)
# Preserve the original summary-block API: callers that do not provide a
# separate local alignment retain ``block_size_tokens`` alignment. Raw
# token routing explicitly supplies eight here while its route span is one.
aligned_local_start = (
clamped_local_start // block_size_tokens
) * block_size_tokens
if local_alignment_tokens > 0:
aligned_local_start = (
clamped_local_start // local_alignment_tokens
) * local_alignment_tokens
local = kv_idx_ssa >= aligned_local_start
return causal & (local | selected_ssa)
@cute.jit
def hierarchical_summary_attn_score_mod(
score_ssa: cute.TensorSSA,
batch_idx_ssa: cute.TensorSSA,
head_idx_ssa: cute.TensorSSA,
q_idx: cute.TensorSSA,
kv_idx: cute.TensorSSA,
seqlen_info: SeqlenInfoQK,
aux_tensors,
meta_length: cutlass.Constexpr[int],
meta_logit_bias: cutlass.Constexpr[float],
):
"""Add log(G) to each physical meta entry before softmax."""
return cute.where(
kv_idx < meta_length,
score_ssa + meta_logit_bias,
score_ssa,
)
@cute.jit
def hierarchical_summary_attn_mask_func(
batch_idx_ssa: cute.TensorSSA,
head_idx_ssa: cute.TensorSSA,
q_idx_ssa: cute.TensorSSA,
kv_idx_ssa: cute.TensorSSA,
seqlen_info: SeqlenInfoQK,
aux_tensors,
chunk_size: cutlass.Constexpr[int],
sliding_chunk_num: cutlass.Constexpr[int],
group_size: cutlass.Constexpr[int],
meta_length: cutlass.Constexpr[int],
swap_indices: cutlass.Constexpr[bool],
):
"""Compressed-KSA mask for K/V laid out as ``[meta, base]``."""
if swap_indices:
q_idx_ssa, kv_idx_ssa = kv_idx_ssa, q_idx_ssa
block_size = chunk_size + 1
q_chunk_id = q_idx_ssa // block_size
q_offset = q_idx_ssa % block_size
is_q_text = q_offset < chunk_size
raw_window_start = q_chunk_id - sliding_chunk_num
window_start = cute.where(
raw_window_start > 0,
raw_window_start,
cute.full_like(raw_window_start, 0),
)
complete_group_count = window_start // group_size
complete_end = complete_group_count * group_size
is_meta = kv_idx_ssa < meta_length
base_idx = kv_idx_ssa - meta_length
kv_chunk_id = base_idx // block_size
kv_offset = base_idx % block_size
is_kv_text = kv_offset < chunk_size
is_kv_summary = kv_offset == chunk_size
meta_group_id = kv_idx_ssa
valid_meta = is_q_text & is_meta & (meta_group_id < complete_group_count)
valid_text = (
is_q_text & (is_meta ^ 1) & is_kv_text
& (((kv_chunk_id >= window_start) & (kv_chunk_id < q_chunk_id))
| ((kv_chunk_id == q_chunk_id) & (base_idx <= q_idx_ssa)))
)
valid_residual = (
is_q_text & (is_meta ^ 1) & is_kv_summary
& (kv_chunk_id >= complete_end) & (kv_chunk_id < window_start)
)
valid_summary = (
(is_q_text ^ 1) & (is_meta ^ 1) & (kv_chunk_id == q_chunk_id)
& (is_kv_text | (is_kv_summary & (base_idx <= q_idx_ssa)))
)
return valid_meta | valid_text | valid_residual | valid_summary
@cute.jit
def summary_attn_mask_func(
batch_idx_ssa: cute.TensorSSA,
head_idx_ssa: cute.TensorSSA,
q_idx_ssa: cute.TensorSSA,
kv_idx_ssa: cute.TensorSSA,
seqlen_info: SeqlenInfoQK,
aux_tensors,
chunk_size: cutlass.Constexpr[int],
summary_num: cutlass.Constexpr[int],
sliding_chunk_num: cutlass.Constexpr[int],
prefix_summary_length: cutlass.Constexpr[int],
swap_indices: cutlass.Constexpr[bool],
):
if swap_indices:
q_idx_ssa, kv_idx_ssa = kv_idx_ssa, q_idx_ssa
q_chunk_id = q_idx_ssa // (chunk_size + summary_num)
kv_chunk_id = kv_idx_ssa // (chunk_size + summary_num)
is_sliding_chunk = (q_chunk_id - kv_chunk_id) <= sliding_chunk_num
is_q_summary = (q_idx_ssa % (chunk_size + summary_num)) >= chunk_size
is_kv_summary = (kv_idx_ssa % (chunk_size + summary_num)) >= chunk_size
is_causal = q_idx_ssa >= kv_idx_ssa
valid_kv_q_summary = is_q_summary & (kv_chunk_id == q_chunk_id) & is_causal
valid_sliding_q_text = is_sliding_chunk & (is_kv_summary ^ 1)
valid_summary_q_text = is_kv_summary & ((q_chunk_id - kv_chunk_id) > sliding_chunk_num)
valid_kv_q_text = (is_q_summary ^ 1) & is_causal & (valid_sliding_q_text | valid_summary_q_text)
return valid_kv_q_summary | valid_kv_q_text
@cute.jit
def summary_attn_contiguous_mask_func(
batch_idx_ssa: cute.TensorSSA,
head_idx_ssa: cute.TensorSSA,
q_idx_ssa: cute.TensorSSA,
kv_idx_ssa: cute.TensorSSA,
seqlen_info: SeqlenInfoQK,
aux_tensors,
chunk_size: cutlass.Constexpr[int],
summary_num: cutlass.Constexpr[int],
sliding_chunk_num: cutlass.Constexpr[int],
prefix_summary_length: cutlass.Constexpr[int],
swap_indices: cutlass.Constexpr[bool],
):
if swap_indices:
q_idx_ssa, kv_idx_ssa = kv_idx_ssa, q_idx_ssa
q_chunk_id = q_idx_ssa // (chunk_size + summary_num)
kv_chunk_id = (kv_idx_ssa - prefix_summary_length) // (chunk_size + summary_num)
is_sliding_chunk = (q_chunk_id - kv_chunk_id) <= sliding_chunk_num
is_q_summary = (q_idx_ssa % (chunk_size + summary_num)) >= chunk_size
is_kv_summary = ((kv_idx_ssa - prefix_summary_length) % (chunk_size + summary_num)) >= chunk_size
is_causal = q_idx_ssa >= (kv_idx_ssa - prefix_summary_length)
valid_kv_q_summary = is_q_summary & (kv_chunk_id == q_chunk_id) & is_causal
valid_sliding_q_text = is_sliding_chunk & (is_kv_summary ^ 1) & (kv_idx_ssa >= prefix_summary_length)
valid_summary_q_text = ((kv_idx_ssa // summary_num) + sliding_chunk_num) < q_chunk_id
valid_kv_q_text = (is_q_summary ^ 1) & ((is_causal & valid_sliding_q_text) | valid_summary_q_text)
return valid_kv_q_summary | valid_kv_q_text
# ---------------------------------------------------------------------------
# Sliding-chunk-only variants: text tokens never attend to old summary tokens
# ---------------------------------------------------------------------------
@cute.jit
def sliding_chunk_mask_func(
batch_idx_ssa: cute.TensorSSA,
head_idx_ssa: cute.TensorSSA,
q_idx_ssa: cute.TensorSSA,
kv_idx_ssa: cute.TensorSSA,
seqlen_info: SeqlenInfoQK,
aux_tensors,
chunk_size: cutlass.Constexpr[int],
summary_num: cutlass.Constexpr[int],
sliding_chunk_num: cutlass.Constexpr[int],
prefix_summary_length: cutlass.Constexpr[int],
swap_indices: cutlass.Constexpr[bool],
):
if swap_indices:
q_idx_ssa, kv_idx_ssa = kv_idx_ssa, q_idx_ssa
q_chunk_id = q_idx_ssa // (chunk_size + summary_num)
kv_chunk_id = kv_idx_ssa // (chunk_size + summary_num)
is_sliding_chunk = (q_chunk_id - kv_chunk_id) <= sliding_chunk_num
is_q_summary = (q_idx_ssa % (chunk_size + summary_num)) >= chunk_size
is_kv_summary = (kv_idx_ssa % (chunk_size + summary_num)) >= chunk_size
is_causal = q_idx_ssa >= kv_idx_ssa
valid_kv_q_summary = is_q_summary & (kv_chunk_id == q_chunk_id) & is_causal
valid_sliding_q_text = is_sliding_chunk & (is_kv_summary ^ 1)
valid_kv_q_text = (is_q_summary ^ 1) & is_causal & valid_sliding_q_text
return valid_kv_q_summary | valid_kv_q_text
@cute.jit
def sliding_chunk_contiguous_mask_func(
batch_idx_ssa: cute.TensorSSA,
head_idx_ssa: cute.TensorSSA,
q_idx_ssa: cute.TensorSSA,
kv_idx_ssa: cute.TensorSSA,
seqlen_info: SeqlenInfoQK,
aux_tensors,
chunk_size: cutlass.Constexpr[int],
summary_num: cutlass.Constexpr[int],
sliding_chunk_num: cutlass.Constexpr[int],
prefix_summary_length: cutlass.Constexpr[int],
swap_indices: cutlass.Constexpr[bool],
):
if swap_indices:
q_idx_ssa, kv_idx_ssa = kv_idx_ssa, q_idx_ssa
q_chunk_id = q_idx_ssa // (chunk_size + summary_num)
kv_chunk_id = (kv_idx_ssa - prefix_summary_length) // (chunk_size + summary_num)
is_sliding_chunk = (q_chunk_id - kv_chunk_id) <= sliding_chunk_num
is_q_summary = (q_idx_ssa % (chunk_size + summary_num)) >= chunk_size
is_kv_summary = ((kv_idx_ssa - prefix_summary_length) % (chunk_size + summary_num)) >= chunk_size
is_causal = q_idx_ssa >= (kv_idx_ssa - prefix_summary_length)
valid_kv_q_summary = is_q_summary & (kv_chunk_id == q_chunk_id) & is_causal
valid_sliding_q_text = is_sliding_chunk & (is_kv_summary ^ 1) & (kv_idx_ssa >= prefix_summary_length)
valid_kv_q_text = (is_q_summary ^ 1) & is_causal & valid_sliding_q_text
return valid_kv_q_summary | valid_kv_q_text