Download kdflow/models/ring_attn_utils.py from codemaivanngu/ctopd: direct link, hf CLI and curl.
- Browser
- Download file 7.4 kB
-
https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/models/ring_attn_utils.py
- Command line
-
hf download hf://codemaivanngu/ctopd/kdflow/models/ring_attn_utils.py
-
curl -L -o ring_attn_utils.py https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/models/ring_attn_utils.py
7.4 kB
| import torch | |
| import torch.distributed as dist | |
| def _require_flash_attn(): | |
| """Deferred import — only needed for packing_samples=True.""" | |
| try: | |
| from flash_attn.bert_padding import index_first_axis, pad_input, rearrange, unpad_input | |
| from flash_attn.utils.distributed import all_gather as _fa_all_gather | |
| except ModuleNotFoundError as e: | |
| raise ModuleNotFoundError( | |
| "flash_attn is required for packing_samples=True. " | |
| "Install with: pip install flash_attn --no-build-isolation" | |
| ) from e | |
| return index_first_axis, pad_input, rearrange, unpad_input, _fa_all_gather | |
| RING_ATTN_GROUP = None | |
| def set_ring_attn_group(group): | |
| global RING_ATTN_GROUP | |
| RING_ATTN_GROUP = group | |
| def get_ring_attn_group(): | |
| return RING_ATTN_GROUP | |
| def reset_ring_attn_position_ids(start, end, packed_seq_lens): | |
| """ | |
| Calculate position ids for packed_seq_ids[start:end]. | |
| For example, if the packed_seq_lens is [3, 2, 4, 1], start=2, end=8, | |
| the position ids will be [2, 0, 1, 0, 1, 2]. | |
| Args: | |
| start: the start position | |
| end: the end position | |
| packed_seq_lens: the sequence lengths of packed sequences | |
| """ | |
| position_ids = torch.zeros((1, end - start), dtype=torch.long, device=torch.cuda.current_device()) | |
| offset = 0 | |
| for seqlen in packed_seq_lens: | |
| seq_start = max(offset, start) | |
| seq_end = min(offset + seqlen, end) | |
| if seq_start < seq_end: | |
| position_ids[0, seq_start - start : seq_end - start] = torch.arange(seq_start - offset, seq_end - offset) | |
| offset += seqlen | |
| if offset >= end: | |
| break | |
| return position_ids | |
| def update_ring_attn_params(cu_seqlens): | |
| """ | |
| Calculate the cu_seqlens for the current forward pass and pass the value to | |
| the substituted ring_flash_attn. | |
| Note that total_seq_len may be larger than the sum of packed_seq_lens because of padding. | |
| """ | |
| assert RING_ATTN_GROUP is not None | |
| from ring_flash_attn import update_ring_flash_attn_params | |
| update_ring_flash_attn_params(cu_seqlens, RING_ATTN_GROUP) | |
| def get_tensor_in_current_ring_attn_rank(tensors: list[torch.Tensor] | torch.Tensor, ring_attn_group, pad_id): | |
| """ | |
| Deal with padding and slice the tensor to current ring_attn_rank. | |
| Args: | |
| tensors: Each tensor shaped (batch, seqlen) or (1, total_seqs) | |
| ring_attn_group: Ring attention group | |
| pad_id: Padding id | |
| Returns: | |
| Processed tensor | |
| """ | |
| if isinstance(tensors, torch.Tensor): | |
| tensors = [tensors] | |
| ring_attn_rank = dist.get_rank(group=ring_attn_group) | |
| ring_attn_size = dist.get_world_size(group=ring_attn_group) | |
| seqlen = tensors[0].shape[-1] | |
| total_seq_len = tensors[0].numel() | |
| ring_attn_pad_len = (ring_attn_size - seqlen % ring_attn_size) % ring_attn_size | |
| output_tensors = [] | |
| for tensor in tensors: | |
| if tensor.numel() != total_seq_len: | |
| raise ValueError(f"tensor.numel() {tensor.numel()} != total_seq_len {total_seq_len}") | |
| tensor = torch.nn.functional.pad(tensor, (0, ring_attn_pad_len), value=pad_id) | |
| local_seq_len = tensor.numel() // ring_attn_size | |
| start, end = ring_attn_rank * local_seq_len, (ring_attn_rank + 1) * local_seq_len | |
| tensor = tensor[:, start:end] | |
| output_tensors.append(tensor) | |
| if len(output_tensors) == 1: | |
| output_tensors = output_tensors[0] | |
| return output_tensors, ring_attn_pad_len | |
| def unpad_and_slice_tensor(sequences, attention_mask, ring_attn_group): | |
| """ | |
| Unpad and slice tensor for distributed training with ring attention. | |
| This function performs several operations: | |
| 1. Removes padding, unpads sequences from (batch, seqlen) to (1, total_seqs) | |
| 2. Adapts to ring_attn_group, pads sequences to be divisible by ring_attn_group | |
| 3. Slices the sequences for the current ring_attn_rank | |
| Example: | |
| >>> # Input sequences shape: (batch=2, seqlen=4) | |
| >>> sequences = [[1, 2, 3, 0], [4, 5, 0, 0]] # 0 is padding | |
| >>> attention_mask = [[1, 1, 1, 0], [1, 1, 0, 0]] | |
| >>> # After unpad: | |
| >>> # sequences: [1, 2, 3, 4, 5] # shape (1, total_seqs=5) | |
| >>> # If ring_attn_group size is 2, it will pad to length 6 | |
| >>> # Then slice for current rank (e.g., rank 0 gets [1,2,3], rank 1 gets [4,5,0]) | |
| Args: | |
| sequences: Input sequences tensor of shape (batch, seqlen) | |
| attention_mask: Attention mask tensor for the sequences | |
| ring_attn_group: Ring attention group for distributed processing | |
| Returns: | |
| tuple: Processed sequences and related tensors for ring attention | |
| Note: | |
| Requires ``flash_attn`` — only called when ``packing_samples=True``. | |
| """ | |
| index_first_axis, pad_input, rearrange, unpad_input, _ = _require_flash_attn() | |
| rolled_sequences = torch.roll(sequences, shifts=-1, dims=1) | |
| sequences, indices, cu_seqlens, _, _ = unpad_input(sequences.unsqueeze(-1), attention_mask) | |
| sequences = sequences.transpose(0, 1) # (1, total_seqs) | |
| rolled_sequences = index_first_axis( | |
| rearrange(rolled_sequences.unsqueeze(-1), "b s ... -> (b s) ..."), indices | |
| ).transpose( | |
| 0, 1 | |
| ) # (1, total_seqs) | |
| position_ids = torch.clip(torch.cumsum(attention_mask, dim=-1) - 1, min=0, max=None) | |
| position_ids = index_first_axis(rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), indices).transpose( | |
| 0, 1 | |
| ) # (1, total_seqs) | |
| ring_attn_pad_len = 0 | |
| if ring_attn_group is not None: | |
| (sequences, position_ids, rolled_sequences), ring_attn_pad_len = get_tensor_in_current_ring_attn_rank( | |
| [sequences, position_ids, rolled_sequences], ring_attn_group, 0 | |
| ) | |
| cu_seqlens[-1] += ring_attn_pad_len | |
| update_ring_attn_params(cu_seqlens) | |
| return sequences, position_ids, rolled_sequences, ring_attn_pad_len, indices | |
| def gather_and_pad_tensor(tensor, ring_attn_group, ring_attn_pad_len, indices, batch, seqlen): | |
| """ | |
| Gather and pad tensor data (such as logits, log_probs, etc.). | |
| Example: | |
| >>> # Input tensor from each rank (shape: (1, local_seq_len)) | |
| >>> # Rank 0: [1, 2, 3] | |
| >>> # Rank 1: [4, 5, 0] # 0 is padding | |
| >>> # After all_gather: | |
| >>> # tensor: [1, 2, 3, 4, 5, 0] # shape (1, total_seqs=6) | |
| >>> # After removing padding (ring_attn_pad_len=1): | |
| >>> # tensor: [1, 2, 3, 4, 5] # shape (1, total_seqs=5) | |
| >>> # After pad_input with original indices: | |
| >>> # tensor: [[1, 2, 3, 0], [4, 5, 0, 0]] # shape (batch=2, seqlen=4) | |
| Args: | |
| tensor: Input tensor, can be logits, log_probs, etc. | |
| ring_attn_group: Ring attention group | |
| ring_attn_pad_len: Padding length | |
| indices: Indices | |
| batch: Batch size | |
| seqlen: Sequence length | |
| Returns: | |
| Padded tensor | |
| Note: | |
| Requires ``flash_attn`` — only called when ``packing_samples=True``. | |
| """ | |
| _, pad_input, _, _, fa_all_gather = _require_flash_attn() | |
| if ring_attn_group is not None: | |
| tensor = fa_all_gather(tensor.transpose(0, 1), ring_attn_group).transpose(0, 1) # (1, total_seqs) | |
| if ring_attn_pad_len > 0: | |
| tensor = tensor[:, :-ring_attn_pad_len] | |
| tensor = pad_input(tensor.transpose(0, 1), indices, batch, seqlen).squeeze(-1) # (batch, seqlen) | |
| return tensor | |