TheFireHacker's picture
Upload folder using huggingface_hub
27cd743 verified
Raw History Blame Contribute Delete
903 Bytes
import torch
def precompute_rope(head_dim: int, max_seq_len: int, theta: float, device=None, dtype=torch.float32):
inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2, device=device, dtype=torch.float32) / head_dim))
positions = torch.arange(max_seq_len, device=device, dtype=torch.float32)
freqs = torch.outer(positions, inv_freq) # (max_seq_len, head_dim/2)
cos = freqs.cos().to(dtype)
sin = freqs.sin().to(dtype)
return cos, sin
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
x1, x2 = x.chunk(2, dim=-1)
return torch.cat((-x2, x1), dim=-1)
def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
"""x: (..., seq_len, head_dim). cos/sin: (seq_len, head_dim/2), broadcast-repeated to head_dim."""
cos = torch.cat((cos, cos), dim=-1)
sin = torch.cat((sin, sin), dim=-1)
return x * cos + _rotate_half(x) * sin