import torch def precompute_rope(head_dim: int, max_seq_len: int, theta: float, device=None, dtype=torch.float32): inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2, device=device, dtype=torch.float32) / head_dim)) positions = torch.arange(max_seq_len, device=device, dtype=torch.float32) freqs = torch.outer(positions, inv_freq) # (max_seq_len, head_dim/2) cos = freqs.cos().to(dtype) sin = freqs.sin().to(dtype) return cos, sin def _rotate_half(x: torch.Tensor) -> torch.Tensor: x1, x2 = x.chunk(2, dim=-1) return torch.cat((-x2, x1), dim=-1) def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: """x: (..., seq_len, head_dim). cos/sin: (seq_len, head_dim/2), broadcast-repeated to head_dim.""" cos = torch.cat((cos, cos), dim=-1) sin = torch.cat((sin, sin), dim=-1) return x * cos + _rotate_half(x) * sin