Download src/model/rope.py from TheFireHacker/bubblspace-moe0.9b-a0.2b: direct link, hf CLI and curl.
- Browser
- Download file 903 Bytes
-
https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/src/model/rope.py
- Command line
-
hf download hf://TheFireHacker/bubblspace-moe0.9b-a0.2b/src/model/rope.py
-
curl -L -o rope.py https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/src/model/rope.py
903 Bytes
| import torch | |
| def precompute_rope(head_dim: int, max_seq_len: int, theta: float, device=None, dtype=torch.float32): | |
| inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2, device=device, dtype=torch.float32) / head_dim)) | |
| positions = torch.arange(max_seq_len, device=device, dtype=torch.float32) | |
| freqs = torch.outer(positions, inv_freq) # (max_seq_len, head_dim/2) | |
| cos = freqs.cos().to(dtype) | |
| sin = freqs.sin().to(dtype) | |
| return cos, sin | |
| def _rotate_half(x: torch.Tensor) -> torch.Tensor: | |
| x1, x2 = x.chunk(2, dim=-1) | |
| return torch.cat((-x2, x1), dim=-1) | |
| def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: | |
| """x: (..., seq_len, head_dim). cos/sin: (seq_len, head_dim/2), broadcast-repeated to head_dim.""" | |
| cos = torch.cat((cos, cos), dim=-1) | |
| sin = torch.cat((sin, sin), dim=-1) | |
| return x * cos + _rotate_half(x) * sin | |