Download src/mulattice/mqar.py from Ex0bit/mLattice: direct link, hf CLI and curl.
- Browser
- Download file 2.61 kB
-
https://huggingface.co/spaces/Ex0bit/mLattice/resolve/main/src/mulattice/mqar.py
- Command line
-
hf download hf://spaces/Ex0bit/mLattice/src/mulattice/mqar.py
-
curl -L -o mqar.py https://huggingface.co/spaces/Ex0bit/mLattice/resolve/main/src/mulattice/mqar.py
2.61 kB
| """MQAR (multi-query associative recall) data generator, Zoology-style. | |
| A sequence is key_1 val_1 ... key_N val_N followed by a query block of Q | |
| keys whose values must be recalled. Train with <=64 pairs, evaluate with the | |
| same generator at a larger pair count (the published protocol: train <=64, | |
| extrapolate to 256, and probe capacity/forgetting at 2^17). | |
| """ | |
| from __future__ import annotations | |
| import torch | |
| def make_mqar_batch( | |
| batch: int, | |
| n_pairs: int, | |
| n_queries: int, | |
| n_keys: int, | |
| n_values: int, | |
| device="cpu", | |
| generator: torch.Generator | None = None, | |
| key_offset: int = 0, | |
| ) -> dict: | |
| """Returns dict with `tokens` (B, L) and `labels` (B, L) (-100 where ignored). | |
| Keys are sampled WITHOUT replacement per sequence (the standard MQAR protocol): | |
| every key in a sequence is distinct, so every query has a unique answer and the | |
| achievable accuracy is 1.0. Sampling WITH replacement makes later keys collide -- | |
| at 256 pairs over 1024 keys ~30 keys repeat with different values, capping accuracy | |
| at ~0.885, measuring the ambiguity ceiling instead of the architecture. | |
| """ | |
| g = generator | |
| B = batch | |
| if n_pairs > n_keys: | |
| raise ValueError(f"n_pairs ({n_pairs}) must be <= n_keys ({n_keys}) to sample distinct keys") | |
| keys = torch.stack([torch.randperm(n_keys, generator=g)[:n_pairs] for _ in range(B)]) | |
| keys = keys.to(device) | |
| if n_pairs > n_values: | |
| raise ValueError(f"n_pairs ({n_pairs}) must be <= n_values ({n_values}) to sample distinct values") | |
| vals = torch.stack([torch.randperm(n_values, generator=g)[:n_pairs] for _ in range(B)]) | |
| vals = vals.to(device) | |
| # query positions: distinct keys from the stored set (recall, not lookup-miss) | |
| qpos = torch.randint(0, n_pairs, (B, n_queries), device=device, generator=g) | |
| qkeys = torch.gather(keys, 1, qpos) | |
| kv = torch.stack([keys + key_offset, vals + key_offset + n_keys], dim=-1).reshape(B, 2 * n_pairs) | |
| qseq = torch.stack( | |
| [qkeys + key_offset, torch.full_like(qkeys, n_keys + n_values + key_offset + 1)], dim=-1 | |
| ).reshape(B, 2 * n_queries) | |
| tokens = torch.cat([kv, qseq], dim=1) | |
| labels = torch.full_like(tokens, -100) | |
| ans_pos = 2 * n_pairs + torch.arange(1, 2 * n_queries, 2, device=device) | |
| answers = torch.gather(vals, 1, qpos) + key_offset + n_keys | |
| labels[:, ans_pos] = answers | |
| return {"tokens": tokens, "labels": labels, "n_pairs": n_pairs, "answers": answers, "ans_pos": ans_pos} | |
| def vocab_size(n_keys: int, n_values: int) -> int: | |
| return n_keys + n_values + 2 # keys | values | separator | |