ATOM (atom-ai): code snapshot (100fae0 + batched-train), README with measured numbers, curated checkpoints (part 3)
969c687 verified Download docs/artifacts/root_cause/gru_baseline.py from thefinalboss/atom-ai: direct link, hf CLI and curl.
- Browser
- Download file 5.25 kB
-
https://huggingface.co/thefinalboss/atom-ai/resolve/main/docs/artifacts/root_cause/gru_baseline.py
- Command line
-
hf download hf://thefinalboss/atom-ai/docs/artifacts/root_cause/gru_baseline.py
-
curl -L -o gru_baseline.py https://huggingface.co/thefinalboss/atom-ai/resolve/main/docs/artifacts/root_cause/gru_baseline.py
5.25 kB
| """(1b) Small byte-level GRU baseline on the same data, same val metric as ATOM. | |
| usage: gru_baseline.py NAME --frac F --minutes M [--hidden H] [--extra PATHS...] [--threads T] [--memorize FILE] | |
| Train set = held-out corpus (val window ranges +-4KB removed; strictly harder than ATOM's setting), | |
| optionally a prefix fraction F of it, optionally + extra French text. | |
| Val = ATOM 2048-transition val, context reset at each window start, nats/byte. | |
| """ | |
| import argparse, json, math, sys, time | |
| from pathlib import Path | |
| import torch, torch.nn as nn, torch.nn.functional as F | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| from common import PROMPTS, val_windows_bytes | |
| from ngram_baseline import heldout_corpus | |
| OUT = Path(__file__).resolve().parent | |
| class GRULM(nn.Module): | |
| def __init__(self, h=512, layers=2, emb=64): | |
| super().__init__() | |
| self.emb = nn.Embedding(256, emb); self.rnn = nn.GRU(emb, h, layers, batch_first=True); self.head = nn.Linear(h, 256) | |
| def forward(self, x, s=None): | |
| y, s = self.rnn(self.emb(x), s); return self.head(y), s | |
| def val_ce(m, wins): | |
| m.eval(); tot = n = 0 | |
| for w in wins: | |
| x = torch.tensor(list(w), dtype=torch.long)[None] | |
| lg, _ = m(x[:, :-1]); tot += F.cross_entropy(lg[0], x[0, 1:], reduction="sum").item(); n += x.shape[1] - 1 | |
| m.train(); return tot / n | |
| def sample(m, prompt, g, T=0.8, k=12, max_len=100, wrap=True): | |
| m.eval(); ctx = (f"Utilisateur: {prompt}\nAssistant: " if wrap else prompt).encode() | |
| lg, s = m(torch.tensor(list(ctx))[None]); out = bytearray(); last = lg[0, -1] | |
| while len(out) < max_len: | |
| v, i = torch.topk(last / T, k); b = int(i[torch.multinomial(v.softmax(-1), 1, generator=g)]) | |
| out.append(b) | |
| if out.endswith(b"\n\n"): break | |
| lg, s = m(torch.tensor([[b]]), s); last = lg[0, -1] | |
| m.train(); return out.decode("utf-8", "replace") | |
| def main(): | |
| ap = argparse.ArgumentParser(); ap.add_argument("name"); ap.add_argument("--frac", type=float, default=1.0) | |
| ap.add_argument("--minutes", type=float, default=20); ap.add_argument("--hidden", type=int, default=512) | |
| ap.add_argument("--layers", type=int, default=2); ap.add_argument("--extra", nargs="*", default=[]) | |
| ap.add_argument("--threads", type=int, default=4); ap.add_argument("--memorize", default=None) | |
| ap.add_argument("--seq", type=int, default=128); ap.add_argument("--bs", type=int, default=32) | |
| a = ap.parse_args(); torch.set_num_threads(a.threads); torch.manual_seed(0) | |
| if a.memorize: | |
| data = Path(a.memorize).read_bytes() | |
| else: | |
| data = heldout_corpus(); data = data[: int(len(data) * a.frac)] | |
| for p in a.extra: data += Path(p).read_bytes() | |
| D = torch.tensor(list(data), dtype=torch.long) | |
| wins = val_windows_bytes(); m = GRULM(a.hidden, a.layers) | |
| nparam = sum(p.numel() for p in m.parameters()) | |
| opt = torch.optim.Adam(m.parameters(), lr=2e-3) | |
| t0 = time.time(); step = 0; log = []; seq = min(a.seq, len(D) - 2) | |
| total = a.minutes * 60 | |
| while time.time() - t0 < total: | |
| frac_t = (time.time() - t0) / total | |
| for gp in opt.param_groups: gp["lr"] = 2e-3 * (0.05 + 0.95 * 0.5 * (1 + math.cos(math.pi * frac_t))) | |
| ix = torch.randint(0, len(D) - seq - 1, (a.bs,)) | |
| x = torch.stack([D[i:i + seq + 1] for i in ix]) | |
| lg, _ = m(x[:, :-1]); loss = F.cross_entropy(lg.reshape(-1, 256), x[:, 1:].reshape(-1)) | |
| opt.zero_grad(); loss.backward(); nn.utils.clip_grad_norm_(m.parameters(), 1.0); opt.step(); step += 1 | |
| if step % 200 == 0: | |
| r = {"step": step, "min": round((time.time() - t0) / 60, 2), "train": round(loss.item(), 4), | |
| "bytes_seen": step * a.bs * seq} | |
| if not a.memorize: r["val"] = round(val_ce(m, wins), 4) | |
| log.append(r); print(json.dumps(r), flush=True) | |
| g = torch.Generator().manual_seed(0) | |
| res = {"name": a.name, "params": nparam, "train_bytes": len(data), "frac": a.frac, "extra": a.extra, "steps": step, | |
| "epochs": round(step * a.bs * seq / len(data), 2), "log": log} | |
| if a.memorize: | |
| # teacher-forced CE on the text + greedy regeneration from its first 32 bytes | |
| with torch.no_grad(): | |
| m.eval(); x = D[None]; lg, _ = m(x[:, :-1]); res["tf_ce"] = F.cross_entropy(lg[0], x[0, 1:]).item() | |
| ctx = data[:32]; lg, s = m(torch.tensor(list(ctx))[None]); out = bytearray(ctx); last = lg[0, -1] | |
| while len(out) < len(data): | |
| b = int(last.argmax()); out.append(b); lg, s = m(torch.tensor([[b]]), s); last = lg[0, -1] | |
| same = next((i for i, (u, v) in enumerate(zip(out, data)) if u != v), len(data)) | |
| res["regen_prefix_match_bytes"] = same; res["regen_head"] = bytes(out[:300]).decode("utf-8", "replace") | |
| else: | |
| res["val_ce"] = val_ce(m, wins) | |
| res["samples"] = {p: [sample(m, p, g) for _ in range(3)] for p in PROMPTS} | |
| torch.save(m.state_dict(), OUT / f"gru_{a.name}.pt") | |
| (OUT / f"gru_{a.name}.json").write_text(json.dumps(res, indent=1, ensure_ascii=False)) | |
| print(json.dumps({k: v for k, v in res.items() if k != "log"}, ensure_ascii=False, indent=1)) | |
| if __name__ == "__main__": | |
| main() | |