"""(1b) Small byte-level GRU baseline on the same data, same val metric as ATOM. usage: gru_baseline.py NAME --frac F --minutes M [--hidden H] [--extra PATHS...] [--threads T] [--memorize FILE] Train set = held-out corpus (val window ranges +-4KB removed; strictly harder than ATOM's setting), optionally a prefix fraction F of it, optionally + extra French text. Val = ATOM 2048-transition val, context reset at each window start, nats/byte. """ import argparse, json, math, sys, time from pathlib import Path import torch, torch.nn as nn, torch.nn.functional as F sys.path.insert(0, str(Path(__file__).resolve().parent)) from common import PROMPTS, val_windows_bytes from ngram_baseline import heldout_corpus OUT = Path(__file__).resolve().parent class GRULM(nn.Module): def __init__(self, h=512, layers=2, emb=64): super().__init__() self.emb = nn.Embedding(256, emb); self.rnn = nn.GRU(emb, h, layers, batch_first=True); self.head = nn.Linear(h, 256) def forward(self, x, s=None): y, s = self.rnn(self.emb(x), s); return self.head(y), s @torch.no_grad() def val_ce(m, wins): m.eval(); tot = n = 0 for w in wins: x = torch.tensor(list(w), dtype=torch.long)[None] lg, _ = m(x[:, :-1]); tot += F.cross_entropy(lg[0], x[0, 1:], reduction="sum").item(); n += x.shape[1] - 1 m.train(); return tot / n @torch.no_grad() def sample(m, prompt, g, T=0.8, k=12, max_len=100, wrap=True): m.eval(); ctx = (f"Utilisateur: {prompt}\nAssistant: " if wrap else prompt).encode() lg, s = m(torch.tensor(list(ctx))[None]); out = bytearray(); last = lg[0, -1] while len(out) < max_len: v, i = torch.topk(last / T, k); b = int(i[torch.multinomial(v.softmax(-1), 1, generator=g)]) out.append(b) if out.endswith(b"\n\n"): break lg, s = m(torch.tensor([[b]]), s); last = lg[0, -1] m.train(); return out.decode("utf-8", "replace") def main(): ap = argparse.ArgumentParser(); ap.add_argument("name"); ap.add_argument("--frac", type=float, default=1.0) ap.add_argument("--minutes", type=float, default=20); ap.add_argument("--hidden", type=int, default=512) ap.add_argument("--layers", type=int, default=2); ap.add_argument("--extra", nargs="*", default=[]) ap.add_argument("--threads", type=int, default=4); ap.add_argument("--memorize", default=None) ap.add_argument("--seq", type=int, default=128); ap.add_argument("--bs", type=int, default=32) a = ap.parse_args(); torch.set_num_threads(a.threads); torch.manual_seed(0) if a.memorize: data = Path(a.memorize).read_bytes() else: data = heldout_corpus(); data = data[: int(len(data) * a.frac)] for p in a.extra: data += Path(p).read_bytes() D = torch.tensor(list(data), dtype=torch.long) wins = val_windows_bytes(); m = GRULM(a.hidden, a.layers) nparam = sum(p.numel() for p in m.parameters()) opt = torch.optim.Adam(m.parameters(), lr=2e-3) t0 = time.time(); step = 0; log = []; seq = min(a.seq, len(D) - 2) total = a.minutes * 60 while time.time() - t0 < total: frac_t = (time.time() - t0) / total for gp in opt.param_groups: gp["lr"] = 2e-3 * (0.05 + 0.95 * 0.5 * (1 + math.cos(math.pi * frac_t))) ix = torch.randint(0, len(D) - seq - 1, (a.bs,)) x = torch.stack([D[i:i + seq + 1] for i in ix]) lg, _ = m(x[:, :-1]); loss = F.cross_entropy(lg.reshape(-1, 256), x[:, 1:].reshape(-1)) opt.zero_grad(); loss.backward(); nn.utils.clip_grad_norm_(m.parameters(), 1.0); opt.step(); step += 1 if step % 200 == 0: r = {"step": step, "min": round((time.time() - t0) / 60, 2), "train": round(loss.item(), 4), "bytes_seen": step * a.bs * seq} if not a.memorize: r["val"] = round(val_ce(m, wins), 4) log.append(r); print(json.dumps(r), flush=True) g = torch.Generator().manual_seed(0) res = {"name": a.name, "params": nparam, "train_bytes": len(data), "frac": a.frac, "extra": a.extra, "steps": step, "epochs": round(step * a.bs * seq / len(data), 2), "log": log} if a.memorize: # teacher-forced CE on the text + greedy regeneration from its first 32 bytes with torch.no_grad(): m.eval(); x = D[None]; lg, _ = m(x[:, :-1]); res["tf_ce"] = F.cross_entropy(lg[0], x[0, 1:]).item() ctx = data[:32]; lg, s = m(torch.tensor(list(ctx))[None]); out = bytearray(ctx); last = lg[0, -1] while len(out) < len(data): b = int(last.argmax()); out.append(b); lg, s = m(torch.tensor([[b]]), s); last = lg[0, -1] same = next((i for i, (u, v) in enumerate(zip(out, data)) if u != v), len(data)) res["regen_prefix_match_bytes"] = same; res["regen_head"] = bytes(out[:300]).decode("utf-8", "replace") else: res["val_ce"] = val_ce(m, wins) res["samples"] = {p: [sample(m, p, g) for _ in range(3)] for p in PROMPTS} torch.save(m.state_dict(), OUT / f"gru_{a.name}.pt") (OUT / f"gru_{a.name}.json").write_text(json.dumps(res, indent=1, ensure_ascii=False)) print(json.dumps({k: v for k, v in res.items() if k != "log"}, ensure_ascii=False, indent=1)) if __name__ == "__main__": main()