atom-ai / docs /artifacts /root_cause /gru_baseline.py
thefinalboss's picture
ATOM (atom-ai): code snapshot (100fae0 + batched-train), README with measured numbers, curated checkpoints (part 3)
969c687 verified
Raw History Blame Contribute Delete
5.25 kB
"""(1b) Small byte-level GRU baseline on the same data, same val metric as ATOM.
usage: gru_baseline.py NAME --frac F --minutes M [--hidden H] [--extra PATHS...] [--threads T] [--memorize FILE]
Train set = held-out corpus (val window ranges +-4KB removed; strictly harder than ATOM's setting),
optionally a prefix fraction F of it, optionally + extra French text.
Val = ATOM 2048-transition val, context reset at each window start, nats/byte.
"""
import argparse, json, math, sys, time
from pathlib import Path
import torch, torch.nn as nn, torch.nn.functional as F
sys.path.insert(0, str(Path(__file__).resolve().parent))
from common import PROMPTS, val_windows_bytes
from ngram_baseline import heldout_corpus
OUT = Path(__file__).resolve().parent
class GRULM(nn.Module):
def __init__(self, h=512, layers=2, emb=64):
super().__init__()
self.emb = nn.Embedding(256, emb); self.rnn = nn.GRU(emb, h, layers, batch_first=True); self.head = nn.Linear(h, 256)
def forward(self, x, s=None):
y, s = self.rnn(self.emb(x), s); return self.head(y), s
@torch.no_grad()
def val_ce(m, wins):
m.eval(); tot = n = 0
for w in wins:
x = torch.tensor(list(w), dtype=torch.long)[None]
lg, _ = m(x[:, :-1]); tot += F.cross_entropy(lg[0], x[0, 1:], reduction="sum").item(); n += x.shape[1] - 1
m.train(); return tot / n
@torch.no_grad()
def sample(m, prompt, g, T=0.8, k=12, max_len=100, wrap=True):
m.eval(); ctx = (f"Utilisateur: {prompt}\nAssistant: " if wrap else prompt).encode()
lg, s = m(torch.tensor(list(ctx))[None]); out = bytearray(); last = lg[0, -1]
while len(out) < max_len:
v, i = torch.topk(last / T, k); b = int(i[torch.multinomial(v.softmax(-1), 1, generator=g)])
out.append(b)
if out.endswith(b"\n\n"): break
lg, s = m(torch.tensor([[b]]), s); last = lg[0, -1]
m.train(); return out.decode("utf-8", "replace")
def main():
ap = argparse.ArgumentParser(); ap.add_argument("name"); ap.add_argument("--frac", type=float, default=1.0)
ap.add_argument("--minutes", type=float, default=20); ap.add_argument("--hidden", type=int, default=512)
ap.add_argument("--layers", type=int, default=2); ap.add_argument("--extra", nargs="*", default=[])
ap.add_argument("--threads", type=int, default=4); ap.add_argument("--memorize", default=None)
ap.add_argument("--seq", type=int, default=128); ap.add_argument("--bs", type=int, default=32)
a = ap.parse_args(); torch.set_num_threads(a.threads); torch.manual_seed(0)
if a.memorize:
data = Path(a.memorize).read_bytes()
else:
data = heldout_corpus(); data = data[: int(len(data) * a.frac)]
for p in a.extra: data += Path(p).read_bytes()
D = torch.tensor(list(data), dtype=torch.long)
wins = val_windows_bytes(); m = GRULM(a.hidden, a.layers)
nparam = sum(p.numel() for p in m.parameters())
opt = torch.optim.Adam(m.parameters(), lr=2e-3)
t0 = time.time(); step = 0; log = []; seq = min(a.seq, len(D) - 2)
total = a.minutes * 60
while time.time() - t0 < total:
frac_t = (time.time() - t0) / total
for gp in opt.param_groups: gp["lr"] = 2e-3 * (0.05 + 0.95 * 0.5 * (1 + math.cos(math.pi * frac_t)))
ix = torch.randint(0, len(D) - seq - 1, (a.bs,))
x = torch.stack([D[i:i + seq + 1] for i in ix])
lg, _ = m(x[:, :-1]); loss = F.cross_entropy(lg.reshape(-1, 256), x[:, 1:].reshape(-1))
opt.zero_grad(); loss.backward(); nn.utils.clip_grad_norm_(m.parameters(), 1.0); opt.step(); step += 1
if step % 200 == 0:
r = {"step": step, "min": round((time.time() - t0) / 60, 2), "train": round(loss.item(), 4),
"bytes_seen": step * a.bs * seq}
if not a.memorize: r["val"] = round(val_ce(m, wins), 4)
log.append(r); print(json.dumps(r), flush=True)
g = torch.Generator().manual_seed(0)
res = {"name": a.name, "params": nparam, "train_bytes": len(data), "frac": a.frac, "extra": a.extra, "steps": step,
"epochs": round(step * a.bs * seq / len(data), 2), "log": log}
if a.memorize:
# teacher-forced CE on the text + greedy regeneration from its first 32 bytes
with torch.no_grad():
m.eval(); x = D[None]; lg, _ = m(x[:, :-1]); res["tf_ce"] = F.cross_entropy(lg[0], x[0, 1:]).item()
ctx = data[:32]; lg, s = m(torch.tensor(list(ctx))[None]); out = bytearray(ctx); last = lg[0, -1]
while len(out) < len(data):
b = int(last.argmax()); out.append(b); lg, s = m(torch.tensor([[b]]), s); last = lg[0, -1]
same = next((i for i, (u, v) in enumerate(zip(out, data)) if u != v), len(data))
res["regen_prefix_match_bytes"] = same; res["regen_head"] = bytes(out[:300]).decode("utf-8", "replace")
else:
res["val_ce"] = val_ce(m, wins)
res["samples"] = {p: [sample(m, p, g) for _ in range(3)] for p in PROMPTS}
torch.save(m.state_dict(), OUT / f"gru_{a.name}.pt")
(OUT / f"gru_{a.name}.json").write_text(json.dumps(res, indent=1, ensure_ascii=False))
print(json.dumps({k: v for k, v in res.items() if k != "log"}, ensure_ascii=False, indent=1))
if __name__ == "__main__":
main()