Download proxy_arch.py from BranchingNLP/Grace-G1-350M: direct link, hf CLI and curl.
- Browser
- Download file 21.6 kB
-
https://huggingface.co/BranchingNLP/Grace-G1-350M/resolve/main/proxy_arch.py
- Command line
-
hf download hf://BranchingNLP/Grace-G1-350M/proxy_arch.py
-
curl -L -o proxy_arch.py https://huggingface.co/BranchingNLP/Grace-G1-350M/resolve/main/proxy_arch.py
21.6 kB
| #!/usr/bin/env python3 | |
| """ | |
| proxy_arch.py — confronto di architetture/ottimizzatori su modelli piccoli (~50M) PRIMA di spendere sul cloud. | |
| Varianti: | |
| --variant attn attenzione piena con output gate (stile Qwen3-Next) in tutti i layer | |
| --variant hybrid 3 layer Gated DeltaNet (flash-linear-attention) : 1 layer di attenzione con gate | |
| --opt adamw|muon Muon sulle matrici 2D interne, AdamW su embedding / norme / resto | |
| Stessi dati, stesso ordine dei batch, stesso schedule WSD per tutte le run: cambia solo quello che stai testando. | |
| python3 proxy_arch.py --variant attn --opt adamw --out proxy_runs/A | |
| python3 proxy_arch.py --variant hybrid --opt adamw --out proxy_runs/B | |
| python3 proxy_arch.py --variant attn --opt muon --out proxy_runs/C (o hybrid, quello che vince tra A e B) | |
| python3 proxy_arch.py --compare proxy_runs/* # tabella finale | |
| Legge data-ark35m/meta.json + {nome}.bin / val_{nome}.bin (uint16) come ark35m.py. | |
| """ | |
| import argparse, glob, inspect, json, math, os, sys, time | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| # ── dati ──────────────────────────────────────────────────────────────────────────────────── | |
| class Mix: | |
| def __init__(self, data_dir, seq, seed, split="train"): | |
| meta = json.load(open(os.path.join(data_dir, "meta.json"))) | |
| self.srcs, ws = [], [] | |
| for name, s in meta["sources"].items(): | |
| path = os.path.join(data_dir, f"{name}.bin" if split == "train" else f"val_{name}.bin") | |
| if not os.path.exists(path) or os.path.getsize(path) < 2 * (seq + 2): | |
| continue | |
| self.srcs.append(np.memmap(path, dtype=np.uint16, mode="r")) | |
| ws.append(float(s.get("weight", 1.0))) | |
| if not self.srcs: | |
| sys.exit(f"nessun file {split} trovato in {data_dir}") | |
| self.p = np.array(ws) / sum(ws) | |
| self.seq, self.rng = seq, np.random.default_rng(seed) | |
| def batch(self, bs, device): | |
| x = np.empty((bs, self.seq + 1), dtype=np.int64) | |
| for i in range(bs): | |
| m = self.srcs[self.rng.choice(len(self.srcs), p=self.p)] | |
| j = self.rng.integers(0, len(m) - self.seq - 1) | |
| x[i] = m[j:j + self.seq + 1] | |
| t = torch.from_numpy(x).to(device, non_blocking=True) | |
| return t[:, :-1], t[:, 1:] | |
| # ── modello ───────────────────────────────────────────────────────────────────────────────── | |
| class RMSNorm(nn.Module): | |
| def __init__(self, d, eps=1e-6): | |
| super().__init__() | |
| self.w, self.eps = nn.Parameter(torch.ones(d)), eps | |
| def forward(self, x): | |
| return (x.float() * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + self.eps)).type_as(x) * self.w | |
| def rope(x, cos, sin): | |
| x1, x2 = x.chunk(2, -1) | |
| return x * cos + torch.cat((-x2, x1), -1) * sin | |
| class GatedAttention(nn.Module): | |
| """GQA + QK-norm + RoPE + output gate sigmoide elemento per elemento (prima di o_proj).""" | |
| def __init__(self, d, h, kv): | |
| super().__init__() | |
| self.h, self.kv, self.hd = h, kv, d // h | |
| self.q = nn.Linear(d, h * self.hd, bias=False) | |
| self.k = nn.Linear(d, kv * self.hd, bias=False) | |
| self.v = nn.Linear(d, kv * self.hd, bias=False) | |
| self.g = nn.Linear(d, h * self.hd, bias=False) | |
| self.o = nn.Linear(h * self.hd, d, bias=False) | |
| self.qn, self.kn = RMSNorm(self.hd), RMSNorm(self.hd) | |
| def forward(self, x, cos, sin): | |
| B, T, _ = x.shape | |
| q = self.qn(self.q(x).view(B, T, self.h, self.hd)).transpose(1, 2) | |
| k = self.kn(self.k(x).view(B, T, self.kv, self.hd)).transpose(1, 2) | |
| v = self.v(x).view(B, T, self.kv, self.hd).transpose(1, 2) | |
| q, k = rope(q, cos, sin), rope(k, cos, sin) | |
| rep = self.h // self.kv | |
| y = F.scaled_dot_product_attention(q, k.repeat_interleave(rep, 1), v.repeat_interleave(rep, 1), is_causal=True) | |
| y = y.transpose(1, 2).reshape(B, T, -1) * torch.sigmoid(self.g(x)) | |
| return self.o(y) | |
| MIXERS = {"gdn": "GatedDeltaNet", # gate di oblio scalare per testa | |
| "kda": "KimiDeltaAttention", # gate di oblio per canale (use_full_rank_gate di Kimi) | |
| "gdn2": "GatedDeltaNet2"} # + gate per canale anche su cancellazione e scrittura | |
| def make_gdn(d, heads, head_dim, layer_idx, mixer="gdn"): | |
| import fla.layers | |
| cls = getattr(fla.layers, MIXERS[mixer]) | |
| want = dict(hidden_size=d, num_heads=heads, head_dim=head_dim, expand_v=2, mode="chunk", | |
| use_gate=True, use_short_conv=True, conv_size=4, layer_idx=layer_idx) | |
| if mixer == "kda": | |
| want["lower_bound"] = -5.0 # come gate_lower_bound di Kimi | |
| ok = inspect.signature(cls.__init__).parameters | |
| return cls(**{k: v for k, v in want.items() if k in ok}) | |
| class GDNWrap(nn.Module): | |
| def __init__(self, d, heads, head_dim, idx, mixer="gdn"): | |
| super().__init__() | |
| self.m = make_gdn(d, heads, head_dim, idx, mixer) | |
| def forward(self, x, cos, sin): | |
| y = self.m(x) | |
| return y[0] if isinstance(y, tuple) else y | |
| class SwiGLU(nn.Module): | |
| def __init__(self, d, ff): | |
| super().__init__() | |
| self.a, self.b, self.c = nn.Linear(d, ff, bias=False), nn.Linear(d, ff, bias=False), nn.Linear(ff, d, bias=False) | |
| def forward(self, x): | |
| return self.c(F.silu(self.a(x)) * self.b(x)) | |
| class Block(nn.Module): | |
| def __init__(self, mixer, d, ff): | |
| super().__init__() | |
| self.n1, self.n2, self.mix, self.ffn = RMSNorm(d), RMSNorm(d), mixer, SwiGLU(d, ff) | |
| def forward(self, x, cos, sin): | |
| x = x + self.mix(self.n1(x), cos, sin) | |
| return x + self.ffn(self.n2(x)) | |
| class LM(nn.Module): | |
| def __init__(self, variant, vocab, d, layers, heads, kv, ff, seq, gdn_heads, gdn_head_dim, attn_every, mixer="gdn"): | |
| super().__init__() | |
| self.emb = nn.Embedding(vocab, d) | |
| blocks = [] | |
| for i in range(layers): | |
| if variant == "hybrid" and (i + 1) % attn_every != 0: | |
| mix = GDNWrap(d, gdn_heads, gdn_head_dim, i, mixer) | |
| else: | |
| mix = GatedAttention(d, heads, kv) | |
| blocks.append(Block(mix, d, ff)) | |
| self.blocks = nn.ModuleList(blocks) | |
| self.norm = RMSNorm(d) | |
| hd = d // heads | |
| inv = 1.0 / (10000 ** (torch.arange(0, hd, 2).float() / hd)) | |
| f = torch.outer(torch.arange(seq).float(), inv) | |
| f = torch.cat((f, f), -1) | |
| self.register_buffer("cos", f.cos()[None, None], persistent=False) | |
| self.register_buffer("sin", f.sin()[None, None], persistent=False) | |
| self.apply(self._init) | |
| for n, p in self.named_parameters(): # residui scalati con la profondità | |
| if n.endswith(("o.weight", "ffn.c.weight")): | |
| nn.init.normal_(p, std=0.02 / math.sqrt(2 * layers)) | |
| def _init(m): | |
| if isinstance(m, nn.Linear): | |
| nn.init.normal_(m.weight, std=0.02) | |
| elif isinstance(m, nn.Embedding): | |
| nn.init.normal_(m.weight, std=0.02) | |
| def forward(self, idx, tgt=None): | |
| T = idx.size(1) | |
| x = self.emb(idx) | |
| cos, sin = self.cos[..., :T, :].to(x.dtype), self.sin[..., :T, :].to(x.dtype) | |
| for b in self.blocks: | |
| x = b(x, cos, sin) | |
| x = self.norm(x) | |
| if tgt is None: | |
| return x @ self.emb.weight.t() | |
| loss = 0.0 # loss a pezzi: logit 32k × 2048 non stanno tutti in 8 GB | |
| xs, ts = x.view(-1, x.size(-1)), tgt.reshape(-1) | |
| for i in range(0, xs.size(0), 4096): | |
| lg = (xs[i:i + 4096] @ self.emb.weight.t()).float() | |
| loss = loss + F.cross_entropy(lg, ts[i:i + 4096], reduction="sum") | |
| return loss / ts.numel() | |
| # ── Muon (Keller Jordan): momentum ortogonalizzato con Newton-Schulz ───────────────────────── | |
| def zeropower_ns5(G, steps=5): | |
| a, b, c = 3.4445, -4.7750, 2.0315 | |
| X = G.bfloat16() | |
| tr = X.size(0) > X.size(1) | |
| if tr: | |
| X = X.T | |
| X = X / (X.norm() + 1e-7) | |
| for _ in range(steps): | |
| A = X @ X.T | |
| X = a * X + (b * A + c * A @ A) @ X | |
| return (X.T if tr else X).to(G.dtype) | |
| class Muon(torch.optim.Optimizer): | |
| def __init__(self, params, lr=0.02, momentum=0.95, weight_decay=0.0): | |
| super().__init__(params, dict(lr=lr, momentum=momentum, weight_decay=weight_decay)) | |
| def step(self): | |
| for g in self.param_groups: | |
| for p in g["params"]: | |
| if p.grad is None: | |
| continue | |
| st = self.state[p] | |
| if "buf" not in st: | |
| st["buf"] = torch.zeros_like(p.grad) | |
| buf = st["buf"] | |
| buf.mul_(g["momentum"]).add_(p.grad) | |
| upd = zeropower_ns5(p.grad.add(buf, alpha=g["momentum"])) # Nesterov | |
| scale = max(1.0, p.size(0) / p.size(1)) ** 0.5 | |
| if g["weight_decay"]: | |
| p.mul_(1 - g["lr"] * g["weight_decay"]) | |
| p.add_(upd, alpha=-g["lr"] * scale) | |
| def build_opts(model, a): | |
| muon_p, adam_dec, adam_nodec = [], [], [] | |
| for n, p in model.named_parameters(): | |
| if not p.requires_grad: | |
| continue | |
| if a.opt == "muon" and p.dim() == 2 and "emb" not in n: | |
| muon_p.append(p) | |
| elif p.dim() >= 2 and "emb" not in n: | |
| adam_dec.append(p) | |
| else: | |
| adam_nodec.append(p) # norme, embedding, conv/bias piccoli | |
| opts = [torch.optim.AdamW([{"params": adam_dec, "weight_decay": a.wd}, {"params": adam_nodec, "weight_decay": 0.0}], | |
| lr=a.lr, betas=(0.9, 0.95), fused=torch.cuda.is_available())] | |
| if muon_p: | |
| opts.append(Muon(muon_p, lr=a.muon_lr, weight_decay=a.muon_wd)) | |
| for o in opts: | |
| for g in o.param_groups: | |
| g["base_lr"] = g["lr"] | |
| return opts | |
| def wsd(step, total, warmup, decay_frac): | |
| if step < warmup: | |
| return (step + 1) / warmup | |
| ds = int(total * (1 - decay_frac)) | |
| if step < ds: | |
| return 1.0 | |
| return max(0.0, 1 - math.sqrt((step - ds) / max(1, total - ds))) | |
| def evaluate(model, val, n, bs, dev): | |
| model.eval() | |
| tot = 0.0 | |
| for _ in range(n): | |
| x, y = val.batch(bs, dev) | |
| with torch.autocast("cuda", torch.bfloat16, enabled=dev == "cuda"): | |
| tot += model(x, y).item() | |
| model.train() | |
| return tot / n | |
| def compare(dirs): | |
| rows = [] | |
| for d in dirs: | |
| f = os.path.join(d, "result.json") | |
| if os.path.exists(f): | |
| rows.append(json.load(open(f))) | |
| if not rows: | |
| sys.exit("nessun result.json trovato") | |
| best = min(r["val_loss"] for r in rows) | |
| print(f"\n{'run':22s} {'variante':9s} {'opt':6s} {'param':>7s} {'val loss':>9s} {'Δ':>7s} {'tok/s':>8s} {'ore':>5s}") | |
| for r in sorted(rows, key=lambda r: r["val_loss"]): | |
| print(f"{os.path.basename(r['out']):22s} {r['variant']:9s} {r['opt']:6s} {r['params'] / 1e6:6.1f}M " | |
| f"{r['val_loss']:9.4f} {r['val_loss'] - best:+7.4f} {r['tok_s']:8,.0f} {r['hours']:5.2f}") | |
| print("\nRegola: differenze di loss < 0.01 sono rumore; un'architettura vince se è meglio in loss SENZA perdere >15% di tok/s.") | |
| def sample(a): | |
| """Text completion con un modello salvato (senza KV cache: ricalcola tutto a ogni token, va bene a 55M).""" | |
| from transformers import AutoTokenizer | |
| ck = torch.load(os.path.join(a.sample, "model.pt"), map_location="cpu", weights_only=False) | |
| r = argparse.Namespace(**ck["args"]) | |
| dev = "cuda" if torch.cuda.is_available() else "cpu" | |
| variant = "hybrid" if r.variant.startswith("hyb") else r.variant | |
| model = LM(variant, r.vocab, r.d, r.layers, r.heads, r.kv, r.ff, r.seq, r.gdn_heads, r.gdn_head_dim, | |
| r.attn_every, getattr(r, "mixer", "gdn")).to(dev) | |
| model.load_state_dict(ck["model"]) | |
| model.eval() | |
| tok = AutoTokenizer.from_pretrained(a.tokenizer, trust_remote_code=True) | |
| prompts = [a.prompt] if a.prompt != "-" else None | |
| while True: | |
| if prompts is None: | |
| try: | |
| p = input("\n>>> ") | |
| except EOFError: | |
| return | |
| if not p.strip(): | |
| continue | |
| else: | |
| p = prompts.pop() if prompts else None | |
| if p is None: | |
| return | |
| ids = tok(p, add_special_tokens=False)["input_ids"] | |
| if tok.bos_token_id is not None: | |
| ids = [tok.bos_token_id] + ids | |
| x = torch.tensor([ids], device=dev) | |
| print(p, end="", flush=True) | |
| prev = tok.decode(ids, skip_special_tokens=True) | |
| for _ in range(a.max_new): | |
| with torch.autocast("cuda", torch.bfloat16, enabled=dev == "cuda"): | |
| lg = model(x[:, -r.seq:])[0, -1].float() | |
| if a.rep_penalty != 1.0: | |
| for tid in set(x[0].tolist()): | |
| lg[tid] = lg[tid] / a.rep_penalty if lg[tid] > 0 else lg[tid] * a.rep_penalty | |
| lg = lg / max(a.temp, 1e-5) | |
| v, i = lg.topk(a.top_k) | |
| nxt = i[torch.multinomial(torch.softmax(v, -1), 1)] | |
| if nxt.item() == tok.eos_token_id: | |
| break | |
| x = torch.cat([x, nxt.view(1, 1)], 1) | |
| cur = tok.decode(x[0].tolist(), skip_special_tokens=True) | |
| print(cur[len(prev):], end="", flush=True) | |
| prev = cur | |
| print() | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--compare", nargs="*") | |
| ap.add_argument("--sample", default=None, help="cartella di una run con model.pt: genera testo") | |
| ap.add_argument("--prompt", default="La capitale d'Italia è") | |
| ap.add_argument("--tokenizer", default="clover-tokenizer") | |
| ap.add_argument("--max-new", type=int, default=80) | |
| ap.add_argument("--temp", type=float, default=0.7) | |
| ap.add_argument("--top-k", type=int, default=40) | |
| ap.add_argument("--rep-penalty", type=float, default=1.3) | |
| ap.add_argument("--variant", choices=["attn", "hybrid"], default="attn") | |
| ap.add_argument("--opt", choices=["adamw", "muon"], default="adamw") | |
| ap.add_argument("--data-dir", default="data-ark35m") | |
| ap.add_argument("--out", default="proxy_runs/run") | |
| ap.add_argument("--tokens", type=float, default=5e8, help="token di training (0.5B ≈ 2-3 h su 3070 a 50M)") | |
| ap.add_argument("--seq", type=int, default=2048) | |
| ap.add_argument("--batch-size", type=int, default=4) | |
| ap.add_argument("--grad-accum", type=int, default=8) | |
| ap.add_argument("--d", type=int, default=512) | |
| ap.add_argument("--layers", type=int, default=12) | |
| ap.add_argument("--heads", type=int, default=8) | |
| ap.add_argument("--kv", type=int, default=2) | |
| ap.add_argument("--ff", type=int, default=1408) | |
| ap.add_argument("--gdn-heads", type=int, default=4) | |
| ap.add_argument("--gdn-head-dim", type=int, default=64) | |
| ap.add_argument("--mixer", choices=list(MIXERS), default="gdn", help="hybrid: gdn | kda | gdn2") | |
| ap.add_argument("--attn-every", type=int, default=4, help="hybrid: un layer di attenzione ogni N") | |
| ap.add_argument("--vocab", type=int, default=32768) | |
| ap.add_argument("--lr", type=float, default=1.5e-3) | |
| ap.add_argument("--muon-lr", type=float, default=0.02) | |
| ap.add_argument("--wd", type=float, default=0.1) | |
| ap.add_argument("--muon-wd", type=float, default=0.0, help="weight decay su Muon (utile nelle run lunghe, es. 0.1)") | |
| ap.add_argument("--warmup", type=int, default=200) | |
| ap.add_argument("--decay-frac", type=float, default=0.2) | |
| ap.add_argument("--eval-every", type=int, default=500) | |
| ap.add_argument("--eval-batches", type=int, default=40) | |
| ap.add_argument("--compile", action="store_true") | |
| ap.add_argument("--seed", type=int, default=0) | |
| ap.add_argument("--save-every", type=int, default=2000, help="checkpoint completo ogni N step (last.pt)") | |
| ap.add_argument("--resume", action="store_true", help="riprendi da <out>/last.pt (stessi --out e argomenti)") | |
| a = ap.parse_args() | |
| if a.compare is not None: | |
| return compare(a.compare or sorted(glob.glob("proxy_runs/*"))) | |
| if a.sample: | |
| return sample(a) | |
| dev = "cuda" if torch.cuda.is_available() else "cpu" | |
| torch.manual_seed(a.seed) | |
| torch.backends.cuda.matmul.allow_tf32 = True | |
| os.makedirs(a.out, exist_ok=True) | |
| model = LM(a.variant, a.vocab, a.d, a.layers, a.heads, a.kv, a.ff, a.seq, | |
| a.gdn_heads, a.gdn_head_dim, a.attn_every, a.mixer).to(dev) | |
| n_par = sum(p.numel() for p in model.parameters()) | |
| kinds = ["A" if isinstance(b.mix, GatedAttention) else "G" for b in model.blocks] # G = layer lineare (gdn/kda/gdn2) | |
| if a.variant == "hybrid": | |
| a.variant = f"hyb-{a.mixer}" | |
| print(f"{a.variant}/{a.opt} | {n_par / 1e6:.1f}M parametri | layer: {''.join(kinds)} (A=attenzione, G=GatedDeltaNet)") | |
| fwd = torch.compile(model) if a.compile else model | |
| train = Mix(a.data_dir, a.seq, a.seed, "train") | |
| val = Mix(a.data_dir, a.seq, 12345, "val") | |
| opts = build_opts(model, a) | |
| tok_step = a.batch_size * a.grad_accum * a.seq | |
| total = int(a.tokens // tok_step) | |
| print(f"{total:,} step × {tok_step:,} token = {total * tok_step / 1e9:.2f}B token", flush=True) | |
| def save_ckpt(path, step): | |
| torch.save({"model": model.state_dict(), "opts": [o.state_dict() for o in opts], "step": step, | |
| "rng": train.rng.bit_generator.state, "torch_rng": torch.get_rng_state(), | |
| "args": vars(a)}, path + ".tmp") | |
| os.replace(path + ".tmp", path) # scrittura atomica: mai un last.pt a metà | |
| start = 0 | |
| last = os.path.join(a.out, "last.pt") | |
| if a.resume and os.path.exists(last): | |
| ck = torch.load(last, map_location=dev, weights_only=False) | |
| model.load_state_dict(ck["model"]) | |
| for o, sd in zip(opts, ck["opts"]): | |
| o.load_state_dict(sd) | |
| for g in o.param_groups: # base_lr non è nello state_dict | |
| g.setdefault("base_lr", a.muon_lr if isinstance(o, Muon) else a.lr) | |
| train.rng.bit_generator.state = ck["rng"] | |
| try: | |
| torch.set_rng_state(ck["torch_rng"].cpu().to(torch.uint8)) | |
| except Exception as e: | |
| print(f"(rng torch non ripristinato: {e})", flush=True) | |
| start = ck["step"] | |
| print(f"↻ ripreso da step {start:,} ({start * tok_step / 1e9:.2f}B token)", flush=True) | |
| log = open(os.path.join(a.out, "log.csv"), "a" if start else "w") | |
| if not start: | |
| log.write("step,tokens,train_loss,val_loss,tok_s\n") | |
| decay_start = int(total * (1 - a.decay_frac)) | |
| t0, tw, run_loss = time.time(), time.time(), 0.0 | |
| for step in range(start, total): | |
| if step == decay_start and step > 0: | |
| save_ckpt(os.path.join(a.out, "pre_decay.pt"), step) # da qui si può allungare il training | |
| print(f" 💾 pre_decay.pt (step {step:,})", flush=True) | |
| m = wsd(step, total, a.warmup, a.decay_frac) | |
| for o in opts: | |
| for g in o.param_groups: | |
| g["lr"] = g["base_lr"] * m | |
| for _ in range(a.grad_accum): | |
| x, y = train.batch(a.batch_size, dev) | |
| with torch.autocast("cuda", torch.bfloat16, enabled=dev == "cuda"): | |
| loss = fwd(x, y) / a.grad_accum | |
| loss.backward() | |
| run_loss += loss.item() | |
| torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| for o in opts: | |
| o.step() | |
| o.zero_grad(set_to_none=True) | |
| if (step + 1) % 50 == 0: | |
| tps = 50 * tok_step / (time.time() - tw) | |
| print(f"step {step + 1:6,}/{total:,} | loss {run_loss / 50:.4f} | lr×{m:.2f} | {tps:,.0f} tok/s", flush=True) | |
| log.write(f"{step + 1},{(step + 1) * tok_step},{run_loss / 50:.5f},,{tps:.0f}\n"); log.flush() | |
| run_loss, tw = 0.0, time.time() | |
| if (step + 1) % a.eval_every == 0 and step + 1 < total: | |
| vl = evaluate(model, val, a.eval_batches // 2, a.batch_size, dev) | |
| print(f" val {vl:.4f}", flush=True) | |
| log.write(f"{step + 1},{(step + 1) * tok_step},,{vl:.5f},\n"); log.flush() | |
| tw = time.time() | |
| if (step + 1) % a.save_every == 0 and step + 1 < total: | |
| save_ckpt(last, step + 1) | |
| tw = time.time() | |
| vl = evaluate(model, val, a.eval_batches, a.batch_size, dev) | |
| hours = (time.time() - t0) / 3600 | |
| tokens_run = (total - start) * tok_step | |
| res = dict(out=a.out, variant=a.variant, opt=a.opt, params=n_par, val_loss=vl, | |
| tokens=total * tok_step, tok_s=tokens_run / max(hours * 3600, 1e-9), hours=hours, args=vars(a)) | |
| json.dump(res, open(os.path.join(a.out, "result.json"), "w"), indent=2) | |
| torch.save({"model": model.state_dict(), "args": vars(a)}, os.path.join(a.out, "model.pt")) | |
| print(f"\n✓ val loss finale {vl:.4f} | {res['tok_s']:,.0f} tok/s | {hours:.2f} h → {a.out}/result.json") | |
| if __name__ == "__main__": | |
| main() | |