#!/usr/bin/env python3 """ proxy_arch.py — confronto di architetture/ottimizzatori su modelli piccoli (~50M) PRIMA di spendere sul cloud. Varianti: --variant attn attenzione piena con output gate (stile Qwen3-Next) in tutti i layer --variant hybrid 3 layer Gated DeltaNet (flash-linear-attention) : 1 layer di attenzione con gate --opt adamw|muon Muon sulle matrici 2D interne, AdamW su embedding / norme / resto Stessi dati, stesso ordine dei batch, stesso schedule WSD per tutte le run: cambia solo quello che stai testando. python3 proxy_arch.py --variant attn --opt adamw --out proxy_runs/A python3 proxy_arch.py --variant hybrid --opt adamw --out proxy_runs/B python3 proxy_arch.py --variant attn --opt muon --out proxy_runs/C (o hybrid, quello che vince tra A e B) python3 proxy_arch.py --compare proxy_runs/* # tabella finale Legge data-ark35m/meta.json + {nome}.bin / val_{nome}.bin (uint16) come ark35m.py. """ import argparse, glob, inspect, json, math, os, sys, time import numpy as np import torch import torch.nn as nn import torch.nn.functional as F # ── dati ──────────────────────────────────────────────────────────────────────────────────── class Mix: def __init__(self, data_dir, seq, seed, split="train"): meta = json.load(open(os.path.join(data_dir, "meta.json"))) self.srcs, ws = [], [] for name, s in meta["sources"].items(): path = os.path.join(data_dir, f"{name}.bin" if split == "train" else f"val_{name}.bin") if not os.path.exists(path) or os.path.getsize(path) < 2 * (seq + 2): continue self.srcs.append(np.memmap(path, dtype=np.uint16, mode="r")) ws.append(float(s.get("weight", 1.0))) if not self.srcs: sys.exit(f"nessun file {split} trovato in {data_dir}") self.p = np.array(ws) / sum(ws) self.seq, self.rng = seq, np.random.default_rng(seed) def batch(self, bs, device): x = np.empty((bs, self.seq + 1), dtype=np.int64) for i in range(bs): m = self.srcs[self.rng.choice(len(self.srcs), p=self.p)] j = self.rng.integers(0, len(m) - self.seq - 1) x[i] = m[j:j + self.seq + 1] t = torch.from_numpy(x).to(device, non_blocking=True) return t[:, :-1], t[:, 1:] # ── modello ───────────────────────────────────────────────────────────────────────────────── class RMSNorm(nn.Module): def __init__(self, d, eps=1e-6): super().__init__() self.w, self.eps = nn.Parameter(torch.ones(d)), eps def forward(self, x): return (x.float() * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + self.eps)).type_as(x) * self.w def rope(x, cos, sin): x1, x2 = x.chunk(2, -1) return x * cos + torch.cat((-x2, x1), -1) * sin class GatedAttention(nn.Module): """GQA + QK-norm + RoPE + output gate sigmoide elemento per elemento (prima di o_proj).""" def __init__(self, d, h, kv): super().__init__() self.h, self.kv, self.hd = h, kv, d // h self.q = nn.Linear(d, h * self.hd, bias=False) self.k = nn.Linear(d, kv * self.hd, bias=False) self.v = nn.Linear(d, kv * self.hd, bias=False) self.g = nn.Linear(d, h * self.hd, bias=False) self.o = nn.Linear(h * self.hd, d, bias=False) self.qn, self.kn = RMSNorm(self.hd), RMSNorm(self.hd) def forward(self, x, cos, sin): B, T, _ = x.shape q = self.qn(self.q(x).view(B, T, self.h, self.hd)).transpose(1, 2) k = self.kn(self.k(x).view(B, T, self.kv, self.hd)).transpose(1, 2) v = self.v(x).view(B, T, self.kv, self.hd).transpose(1, 2) q, k = rope(q, cos, sin), rope(k, cos, sin) rep = self.h // self.kv y = F.scaled_dot_product_attention(q, k.repeat_interleave(rep, 1), v.repeat_interleave(rep, 1), is_causal=True) y = y.transpose(1, 2).reshape(B, T, -1) * torch.sigmoid(self.g(x)) return self.o(y) MIXERS = {"gdn": "GatedDeltaNet", # gate di oblio scalare per testa "kda": "KimiDeltaAttention", # gate di oblio per canale (use_full_rank_gate di Kimi) "gdn2": "GatedDeltaNet2"} # + gate per canale anche su cancellazione e scrittura def make_gdn(d, heads, head_dim, layer_idx, mixer="gdn"): import fla.layers cls = getattr(fla.layers, MIXERS[mixer]) want = dict(hidden_size=d, num_heads=heads, head_dim=head_dim, expand_v=2, mode="chunk", use_gate=True, use_short_conv=True, conv_size=4, layer_idx=layer_idx) if mixer == "kda": want["lower_bound"] = -5.0 # come gate_lower_bound di Kimi ok = inspect.signature(cls.__init__).parameters return cls(**{k: v for k, v in want.items() if k in ok}) class GDNWrap(nn.Module): def __init__(self, d, heads, head_dim, idx, mixer="gdn"): super().__init__() self.m = make_gdn(d, heads, head_dim, idx, mixer) def forward(self, x, cos, sin): y = self.m(x) return y[0] if isinstance(y, tuple) else y class SwiGLU(nn.Module): def __init__(self, d, ff): super().__init__() self.a, self.b, self.c = nn.Linear(d, ff, bias=False), nn.Linear(d, ff, bias=False), nn.Linear(ff, d, bias=False) def forward(self, x): return self.c(F.silu(self.a(x)) * self.b(x)) class Block(nn.Module): def __init__(self, mixer, d, ff): super().__init__() self.n1, self.n2, self.mix, self.ffn = RMSNorm(d), RMSNorm(d), mixer, SwiGLU(d, ff) def forward(self, x, cos, sin): x = x + self.mix(self.n1(x), cos, sin) return x + self.ffn(self.n2(x)) class LM(nn.Module): def __init__(self, variant, vocab, d, layers, heads, kv, ff, seq, gdn_heads, gdn_head_dim, attn_every, mixer="gdn"): super().__init__() self.emb = nn.Embedding(vocab, d) blocks = [] for i in range(layers): if variant == "hybrid" and (i + 1) % attn_every != 0: mix = GDNWrap(d, gdn_heads, gdn_head_dim, i, mixer) else: mix = GatedAttention(d, heads, kv) blocks.append(Block(mix, d, ff)) self.blocks = nn.ModuleList(blocks) self.norm = RMSNorm(d) hd = d // heads inv = 1.0 / (10000 ** (torch.arange(0, hd, 2).float() / hd)) f = torch.outer(torch.arange(seq).float(), inv) f = torch.cat((f, f), -1) self.register_buffer("cos", f.cos()[None, None], persistent=False) self.register_buffer("sin", f.sin()[None, None], persistent=False) self.apply(self._init) for n, p in self.named_parameters(): # residui scalati con la profondità if n.endswith(("o.weight", "ffn.c.weight")): nn.init.normal_(p, std=0.02 / math.sqrt(2 * layers)) @staticmethod def _init(m): if isinstance(m, nn.Linear): nn.init.normal_(m.weight, std=0.02) elif isinstance(m, nn.Embedding): nn.init.normal_(m.weight, std=0.02) def forward(self, idx, tgt=None): T = idx.size(1) x = self.emb(idx) cos, sin = self.cos[..., :T, :].to(x.dtype), self.sin[..., :T, :].to(x.dtype) for b in self.blocks: x = b(x, cos, sin) x = self.norm(x) if tgt is None: return x @ self.emb.weight.t() loss = 0.0 # loss a pezzi: logit 32k × 2048 non stanno tutti in 8 GB xs, ts = x.view(-1, x.size(-1)), tgt.reshape(-1) for i in range(0, xs.size(0), 4096): lg = (xs[i:i + 4096] @ self.emb.weight.t()).float() loss = loss + F.cross_entropy(lg, ts[i:i + 4096], reduction="sum") return loss / ts.numel() # ── Muon (Keller Jordan): momentum ortogonalizzato con Newton-Schulz ───────────────────────── def zeropower_ns5(G, steps=5): a, b, c = 3.4445, -4.7750, 2.0315 X = G.bfloat16() tr = X.size(0) > X.size(1) if tr: X = X.T X = X / (X.norm() + 1e-7) for _ in range(steps): A = X @ X.T X = a * X + (b * A + c * A @ A) @ X return (X.T if tr else X).to(G.dtype) class Muon(torch.optim.Optimizer): def __init__(self, params, lr=0.02, momentum=0.95, weight_decay=0.0): super().__init__(params, dict(lr=lr, momentum=momentum, weight_decay=weight_decay)) @torch.no_grad() def step(self): for g in self.param_groups: for p in g["params"]: if p.grad is None: continue st = self.state[p] if "buf" not in st: st["buf"] = torch.zeros_like(p.grad) buf = st["buf"] buf.mul_(g["momentum"]).add_(p.grad) upd = zeropower_ns5(p.grad.add(buf, alpha=g["momentum"])) # Nesterov scale = max(1.0, p.size(0) / p.size(1)) ** 0.5 if g["weight_decay"]: p.mul_(1 - g["lr"] * g["weight_decay"]) p.add_(upd, alpha=-g["lr"] * scale) def build_opts(model, a): muon_p, adam_dec, adam_nodec = [], [], [] for n, p in model.named_parameters(): if not p.requires_grad: continue if a.opt == "muon" and p.dim() == 2 and "emb" not in n: muon_p.append(p) elif p.dim() >= 2 and "emb" not in n: adam_dec.append(p) else: adam_nodec.append(p) # norme, embedding, conv/bias piccoli opts = [torch.optim.AdamW([{"params": adam_dec, "weight_decay": a.wd}, {"params": adam_nodec, "weight_decay": 0.0}], lr=a.lr, betas=(0.9, 0.95), fused=torch.cuda.is_available())] if muon_p: opts.append(Muon(muon_p, lr=a.muon_lr, weight_decay=a.muon_wd)) for o in opts: for g in o.param_groups: g["base_lr"] = g["lr"] return opts def wsd(step, total, warmup, decay_frac): if step < warmup: return (step + 1) / warmup ds = int(total * (1 - decay_frac)) if step < ds: return 1.0 return max(0.0, 1 - math.sqrt((step - ds) / max(1, total - ds))) @torch.no_grad() def evaluate(model, val, n, bs, dev): model.eval() tot = 0.0 for _ in range(n): x, y = val.batch(bs, dev) with torch.autocast("cuda", torch.bfloat16, enabled=dev == "cuda"): tot += model(x, y).item() model.train() return tot / n def compare(dirs): rows = [] for d in dirs: f = os.path.join(d, "result.json") if os.path.exists(f): rows.append(json.load(open(f))) if not rows: sys.exit("nessun result.json trovato") best = min(r["val_loss"] for r in rows) print(f"\n{'run':22s} {'variante':9s} {'opt':6s} {'param':>7s} {'val loss':>9s} {'Δ':>7s} {'tok/s':>8s} {'ore':>5s}") for r in sorted(rows, key=lambda r: r["val_loss"]): print(f"{os.path.basename(r['out']):22s} {r['variant']:9s} {r['opt']:6s} {r['params'] / 1e6:6.1f}M " f"{r['val_loss']:9.4f} {r['val_loss'] - best:+7.4f} {r['tok_s']:8,.0f} {r['hours']:5.2f}") print("\nRegola: differenze di loss < 0.01 sono rumore; un'architettura vince se è meglio in loss SENZA perdere >15% di tok/s.") @torch.no_grad() def sample(a): """Text completion con un modello salvato (senza KV cache: ricalcola tutto a ogni token, va bene a 55M).""" from transformers import AutoTokenizer ck = torch.load(os.path.join(a.sample, "model.pt"), map_location="cpu", weights_only=False) r = argparse.Namespace(**ck["args"]) dev = "cuda" if torch.cuda.is_available() else "cpu" variant = "hybrid" if r.variant.startswith("hyb") else r.variant model = LM(variant, r.vocab, r.d, r.layers, r.heads, r.kv, r.ff, r.seq, r.gdn_heads, r.gdn_head_dim, r.attn_every, getattr(r, "mixer", "gdn")).to(dev) model.load_state_dict(ck["model"]) model.eval() tok = AutoTokenizer.from_pretrained(a.tokenizer, trust_remote_code=True) prompts = [a.prompt] if a.prompt != "-" else None while True: if prompts is None: try: p = input("\n>>> ") except EOFError: return if not p.strip(): continue else: p = prompts.pop() if prompts else None if p is None: return ids = tok(p, add_special_tokens=False)["input_ids"] if tok.bos_token_id is not None: ids = [tok.bos_token_id] + ids x = torch.tensor([ids], device=dev) print(p, end="", flush=True) prev = tok.decode(ids, skip_special_tokens=True) for _ in range(a.max_new): with torch.autocast("cuda", torch.bfloat16, enabled=dev == "cuda"): lg = model(x[:, -r.seq:])[0, -1].float() if a.rep_penalty != 1.0: for tid in set(x[0].tolist()): lg[tid] = lg[tid] / a.rep_penalty if lg[tid] > 0 else lg[tid] * a.rep_penalty lg = lg / max(a.temp, 1e-5) v, i = lg.topk(a.top_k) nxt = i[torch.multinomial(torch.softmax(v, -1), 1)] if nxt.item() == tok.eos_token_id: break x = torch.cat([x, nxt.view(1, 1)], 1) cur = tok.decode(x[0].tolist(), skip_special_tokens=True) print(cur[len(prev):], end="", flush=True) prev = cur print() def main(): ap = argparse.ArgumentParser() ap.add_argument("--compare", nargs="*") ap.add_argument("--sample", default=None, help="cartella di una run con model.pt: genera testo") ap.add_argument("--prompt", default="La capitale d'Italia è") ap.add_argument("--tokenizer", default="clover-tokenizer") ap.add_argument("--max-new", type=int, default=80) ap.add_argument("--temp", type=float, default=0.7) ap.add_argument("--top-k", type=int, default=40) ap.add_argument("--rep-penalty", type=float, default=1.3) ap.add_argument("--variant", choices=["attn", "hybrid"], default="attn") ap.add_argument("--opt", choices=["adamw", "muon"], default="adamw") ap.add_argument("--data-dir", default="data-ark35m") ap.add_argument("--out", default="proxy_runs/run") ap.add_argument("--tokens", type=float, default=5e8, help="token di training (0.5B ≈ 2-3 h su 3070 a 50M)") ap.add_argument("--seq", type=int, default=2048) ap.add_argument("--batch-size", type=int, default=4) ap.add_argument("--grad-accum", type=int, default=8) ap.add_argument("--d", type=int, default=512) ap.add_argument("--layers", type=int, default=12) ap.add_argument("--heads", type=int, default=8) ap.add_argument("--kv", type=int, default=2) ap.add_argument("--ff", type=int, default=1408) ap.add_argument("--gdn-heads", type=int, default=4) ap.add_argument("--gdn-head-dim", type=int, default=64) ap.add_argument("--mixer", choices=list(MIXERS), default="gdn", help="hybrid: gdn | kda | gdn2") ap.add_argument("--attn-every", type=int, default=4, help="hybrid: un layer di attenzione ogni N") ap.add_argument("--vocab", type=int, default=32768) ap.add_argument("--lr", type=float, default=1.5e-3) ap.add_argument("--muon-lr", type=float, default=0.02) ap.add_argument("--wd", type=float, default=0.1) ap.add_argument("--muon-wd", type=float, default=0.0, help="weight decay su Muon (utile nelle run lunghe, es. 0.1)") ap.add_argument("--warmup", type=int, default=200) ap.add_argument("--decay-frac", type=float, default=0.2) ap.add_argument("--eval-every", type=int, default=500) ap.add_argument("--eval-batches", type=int, default=40) ap.add_argument("--compile", action="store_true") ap.add_argument("--seed", type=int, default=0) ap.add_argument("--save-every", type=int, default=2000, help="checkpoint completo ogni N step (last.pt)") ap.add_argument("--resume", action="store_true", help="riprendi da /last.pt (stessi --out e argomenti)") a = ap.parse_args() if a.compare is not None: return compare(a.compare or sorted(glob.glob("proxy_runs/*"))) if a.sample: return sample(a) dev = "cuda" if torch.cuda.is_available() else "cpu" torch.manual_seed(a.seed) torch.backends.cuda.matmul.allow_tf32 = True os.makedirs(a.out, exist_ok=True) model = LM(a.variant, a.vocab, a.d, a.layers, a.heads, a.kv, a.ff, a.seq, a.gdn_heads, a.gdn_head_dim, a.attn_every, a.mixer).to(dev) n_par = sum(p.numel() for p in model.parameters()) kinds = ["A" if isinstance(b.mix, GatedAttention) else "G" for b in model.blocks] # G = layer lineare (gdn/kda/gdn2) if a.variant == "hybrid": a.variant = f"hyb-{a.mixer}" print(f"{a.variant}/{a.opt} | {n_par / 1e6:.1f}M parametri | layer: {''.join(kinds)} (A=attenzione, G=GatedDeltaNet)") fwd = torch.compile(model) if a.compile else model train = Mix(a.data_dir, a.seq, a.seed, "train") val = Mix(a.data_dir, a.seq, 12345, "val") opts = build_opts(model, a) tok_step = a.batch_size * a.grad_accum * a.seq total = int(a.tokens // tok_step) print(f"{total:,} step × {tok_step:,} token = {total * tok_step / 1e9:.2f}B token", flush=True) def save_ckpt(path, step): torch.save({"model": model.state_dict(), "opts": [o.state_dict() for o in opts], "step": step, "rng": train.rng.bit_generator.state, "torch_rng": torch.get_rng_state(), "args": vars(a)}, path + ".tmp") os.replace(path + ".tmp", path) # scrittura atomica: mai un last.pt a metà start = 0 last = os.path.join(a.out, "last.pt") if a.resume and os.path.exists(last): ck = torch.load(last, map_location=dev, weights_only=False) model.load_state_dict(ck["model"]) for o, sd in zip(opts, ck["opts"]): o.load_state_dict(sd) for g in o.param_groups: # base_lr non è nello state_dict g.setdefault("base_lr", a.muon_lr if isinstance(o, Muon) else a.lr) train.rng.bit_generator.state = ck["rng"] try: torch.set_rng_state(ck["torch_rng"].cpu().to(torch.uint8)) except Exception as e: print(f"(rng torch non ripristinato: {e})", flush=True) start = ck["step"] print(f"↻ ripreso da step {start:,} ({start * tok_step / 1e9:.2f}B token)", flush=True) log = open(os.path.join(a.out, "log.csv"), "a" if start else "w") if not start: log.write("step,tokens,train_loss,val_loss,tok_s\n") decay_start = int(total * (1 - a.decay_frac)) t0, tw, run_loss = time.time(), time.time(), 0.0 for step in range(start, total): if step == decay_start and step > 0: save_ckpt(os.path.join(a.out, "pre_decay.pt"), step) # da qui si può allungare il training print(f" 💾 pre_decay.pt (step {step:,})", flush=True) m = wsd(step, total, a.warmup, a.decay_frac) for o in opts: for g in o.param_groups: g["lr"] = g["base_lr"] * m for _ in range(a.grad_accum): x, y = train.batch(a.batch_size, dev) with torch.autocast("cuda", torch.bfloat16, enabled=dev == "cuda"): loss = fwd(x, y) / a.grad_accum loss.backward() run_loss += loss.item() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) for o in opts: o.step() o.zero_grad(set_to_none=True) if (step + 1) % 50 == 0: tps = 50 * tok_step / (time.time() - tw) print(f"step {step + 1:6,}/{total:,} | loss {run_loss / 50:.4f} | lr×{m:.2f} | {tps:,.0f} tok/s", flush=True) log.write(f"{step + 1},{(step + 1) * tok_step},{run_loss / 50:.5f},,{tps:.0f}\n"); log.flush() run_loss, tw = 0.0, time.time() if (step + 1) % a.eval_every == 0 and step + 1 < total: vl = evaluate(model, val, a.eval_batches // 2, a.batch_size, dev) print(f" val {vl:.4f}", flush=True) log.write(f"{step + 1},{(step + 1) * tok_step},,{vl:.5f},\n"); log.flush() tw = time.time() if (step + 1) % a.save_every == 0 and step + 1 < total: save_ckpt(last, step + 1) tw = time.time() vl = evaluate(model, val, a.eval_batches, a.batch_size, dev) hours = (time.time() - t0) / 3600 tokens_run = (total - start) * tok_step res = dict(out=a.out, variant=a.variant, opt=a.opt, params=n_par, val_loss=vl, tokens=total * tok_step, tok_s=tokens_run / max(hours * 3600, 1e-9), hours=hours, args=vars(a)) json.dump(res, open(os.path.join(a.out, "result.json"), "w"), indent=2) torch.save({"model": model.state_dict(), "args": vars(a)}, os.path.join(a.out, "model.pt")) print(f"\n✓ val loss finale {vl:.4f} | {res['tok_s']:,.0f} tok/s | {hours:.2f} h → {a.out}/result.json") if __name__ == "__main__": main()