Grace-G1-55M-Instruct / proxy_arch.py
ThingsAI's picture
Upload folder using huggingface_hub
15b55f4 verified
Raw History Blame Contribute Delete
21.6 kB
#!/usr/bin/env python3
"""
proxy_arch.py — confronto di architetture/ottimizzatori su modelli piccoli (~50M) PRIMA di spendere sul cloud.
Varianti:
--variant attn attenzione piena con output gate (stile Qwen3-Next) in tutti i layer
--variant hybrid 3 layer Gated DeltaNet (flash-linear-attention) : 1 layer di attenzione con gate
--opt adamw|muon Muon sulle matrici 2D interne, AdamW su embedding / norme / resto
Stessi dati, stesso ordine dei batch, stesso schedule WSD per tutte le run: cambia solo quello che stai testando.
python3 proxy_arch.py --variant attn --opt adamw --out proxy_runs/A
python3 proxy_arch.py --variant hybrid --opt adamw --out proxy_runs/B
python3 proxy_arch.py --variant attn --opt muon --out proxy_runs/C (o hybrid, quello che vince tra A e B)
python3 proxy_arch.py --compare proxy_runs/* # tabella finale
Legge data-ark35m/meta.json + {nome}.bin / val_{nome}.bin (uint16) come ark35m.py.
"""
import argparse, glob, inspect, json, math, os, sys, time
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
# ── dati ────────────────────────────────────────────────────────────────────────────────────
class Mix:
def __init__(self, data_dir, seq, seed, split="train"):
meta = json.load(open(os.path.join(data_dir, "meta.json")))
self.srcs, ws = [], []
for name, s in meta["sources"].items():
path = os.path.join(data_dir, f"{name}.bin" if split == "train" else f"val_{name}.bin")
if not os.path.exists(path) or os.path.getsize(path) < 2 * (seq + 2):
continue
self.srcs.append(np.memmap(path, dtype=np.uint16, mode="r"))
ws.append(float(s.get("weight", 1.0)))
if not self.srcs:
sys.exit(f"nessun file {split} trovato in {data_dir}")
self.p = np.array(ws) / sum(ws)
self.seq, self.rng = seq, np.random.default_rng(seed)
def batch(self, bs, device):
x = np.empty((bs, self.seq + 1), dtype=np.int64)
for i in range(bs):
m = self.srcs[self.rng.choice(len(self.srcs), p=self.p)]
j = self.rng.integers(0, len(m) - self.seq - 1)
x[i] = m[j:j + self.seq + 1]
t = torch.from_numpy(x).to(device, non_blocking=True)
return t[:, :-1], t[:, 1:]
# ── modello ─────────────────────────────────────────────────────────────────────────────────
class RMSNorm(nn.Module):
def __init__(self, d, eps=1e-6):
super().__init__()
self.w, self.eps = nn.Parameter(torch.ones(d)), eps
def forward(self, x):
return (x.float() * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + self.eps)).type_as(x) * self.w
def rope(x, cos, sin):
x1, x2 = x.chunk(2, -1)
return x * cos + torch.cat((-x2, x1), -1) * sin
class GatedAttention(nn.Module):
"""GQA + QK-norm + RoPE + output gate sigmoide elemento per elemento (prima di o_proj)."""
def __init__(self, d, h, kv):
super().__init__()
self.h, self.kv, self.hd = h, kv, d // h
self.q = nn.Linear(d, h * self.hd, bias=False)
self.k = nn.Linear(d, kv * self.hd, bias=False)
self.v = nn.Linear(d, kv * self.hd, bias=False)
self.g = nn.Linear(d, h * self.hd, bias=False)
self.o = nn.Linear(h * self.hd, d, bias=False)
self.qn, self.kn = RMSNorm(self.hd), RMSNorm(self.hd)
def forward(self, x, cos, sin):
B, T, _ = x.shape
q = self.qn(self.q(x).view(B, T, self.h, self.hd)).transpose(1, 2)
k = self.kn(self.k(x).view(B, T, self.kv, self.hd)).transpose(1, 2)
v = self.v(x).view(B, T, self.kv, self.hd).transpose(1, 2)
q, k = rope(q, cos, sin), rope(k, cos, sin)
rep = self.h // self.kv
y = F.scaled_dot_product_attention(q, k.repeat_interleave(rep, 1), v.repeat_interleave(rep, 1), is_causal=True)
y = y.transpose(1, 2).reshape(B, T, -1) * torch.sigmoid(self.g(x))
return self.o(y)
MIXERS = {"gdn": "GatedDeltaNet", # gate di oblio scalare per testa
"kda": "KimiDeltaAttention", # gate di oblio per canale (use_full_rank_gate di Kimi)
"gdn2": "GatedDeltaNet2"} # + gate per canale anche su cancellazione e scrittura
def make_gdn(d, heads, head_dim, layer_idx, mixer="gdn"):
import fla.layers
cls = getattr(fla.layers, MIXERS[mixer])
want = dict(hidden_size=d, num_heads=heads, head_dim=head_dim, expand_v=2, mode="chunk",
use_gate=True, use_short_conv=True, conv_size=4, layer_idx=layer_idx)
if mixer == "kda":
want["lower_bound"] = -5.0 # come gate_lower_bound di Kimi
ok = inspect.signature(cls.__init__).parameters
return cls(**{k: v for k, v in want.items() if k in ok})
class GDNWrap(nn.Module):
def __init__(self, d, heads, head_dim, idx, mixer="gdn"):
super().__init__()
self.m = make_gdn(d, heads, head_dim, idx, mixer)
def forward(self, x, cos, sin):
y = self.m(x)
return y[0] if isinstance(y, tuple) else y
class SwiGLU(nn.Module):
def __init__(self, d, ff):
super().__init__()
self.a, self.b, self.c = nn.Linear(d, ff, bias=False), nn.Linear(d, ff, bias=False), nn.Linear(ff, d, bias=False)
def forward(self, x):
return self.c(F.silu(self.a(x)) * self.b(x))
class Block(nn.Module):
def __init__(self, mixer, d, ff):
super().__init__()
self.n1, self.n2, self.mix, self.ffn = RMSNorm(d), RMSNorm(d), mixer, SwiGLU(d, ff)
def forward(self, x, cos, sin):
x = x + self.mix(self.n1(x), cos, sin)
return x + self.ffn(self.n2(x))
class LM(nn.Module):
def __init__(self, variant, vocab, d, layers, heads, kv, ff, seq, gdn_heads, gdn_head_dim, attn_every, mixer="gdn"):
super().__init__()
self.emb = nn.Embedding(vocab, d)
blocks = []
for i in range(layers):
if variant == "hybrid" and (i + 1) % attn_every != 0:
mix = GDNWrap(d, gdn_heads, gdn_head_dim, i, mixer)
else:
mix = GatedAttention(d, heads, kv)
blocks.append(Block(mix, d, ff))
self.blocks = nn.ModuleList(blocks)
self.norm = RMSNorm(d)
hd = d // heads
inv = 1.0 / (10000 ** (torch.arange(0, hd, 2).float() / hd))
f = torch.outer(torch.arange(seq).float(), inv)
f = torch.cat((f, f), -1)
self.register_buffer("cos", f.cos()[None, None], persistent=False)
self.register_buffer("sin", f.sin()[None, None], persistent=False)
self.apply(self._init)
for n, p in self.named_parameters(): # residui scalati con la profondità
if n.endswith(("o.weight", "ffn.c.weight")):
nn.init.normal_(p, std=0.02 / math.sqrt(2 * layers))
@staticmethod
def _init(m):
if isinstance(m, nn.Linear):
nn.init.normal_(m.weight, std=0.02)
elif isinstance(m, nn.Embedding):
nn.init.normal_(m.weight, std=0.02)
def forward(self, idx, tgt=None):
T = idx.size(1)
x = self.emb(idx)
cos, sin = self.cos[..., :T, :].to(x.dtype), self.sin[..., :T, :].to(x.dtype)
for b in self.blocks:
x = b(x, cos, sin)
x = self.norm(x)
if tgt is None:
return x @ self.emb.weight.t()
loss = 0.0 # loss a pezzi: logit 32k × 2048 non stanno tutti in 8 GB
xs, ts = x.view(-1, x.size(-1)), tgt.reshape(-1)
for i in range(0, xs.size(0), 4096):
lg = (xs[i:i + 4096] @ self.emb.weight.t()).float()
loss = loss + F.cross_entropy(lg, ts[i:i + 4096], reduction="sum")
return loss / ts.numel()
# ── Muon (Keller Jordan): momentum ortogonalizzato con Newton-Schulz ─────────────────────────
def zeropower_ns5(G, steps=5):
a, b, c = 3.4445, -4.7750, 2.0315
X = G.bfloat16()
tr = X.size(0) > X.size(1)
if tr:
X = X.T
X = X / (X.norm() + 1e-7)
for _ in range(steps):
A = X @ X.T
X = a * X + (b * A + c * A @ A) @ X
return (X.T if tr else X).to(G.dtype)
class Muon(torch.optim.Optimizer):
def __init__(self, params, lr=0.02, momentum=0.95, weight_decay=0.0):
super().__init__(params, dict(lr=lr, momentum=momentum, weight_decay=weight_decay))
@torch.no_grad()
def step(self):
for g in self.param_groups:
for p in g["params"]:
if p.grad is None:
continue
st = self.state[p]
if "buf" not in st:
st["buf"] = torch.zeros_like(p.grad)
buf = st["buf"]
buf.mul_(g["momentum"]).add_(p.grad)
upd = zeropower_ns5(p.grad.add(buf, alpha=g["momentum"])) # Nesterov
scale = max(1.0, p.size(0) / p.size(1)) ** 0.5
if g["weight_decay"]:
p.mul_(1 - g["lr"] * g["weight_decay"])
p.add_(upd, alpha=-g["lr"] * scale)
def build_opts(model, a):
muon_p, adam_dec, adam_nodec = [], [], []
for n, p in model.named_parameters():
if not p.requires_grad:
continue
if a.opt == "muon" and p.dim() == 2 and "emb" not in n:
muon_p.append(p)
elif p.dim() >= 2 and "emb" not in n:
adam_dec.append(p)
else:
adam_nodec.append(p) # norme, embedding, conv/bias piccoli
opts = [torch.optim.AdamW([{"params": adam_dec, "weight_decay": a.wd}, {"params": adam_nodec, "weight_decay": 0.0}],
lr=a.lr, betas=(0.9, 0.95), fused=torch.cuda.is_available())]
if muon_p:
opts.append(Muon(muon_p, lr=a.muon_lr, weight_decay=a.muon_wd))
for o in opts:
for g in o.param_groups:
g["base_lr"] = g["lr"]
return opts
def wsd(step, total, warmup, decay_frac):
if step < warmup:
return (step + 1) / warmup
ds = int(total * (1 - decay_frac))
if step < ds:
return 1.0
return max(0.0, 1 - math.sqrt((step - ds) / max(1, total - ds)))
@torch.no_grad()
def evaluate(model, val, n, bs, dev):
model.eval()
tot = 0.0
for _ in range(n):
x, y = val.batch(bs, dev)
with torch.autocast("cuda", torch.bfloat16, enabled=dev == "cuda"):
tot += model(x, y).item()
model.train()
return tot / n
def compare(dirs):
rows = []
for d in dirs:
f = os.path.join(d, "result.json")
if os.path.exists(f):
rows.append(json.load(open(f)))
if not rows:
sys.exit("nessun result.json trovato")
best = min(r["val_loss"] for r in rows)
print(f"\n{'run':22s} {'variante':9s} {'opt':6s} {'param':>7s} {'val loss':>9s} {'Δ':>7s} {'tok/s':>8s} {'ore':>5s}")
for r in sorted(rows, key=lambda r: r["val_loss"]):
print(f"{os.path.basename(r['out']):22s} {r['variant']:9s} {r['opt']:6s} {r['params'] / 1e6:6.1f}M "
f"{r['val_loss']:9.4f} {r['val_loss'] - best:+7.4f} {r['tok_s']:8,.0f} {r['hours']:5.2f}")
print("\nRegola: differenze di loss < 0.01 sono rumore; un'architettura vince se è meglio in loss SENZA perdere >15% di tok/s.")
@torch.no_grad()
def sample(a):
"""Text completion con un modello salvato (senza KV cache: ricalcola tutto a ogni token, va bene a 55M)."""
from transformers import AutoTokenizer
ck = torch.load(os.path.join(a.sample, "model.pt"), map_location="cpu", weights_only=False)
r = argparse.Namespace(**ck["args"])
dev = "cuda" if torch.cuda.is_available() else "cpu"
variant = "hybrid" if r.variant.startswith("hyb") else r.variant
model = LM(variant, r.vocab, r.d, r.layers, r.heads, r.kv, r.ff, r.seq, r.gdn_heads, r.gdn_head_dim,
r.attn_every, getattr(r, "mixer", "gdn")).to(dev)
model.load_state_dict(ck["model"])
model.eval()
tok = AutoTokenizer.from_pretrained(a.tokenizer, trust_remote_code=True)
prompts = [a.prompt] if a.prompt != "-" else None
while True:
if prompts is None:
try:
p = input("\n>>> ")
except EOFError:
return
if not p.strip():
continue
else:
p = prompts.pop() if prompts else None
if p is None:
return
ids = tok(p, add_special_tokens=False)["input_ids"]
if tok.bos_token_id is not None:
ids = [tok.bos_token_id] + ids
x = torch.tensor([ids], device=dev)
print(p, end="", flush=True)
prev = tok.decode(ids, skip_special_tokens=True)
for _ in range(a.max_new):
with torch.autocast("cuda", torch.bfloat16, enabled=dev == "cuda"):
lg = model(x[:, -r.seq:])[0, -1].float()
if a.rep_penalty != 1.0:
for tid in set(x[0].tolist()):
lg[tid] = lg[tid] / a.rep_penalty if lg[tid] > 0 else lg[tid] * a.rep_penalty
lg = lg / max(a.temp, 1e-5)
v, i = lg.topk(a.top_k)
nxt = i[torch.multinomial(torch.softmax(v, -1), 1)]
if nxt.item() == tok.eos_token_id:
break
x = torch.cat([x, nxt.view(1, 1)], 1)
cur = tok.decode(x[0].tolist(), skip_special_tokens=True)
print(cur[len(prev):], end="", flush=True)
prev = cur
print()
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--compare", nargs="*")
ap.add_argument("--sample", default=None, help="cartella di una run con model.pt: genera testo")
ap.add_argument("--prompt", default="La capitale d'Italia è")
ap.add_argument("--tokenizer", default="clover-tokenizer")
ap.add_argument("--max-new", type=int, default=80)
ap.add_argument("--temp", type=float, default=0.7)
ap.add_argument("--top-k", type=int, default=40)
ap.add_argument("--rep-penalty", type=float, default=1.3)
ap.add_argument("--variant", choices=["attn", "hybrid"], default="attn")
ap.add_argument("--opt", choices=["adamw", "muon"], default="adamw")
ap.add_argument("--data-dir", default="data-ark35m")
ap.add_argument("--out", default="proxy_runs/run")
ap.add_argument("--tokens", type=float, default=5e8, help="token di training (0.5B ≈ 2-3 h su 3070 a 50M)")
ap.add_argument("--seq", type=int, default=2048)
ap.add_argument("--batch-size", type=int, default=4)
ap.add_argument("--grad-accum", type=int, default=8)
ap.add_argument("--d", type=int, default=512)
ap.add_argument("--layers", type=int, default=12)
ap.add_argument("--heads", type=int, default=8)
ap.add_argument("--kv", type=int, default=2)
ap.add_argument("--ff", type=int, default=1408)
ap.add_argument("--gdn-heads", type=int, default=4)
ap.add_argument("--gdn-head-dim", type=int, default=64)
ap.add_argument("--mixer", choices=list(MIXERS), default="gdn", help="hybrid: gdn | kda | gdn2")
ap.add_argument("--attn-every", type=int, default=4, help="hybrid: un layer di attenzione ogni N")
ap.add_argument("--vocab", type=int, default=32768)
ap.add_argument("--lr", type=float, default=1.5e-3)
ap.add_argument("--muon-lr", type=float, default=0.02)
ap.add_argument("--wd", type=float, default=0.1)
ap.add_argument("--muon-wd", type=float, default=0.0, help="weight decay su Muon (utile nelle run lunghe, es. 0.1)")
ap.add_argument("--warmup", type=int, default=200)
ap.add_argument("--decay-frac", type=float, default=0.2)
ap.add_argument("--eval-every", type=int, default=500)
ap.add_argument("--eval-batches", type=int, default=40)
ap.add_argument("--compile", action="store_true")
ap.add_argument("--seed", type=int, default=0)
ap.add_argument("--save-every", type=int, default=2000, help="checkpoint completo ogni N step (last.pt)")
ap.add_argument("--resume", action="store_true", help="riprendi da <out>/last.pt (stessi --out e argomenti)")
a = ap.parse_args()
if a.compare is not None:
return compare(a.compare or sorted(glob.glob("proxy_runs/*")))
if a.sample:
return sample(a)
dev = "cuda" if torch.cuda.is_available() else "cpu"
torch.manual_seed(a.seed)
torch.backends.cuda.matmul.allow_tf32 = True
os.makedirs(a.out, exist_ok=True)
model = LM(a.variant, a.vocab, a.d, a.layers, a.heads, a.kv, a.ff, a.seq,
a.gdn_heads, a.gdn_head_dim, a.attn_every, a.mixer).to(dev)
n_par = sum(p.numel() for p in model.parameters())
kinds = ["A" if isinstance(b.mix, GatedAttention) else "G" for b in model.blocks] # G = layer lineare (gdn/kda/gdn2)
if a.variant == "hybrid":
a.variant = f"hyb-{a.mixer}"
print(f"{a.variant}/{a.opt} | {n_par / 1e6:.1f}M parametri | layer: {''.join(kinds)} (A=attenzione, G=GatedDeltaNet)")
fwd = torch.compile(model) if a.compile else model
train = Mix(a.data_dir, a.seq, a.seed, "train")
val = Mix(a.data_dir, a.seq, 12345, "val")
opts = build_opts(model, a)
tok_step = a.batch_size * a.grad_accum * a.seq
total = int(a.tokens // tok_step)
print(f"{total:,} step × {tok_step:,} token = {total * tok_step / 1e9:.2f}B token", flush=True)
def save_ckpt(path, step):
torch.save({"model": model.state_dict(), "opts": [o.state_dict() for o in opts], "step": step,
"rng": train.rng.bit_generator.state, "torch_rng": torch.get_rng_state(),
"args": vars(a)}, path + ".tmp")
os.replace(path + ".tmp", path) # scrittura atomica: mai un last.pt a metà
start = 0
last = os.path.join(a.out, "last.pt")
if a.resume and os.path.exists(last):
ck = torch.load(last, map_location=dev, weights_only=False)
model.load_state_dict(ck["model"])
for o, sd in zip(opts, ck["opts"]):
o.load_state_dict(sd)
for g in o.param_groups: # base_lr non è nello state_dict
g.setdefault("base_lr", a.muon_lr if isinstance(o, Muon) else a.lr)
train.rng.bit_generator.state = ck["rng"]
try:
torch.set_rng_state(ck["torch_rng"].cpu().to(torch.uint8))
except Exception as e:
print(f"(rng torch non ripristinato: {e})", flush=True)
start = ck["step"]
print(f"↻ ripreso da step {start:,} ({start * tok_step / 1e9:.2f}B token)", flush=True)
log = open(os.path.join(a.out, "log.csv"), "a" if start else "w")
if not start:
log.write("step,tokens,train_loss,val_loss,tok_s\n")
decay_start = int(total * (1 - a.decay_frac))
t0, tw, run_loss = time.time(), time.time(), 0.0
for step in range(start, total):
if step == decay_start and step > 0:
save_ckpt(os.path.join(a.out, "pre_decay.pt"), step) # da qui si può allungare il training
print(f" 💾 pre_decay.pt (step {step:,})", flush=True)
m = wsd(step, total, a.warmup, a.decay_frac)
for o in opts:
for g in o.param_groups:
g["lr"] = g["base_lr"] * m
for _ in range(a.grad_accum):
x, y = train.batch(a.batch_size, dev)
with torch.autocast("cuda", torch.bfloat16, enabled=dev == "cuda"):
loss = fwd(x, y) / a.grad_accum
loss.backward()
run_loss += loss.item()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
for o in opts:
o.step()
o.zero_grad(set_to_none=True)
if (step + 1) % 50 == 0:
tps = 50 * tok_step / (time.time() - tw)
print(f"step {step + 1:6,}/{total:,} | loss {run_loss / 50:.4f} | lr×{m:.2f} | {tps:,.0f} tok/s", flush=True)
log.write(f"{step + 1},{(step + 1) * tok_step},{run_loss / 50:.5f},,{tps:.0f}\n"); log.flush()
run_loss, tw = 0.0, time.time()
if (step + 1) % a.eval_every == 0 and step + 1 < total:
vl = evaluate(model, val, a.eval_batches // 2, a.batch_size, dev)
print(f" val {vl:.4f}", flush=True)
log.write(f"{step + 1},{(step + 1) * tok_step},,{vl:.5f},\n"); log.flush()
tw = time.time()
if (step + 1) % a.save_every == 0 and step + 1 < total:
save_ckpt(last, step + 1)
tw = time.time()
vl = evaluate(model, val, a.eval_batches, a.batch_size, dev)
hours = (time.time() - t0) / 3600
tokens_run = (total - start) * tok_step
res = dict(out=a.out, variant=a.variant, opt=a.opt, params=n_par, val_loss=vl,
tokens=total * tok_step, tok_s=tokens_run / max(hours * 3600, 1e-9), hours=hours, args=vars(a))
json.dump(res, open(os.path.join(a.out, "result.json"), "w"), indent=2)
torch.save({"model": model.state_dict(), "args": vars(a)}, os.path.join(a.out, "model.pt"))
print(f"\n✓ val loss finale {vl:.4f} | {res['tok_s']:,.0f} tok/s | {hours:.2f} h → {a.out}/result.json")
if __name__ == "__main__":
main()