Grace-G1-55M-Instruct / config.json
ThingsAI's picture
Upload folder using huggingface_hub
15b55f4 verified
Raw History Blame Contribute Delete
824 Bytes
{
"compare": null,
"sample": null,
"prompt": "La capitale d'Italia \u00e8",
"tokenizer": "clover-tokenizer",
"max_new": 80,
"temp": 0.7,
"top_k": 40,
"rep_penalty": 1.3,
"variant": "hyb-gdn",
"opt": "muon",
"data_dir": "data-ark35m",
"out": "runs/grace55m",
"tokens": 10000000000.0,
"seq": 2048,
"batch_size": 4,
"grad_accum": 8,
"d": 512,
"layers": 12,
"heads": 8,
"kv": 2,
"ff": 1408,
"gdn_heads": 4,
"gdn_head_dim": 64,
"mixer": "gdn",
"attn_every": 4,
"vocab": 32768,
"lr": 0.0015,
"muon_lr": 0.02,
"wd": 0.1,
"muon_wd": 0.1,
"warmup": 200,
"decay_frac": 0.2,
"eval_every": 2000,
"eval_batches": 40,
"compile": false,
"seed": 0,
"save_every": 2000,
"resume": true,
"params": 55007304,
"architecture": "GraceHybridGDN",
"val_loss": "n/d"
}