| { | |
| "step": 380, | |
| "loss": 6.184170246124268, | |
| "cfg": { | |
| "vocab_size": 248320, | |
| "hidden_size": 1536, | |
| "n_layers": 21, | |
| "n_heads": 12, | |
| "n_kv_heads": 4, | |
| "linear_heads": 12, | |
| "head_dim": 128, | |
| "ffn_hidden": 4096, | |
| "hybrid_ratio": 3, | |
| "linear": "gdn", | |
| "rms_norm_eps": 1e-06, | |
| "tie_embeddings": true, | |
| "conv_size": 4, | |
| "expand_v": 1.0, | |
| "use_fla": false, | |
| "logit_softcap": 15.0, | |
| "mtp": 1 | |
| }, | |
| "pass": "sft" | |
| } |