File size: 1,430 Bytes
1a4f94d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | {
"depth": 20,
"aspect_ratio": 64,
"head_dim": 128,
"max_seq_len": 2048,
"vocab_size": 16384,
"params_total": 644875506,
"params_scaling": 414188720,
"flops_per_token": 3114278064,
"unique_train_tokens": 421945781,
"target_tokens_ratio": 4348981560,
"target_tokens_epochs": 1687783124,
"horizon": "epochs",
"num_iterations": 4462,
"total_batch_size": 524288,
"device_batch_size": 32,
"world_size": 4,
"device_batch_size_source": "ladder.json device_batch_size",
"grad_accum_steps": 2,
"train_tokens": 2339373056,
"epochs": 5.54,
"tokens_per_param": 5.65,
"train_flops": 7285458191813443584,
"window_pattern": "SSSL",
"attention": "fa3",
"fp8": false,
"eval_tokens": 6291456,
"eval_every": 223,
"save_every": 402,
"keep_checkpoints": true,
"base_overrides": {},
"warmdown_ratio": 0.199708,
"stream": true,
"decay_at": 3571,
"plain_at": 4016,
"loader_tails": true,
"regularization": {
"wd_schedule": "floor",
"wd_floor": 0.5,
"table_wd_mult": 1.0,
"dropout": 0.0,
"dropout_from": 0.5
},
"dropout_from_step": null,
"weight_decay": {
"horizon": "trained",
"base": 0.28,
"nanochat_scaled": 0.065924,
"scaled": 0.065924,
"decay_per_step": 0.001318,
"ema_tokens": 397644859
},
"wd_horizon": null,
"save_steps": [],
"seed": null,
"arm": null,
"device_type": "cuda",
"frozen_at": "2026-10-11T01:36:55"
} |