| { |
| "depth": 20, |
| "aspect_ratio": 64, |
| "head_dim": 128, |
| "max_seq_len": 2048, |
| "vocab_size": 16384, |
| "params_total": 644875506, |
| "params_scaling": 414188720, |
| "flops_per_token": 3114278064, |
| "unique_train_tokens": 421945781, |
| "target_tokens_ratio": 4348981560, |
| "target_tokens_epochs": 1687783124, |
| "horizon": "epochs", |
| "num_iterations": 4462, |
| "total_batch_size": 524288, |
| "device_batch_size": 32, |
| "world_size": 4, |
| "device_batch_size_source": "ladder.json device_batch_size", |
| "grad_accum_steps": 2, |
| "train_tokens": 2339373056, |
| "epochs": 5.54, |
| "tokens_per_param": 5.65, |
| "train_flops": 7285458191813443584, |
| "window_pattern": "SSSL", |
| "attention": "fa3", |
| "fp8": false, |
| "eval_tokens": 6291456, |
| "eval_every": 223, |
| "save_every": 402, |
| "keep_checkpoints": true, |
| "base_overrides": {}, |
| "warmdown_ratio": 0.199708, |
| "stream": true, |
| "decay_at": 3571, |
| "plain_at": 4016, |
| "loader_tails": true, |
| "regularization": { |
| "wd_schedule": "floor", |
| "wd_floor": 0.5, |
| "table_wd_mult": 1.0, |
| "dropout": 0.0, |
| "dropout_from": 0.5 |
| }, |
| "dropout_from_step": null, |
| "weight_decay": { |
| "horizon": "trained", |
| "base": 0.28, |
| "nanochat_scaled": 0.065924, |
| "scaled": 0.065924, |
| "decay_per_step": 0.001318, |
| "ema_tokens": 397644859 |
| }, |
| "wd_horizon": null, |
| "save_steps": [], |
| "seed": null, |
| "arm": null, |
| "device_type": "cuda", |
| "frozen_at": "2026-10-11T01:36:55" |
| } |
