File size: 1,631 Bytes
54cb090 402944a 54cb090 402944a 54cb090 402944a 54cb090 402944a 54cb090 402944a 54cb090 402944a 54cb090 402944a 54cb090 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | {
"depth": 12,
"aspect_ratio": 64,
"head_dim": 128,
"max_seq_len": 2048,
"vocab_size": 16384,
"params_total": 185598434,
"params_scaling": 97518000,
"flops_per_token": 811600560,
"unique_train_tokens": 183061543,
"target_tokens_ratio": 1023939000,
"target_tokens_epochs": 732246172,
"horizon": "branch x1.5 of the stable run",
"num_iterations": 3605,
"total_batch_size": 262144,
"device_batch_size": 4,
"world_size": 1,
"device_batch_size_source": "ladder.json device_batch_size_by_vram",
"grad_accum_steps": 32,
"train_tokens": 945029120,
"epochs": 5.16,
"tokens_per_param": 9.69,
"train_flops": 818260411353661440,
"window_pattern": "L",
"attention": "sdpa",
"fp8": false,
"eval_tokens": 3719168,
"eval_every": 90,
"save_every": 180,
"keep_checkpoints": true,
"base_overrides": {},
"warmdown_ratio": 0.2,
"stream": true,
"decay_at": 2884,
"plain_at": 3245,
"loader_tails": true,
"regularization": {
"wd_schedule": "floor",
"wd_floor": 0.5,
"table_wd_mult": 1.0,
"dropout": 0.0,
"dropout_from": 0.5
},
"dropout_from_step": null,
"wd_horizon": 3846,
"save_steps": [],
"seed": null,
"arm": "R1",
"branches": {
"1": {
"decay_at": 1923,
"decay_steps": 481,
"num_iterations": 2404
},
"1.5": {
"decay_at": 2884,
"decay_steps": 721,
"num_iterations": 3605
},
"2": {
"decay_at": 3846,
"decay_steps": 962,
"num_iterations": 4808
}
},
"tag": "queen-v4-r1-x1.5",
"branch": {
"of": "queen-v4-r1-stable",
"at": 2884,
"decay_steps": 721
}
} |