| { |
| "best_val_loss": 2.5563359551876785, |
| "ema": 2.4005908938776055, |
| "initialization": "normal-0.02-residual-scaled-v1", |
| "lr_scheduler": { |
| "bad_evals": 0, |
| "best": 2.5563359551876785, |
| "cap": 3.221225472000002e-05, |
| "config_fingerprint": "4f636b3a8d4804bd7b10b26b34967c380d566268898be845bf324f8df4e1459f", |
| "cooldown_remaining": 0, |
| "evaluations": 242, |
| "format": "sml-v2-plateau-v1", |
| "last_loss": 2.5563359551876785, |
| "last_step": 132005, |
| "last_tokens": 17300111360, |
| "reductions": 9 |
| }, |
| "recipe": { |
| "batch_rebalance_transition": { |
| "format": "sml-research-batch-rebalance-v1", |
| "parent_batches": [ |
| 6, |
| 4, |
| 2, |
| 2, |
| 2 |
| ], |
| "parent_contract": { |
| "code": "2c7e9a69271f9817017c2371ae5e874c9229da489b6c0128aadda943c109201d", |
| "data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76", |
| "recipe": "bb8108e126d9752d656ad7f58d9b81db869e0a624341de57e8e02cbbaaf226ce", |
| "tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb" |
| }, |
| "parent_replica_hash": "678b8e8364d8ce87951628aa4ad68022f378c1c5c122f2e7ef7a8e5c7731dca7", |
| "parent_step": 963, |
| "start_tokens": 124174336 |
| }, |
| "batch_transition": { |
| "format": "sml-research-batch-transition-v1", |
| "parent_batches": [ |
| 4, |
| 4, |
| 2, |
| 2, |
| 2 |
| ], |
| "parent_contract": { |
| "code": "58f75d5d92576c268370e8f3dbb59f7cbc5ef58bcd1cf247a7cf6008c47e7e3c", |
| "data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76", |
| "recipe": "0a2de034cbafd3bbc101d6ac151602d2ea4889b51deaf9bf9fc49e2c827a93da", |
| "tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb" |
| }, |
| "parent_replica_hash": "6a63305e36520f376da3db369a11350c959485ed673f410c017bd6a79190ce5f", |
| "parent_step": 125, |
| "start_tokens": 14336000 |
| }, |
| "batches": [ |
| 7, |
| 3, |
| 2, |
| 2, |
| 2 |
| ], |
| "betas": [ |
| 0.9, |
| 0.95 |
| ], |
| "cache_gib_by_mac": [ |
| 16, |
| 16, |
| 4, |
| 4, |
| 4 |
| ], |
| "continuation": { |
| "format": "sml-v2-continuation-v1", |
| "parent_contract": { |
| "code": "db42e945868bbf75a15d49a3fd95692493d1e1262b465834779ce40ee9f0e59a", |
| "data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76", |
| "recipe": "e530ae78f2a7d2e70c3c55b0507ce74842d936825eedd27583802cf365a2a793", |
| "tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb" |
| }, |
| "parent_replica_hash": "a7a6b1a5986e0b752e502b73a212a01a7c02c36d0d16b889a1ba255267eed819", |
| "parent_step": 113313, |
| "rewarm_tokens": 50000000, |
| "start_lr": 0.0003, |
| "start_tokens": 14850113536 |
| }, |
| "data_mode": "hf_stream", |
| "decay_start_fraction": 0.8, |
| "eps": 1e-08, |
| "eval_batch_size": 2, |
| "eval_batches_per_source": 32, |
| "eval_every_tokens": 50000000, |
| "grad_accum": 4, |
| "grad_clip": 1.0, |
| "keep_checkpoints": 3, |
| "log_every": 10, |
| "min_lr_ratio": 0.1, |
| "model": { |
| "attention_impl": "fast", |
| "bias": false, |
| "ce_impl": "metal", |
| "d_model": 768, |
| "ffn_impl": "packed-metal", |
| "loss_dtype": "float32", |
| "max_seq_len": 2048, |
| "mlp_multiple_of": 256, |
| "mlp_ratio": 4.0, |
| "n_heads": 12, |
| "n_kv_heads": 4, |
| "n_layers": 24, |
| "qk_norm": true, |
| "vocab_size": 32000 |
| }, |
| "name": "research-20b-python-176m-continued-to-30000000000", |
| "peak_lr": 0.0003, |
| "plateau_scheduler": { |
| "cooldown": 10, |
| "factor": 0.8, |
| "format": "sml-v2-plateau-v1", |
| "initial_lr": 0.00024, |
| "min_delta": 0.002, |
| "min_lr": 3e-05, |
| "patience": 12 |
| }, |
| "playground_publish_best": true, |
| "reserve_gib": 16, |
| "sample_every_tokens": 50000000, |
| "sample_prompts_file": "sample_prompts.json", |
| "save_every_tokens": 100000000, |
| "scheduler_transition": { |
| "baseline_best": 2.801170878969133, |
| "format": "sml-v2-scheduler-transition-v1", |
| "parent_contract": { |
| "code": "db42e945868bbf75a15d49a3fd95692493d1e1262b465834779ce40ee9f0e59a", |
| "data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76", |
| "recipe": "970e96ed6919947a45d3135a5210496650e7e0f177afe311064ca0532fc9ab0e", |
| "tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb" |
| }, |
| "parent_replica_hash": "e7d76ee273de9c808e90454e4f2cc31228594ca68e4619b675ba0b4e6a451861", |
| "parent_step": 39896, |
| "start_tokens": 5227200512 |
| }, |
| "seed": 7337, |
| "stream": { |
| "dedup_window": 100000, |
| "prefetch_batches": 4, |
| "read_attempts": 5, |
| "shuffle_buffer": 256, |
| "tokenizer_corpus_policy": "reuse-frozen" |
| }, |
| "target_tokens": 30000000000, |
| "warmup_fraction": 0.01, |
| "weight_decay": 0.1 |
| }, |
| "replica_hash": "c5b173e6cd9f07144528d9900608275c00be13c1ae0eb543503c05a6db40e8a5", |
| "step": 132005, |
| "tokens": 17300111360, |
| "v2_contract": { |
| "code": "6e444060dba8c63ba33e128a43da03df6d927d289318ec51a2c0026fb494ab9f", |
| "data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76", |
| "recipe": "79ca696d2052326b32a71c4fc82729916312a9f97e61c358f5532131a9575d76", |
| "tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb" |
| }, |
| "validation_fingerprint": "4ddd461c131d4d27d07bb07f5aab3fcb5a00d4b1202066e3d9b6ee6b3129087b" |
| } |
|
|
