File size: 5,523 Bytes
ed99a2e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 | {
"best_val_loss": 2.5563359551876785,
"ema": 2.4005908938776055,
"initialization": "normal-0.02-residual-scaled-v1",
"lr_scheduler": {
"bad_evals": 0,
"best": 2.5563359551876785,
"cap": 3.221225472000002e-05,
"config_fingerprint": "4f636b3a8d4804bd7b10b26b34967c380d566268898be845bf324f8df4e1459f",
"cooldown_remaining": 0,
"evaluations": 242,
"format": "sml-v2-plateau-v1",
"last_loss": 2.5563359551876785,
"last_step": 132005,
"last_tokens": 17300111360,
"reductions": 9
},
"recipe": {
"batch_rebalance_transition": {
"format": "sml-research-batch-rebalance-v1",
"parent_batches": [
6,
4,
2,
2,
2
],
"parent_contract": {
"code": "2c7e9a69271f9817017c2371ae5e874c9229da489b6c0128aadda943c109201d",
"data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76",
"recipe": "bb8108e126d9752d656ad7f58d9b81db869e0a624341de57e8e02cbbaaf226ce",
"tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb"
},
"parent_replica_hash": "678b8e8364d8ce87951628aa4ad68022f378c1c5c122f2e7ef7a8e5c7731dca7",
"parent_step": 963,
"start_tokens": 124174336
},
"batch_transition": {
"format": "sml-research-batch-transition-v1",
"parent_batches": [
4,
4,
2,
2,
2
],
"parent_contract": {
"code": "58f75d5d92576c268370e8f3dbb59f7cbc5ef58bcd1cf247a7cf6008c47e7e3c",
"data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76",
"recipe": "0a2de034cbafd3bbc101d6ac151602d2ea4889b51deaf9bf9fc49e2c827a93da",
"tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb"
},
"parent_replica_hash": "6a63305e36520f376da3db369a11350c959485ed673f410c017bd6a79190ce5f",
"parent_step": 125,
"start_tokens": 14336000
},
"batches": [
7,
3,
2,
2,
2
],
"betas": [
0.9,
0.95
],
"cache_gib_by_mac": [
16,
16,
4,
4,
4
],
"continuation": {
"format": "sml-v2-continuation-v1",
"parent_contract": {
"code": "db42e945868bbf75a15d49a3fd95692493d1e1262b465834779ce40ee9f0e59a",
"data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76",
"recipe": "e530ae78f2a7d2e70c3c55b0507ce74842d936825eedd27583802cf365a2a793",
"tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb"
},
"parent_replica_hash": "a7a6b1a5986e0b752e502b73a212a01a7c02c36d0d16b889a1ba255267eed819",
"parent_step": 113313,
"rewarm_tokens": 50000000,
"start_lr": 0.0003,
"start_tokens": 14850113536
},
"data_mode": "hf_stream",
"decay_start_fraction": 0.8,
"eps": 1e-08,
"eval_batch_size": 2,
"eval_batches_per_source": 32,
"eval_every_tokens": 50000000,
"grad_accum": 4,
"grad_clip": 1.0,
"keep_checkpoints": 3,
"log_every": 10,
"min_lr_ratio": 0.1,
"model": {
"attention_impl": "fast",
"bias": false,
"ce_impl": "metal",
"d_model": 768,
"ffn_impl": "packed-metal",
"loss_dtype": "float32",
"max_seq_len": 2048,
"mlp_multiple_of": 256,
"mlp_ratio": 4.0,
"n_heads": 12,
"n_kv_heads": 4,
"n_layers": 24,
"qk_norm": true,
"vocab_size": 32000
},
"name": "research-20b-python-176m-continued-to-30000000000",
"peak_lr": 0.0003,
"plateau_scheduler": {
"cooldown": 10,
"factor": 0.8,
"format": "sml-v2-plateau-v1",
"initial_lr": 0.00024,
"min_delta": 0.002,
"min_lr": 3e-05,
"patience": 12
},
"playground_publish_best": true,
"reserve_gib": 16,
"sample_every_tokens": 50000000,
"sample_prompts_file": "sample_prompts.json",
"save_every_tokens": 100000000,
"scheduler_transition": {
"baseline_best": 2.801170878969133,
"format": "sml-v2-scheduler-transition-v1",
"parent_contract": {
"code": "db42e945868bbf75a15d49a3fd95692493d1e1262b465834779ce40ee9f0e59a",
"data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76",
"recipe": "970e96ed6919947a45d3135a5210496650e7e0f177afe311064ca0532fc9ab0e",
"tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb"
},
"parent_replica_hash": "e7d76ee273de9c808e90454e4f2cc31228594ca68e4619b675ba0b4e6a451861",
"parent_step": 39896,
"start_tokens": 5227200512
},
"seed": 7337,
"stream": {
"dedup_window": 100000,
"prefetch_batches": 4,
"read_attempts": 5,
"shuffle_buffer": 256,
"tokenizer_corpus_policy": "reuse-frozen"
},
"target_tokens": 30000000000,
"warmup_fraction": 0.01,
"weight_decay": 0.1
},
"replica_hash": "c5b173e6cd9f07144528d9900608275c00be13c1ae0eb543503c05a6db40e8a5",
"step": 132005,
"tokens": 17300111360,
"v2_contract": {
"code": "6e444060dba8c63ba33e128a43da03df6d927d289318ec51a2c0026fb494ab9f",
"data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76",
"recipe": "79ca696d2052326b32a71c4fc82729916312a9f97e61c358f5532131a9575d76",
"tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb"
},
"validation_fingerprint": "4ddd461c131d4d27d07bb07f5aab3fcb5a00d4b1202066e3d9b6ee6b3129087b"
}
|