Archive step-000043000: 3,256,889,455 phase targets
Browse files
checkpoints/step-000043000/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:295bd44a6291f07bce58820fd1f28eb36866a471ace2f3fb62ef98be46a8d01a
|
| 3 |
+
size 598238584
|
checkpoints/step-000043000/provenance.json
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "scholarly-ddp-v1",
|
| 3 |
+
"config": {
|
| 4 |
+
"world_size": 8,
|
| 5 |
+
"global_windows": 32,
|
| 6 |
+
"microbatch": 2,
|
| 7 |
+
"crop_padding": true,
|
| 8 |
+
"loss_chunk_size": 0,
|
| 9 |
+
"compile_model": true,
|
| 10 |
+
"cache_rope": true,
|
| 11 |
+
"context": 8192,
|
| 12 |
+
"target_tokens": 5000000000,
|
| 13 |
+
"warmup_tokens": 1,
|
| 14 |
+
"peak_lr": 3e-05,
|
| 15 |
+
"min_lr_ratio": 1.0,
|
| 16 |
+
"weight_decay": 0.1,
|
| 17 |
+
"seed": 42,
|
| 18 |
+
"checkpoint_every": 1000,
|
| 19 |
+
"evaluate_every": 500,
|
| 20 |
+
"pilot": false,
|
| 21 |
+
"experimental_data": true,
|
| 22 |
+
"prefetch_batches": 4,
|
| 23 |
+
"n_layer": 14,
|
| 24 |
+
"n_head": 12,
|
| 25 |
+
"n_embd": 768
|
| 26 |
+
},
|
| 27 |
+
"data_identity": {
|
| 28 |
+
"manifest_sha256": "16fe8f60a1185260c291341ac87b49b94d59eac5004ad2b048d781ddaf82ce8f",
|
| 29 |
+
"context": 8192,
|
| 30 |
+
"split": "train",
|
| 31 |
+
"tokenizer_sha256": "add5b2c7542e4efded329735825ecdcb94df1432617c3d3e4b7508aa459432f9"
|
| 32 |
+
},
|
| 33 |
+
"source_sha256": {
|
| 34 |
+
"trainer": "0e117622b1305b52d82e4d46544512f692a213c864ee85f67a881c98432f9949",
|
| 35 |
+
"config": "035cb07e7194ed6559f66c1d12425833904cd571dd76f4090384b1e1f7511c4f",
|
| 36 |
+
"model": "2c978432e84cad6b2f661c63e7e62bccfc393a2aeb9f0519f7ca3db6775ed326",
|
| 37 |
+
"loader": "7821e13e765a9f2c0e0a94744bc7f2df8eea647f26d7aa0069f62556618a0ef2",
|
| 38 |
+
"raw_loader": "fe7952d3adbd33666875117bd6a9ac10ee0f06dea2eb5ecc4b1f201584ef2a3f",
|
| 39 |
+
"prefetch": "1786b1423f5881b23223ff0db47284a522020237d77e3d7fa28d542deea5c218",
|
| 40 |
+
"live_loader": "64457d0e60f1f1b664d3b174183c6e6f3d0c618973fe56712f9e72cd2a118a75",
|
| 41 |
+
"tokenizer": "00e32adc13dab74f5a029c63304b4591bd48f7cd6b086b22d4624d244667b4da"
|
| 42 |
+
},
|
| 43 |
+
"torch_version": "2.6.0+cu124",
|
| 44 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 45 |
+
"parameters": 124389632,
|
| 46 |
+
"resume_from": "/runs/simple-llm-code60-fresh-5b-20261019-r02-s01/step-000000004.pt",
|
| 47 |
+
"lineage": {
|
| 48 |
+
"parent_checkpoint": "/runs/simple-llm-124m-eu-onepass-20261018-s02/step-000003838.pt",
|
| 49 |
+
"parent_sha256": "a9b4ff39352fae4aa163433df7130dbede1437657bee7c44fc26a6eb020af565",
|
| 50 |
+
"parent_tokens": 5878591538,
|
| 51 |
+
"immediate_parent_phase_tokens": 878610362,
|
| 52 |
+
"immediate_parent_step": 3838,
|
| 53 |
+
"parent_source_sha256": {
|
| 54 |
+
"trainer": "0e117622b1305b52d82e4d46544512f692a213c864ee85f67a881c98432f9949",
|
| 55 |
+
"config": "035cb07e7194ed6559f66c1d12425833904cd571dd76f4090384b1e1f7511c4f",
|
| 56 |
+
"model": "2c978432e84cad6b2f661c63e7e62bccfc393a2aeb9f0519f7ca3db6775ed326",
|
| 57 |
+
"loader": "7821e13e765a9f2c0e0a94744bc7f2df8eea647f26d7aa0069f62556618a0ef2",
|
| 58 |
+
"raw_loader": "fe7952d3adbd33666875117bd6a9ac10ee0f06dea2eb5ecc4b1f201584ef2a3f",
|
| 59 |
+
"prefetch": "1786b1423f5881b23223ff0db47284a522020237d77e3d7fa28d542deea5c218",
|
| 60 |
+
"live_loader": "64457d0e60f1f1b664d3b174183c6e6f3d0c618973fe56712f9e72cd2a118a75",
|
| 61 |
+
"tokenizer": "00e32adc13dab74f5a029c63304b4591bd48f7cd6b086b22d4624d244667b4da"
|
| 62 |
+
},
|
| 63 |
+
"prior_lineage": {
|
| 64 |
+
"parent_checkpoint": "/runs/simple-llm-124m-8k-5b-20261018-s03/step-000064227.pt",
|
| 65 |
+
"parent_sha256": "ce26b11aed843008fb89893a5e90219b1a5c96e44ef32d42eb9188e427df1876",
|
| 66 |
+
"parent_tokens": 4999981176,
|
| 67 |
+
"parent_step": 64227,
|
| 68 |
+
"parent_source_sha256": {
|
| 69 |
+
"trainer": "51e57355cb0a58a826c61b8d293647f15f9bea94a8fe312ed3751e17b5ec0f8f",
|
| 70 |
+
"config": "96e64243a30c5c4379d8493ce085f92338d0522daddfbd1916358aaafa21408b",
|
| 71 |
+
"model": "96bd95eb51ff5b2aa3d4cd9fc07073a6d21dc9003b36bbe9461fa5780a76882d",
|
| 72 |
+
"loader": "7821e13e765a9f2c0e0a94744bc7f2df8eea647f26d7aa0069f62556618a0ef2",
|
| 73 |
+
"raw_loader": "b5cc8623d84b3916c610faf15d248824423085f2d4070b095df33ac8e80c749c",
|
| 74 |
+
"prefetch": "1786b1423f5881b23223ff0db47284a522020237d77e3d7fa28d542deea5c218",
|
| 75 |
+
"live_loader": "64457d0e60f1f1b664d3b174183c6e6f3d0c618973fe56712f9e72cd2a118a75",
|
| 76 |
+
"tokenizer": "00e32adc13dab74f5a029c63304b4591bd48f7cd6b086b22d4624d244667b4da"
|
| 77 |
+
},
|
| 78 |
+
"optimizer": "preserved Adam moments and optimizer update counts",
|
| 79 |
+
"schedule": "new phase, constant parent terminal LR; new data cursor",
|
| 80 |
+
"phase_tokens_start": 0
|
| 81 |
+
},
|
| 82 |
+
"phase_request": {
|
| 83 |
+
"fresh_only": true,
|
| 84 |
+
"coding_share": 60,
|
| 85 |
+
"target_tokens": 5000000000,
|
| 86 |
+
"book_allocation": "fresh-book-pool-v1"
|
| 87 |
+
},
|
| 88 |
+
"optimizer": "Adam moments/update counts preserved",
|
| 89 |
+
"schedule": {
|
| 90 |
+
"peak_lr": 3e-05,
|
| 91 |
+
"min_lr_ratio": 1.0,
|
| 92 |
+
"warmup_tokens": 1,
|
| 93 |
+
"counter": "phase loss tokens"
|
| 94 |
+
}
|
| 95 |
+
},
|
| 96 |
+
"step": 43000,
|
| 97 |
+
"tokens": 3256889455,
|
| 98 |
+
"run_id": "simple-llm-code60-fresh-5b-20261019-r02",
|
| 99 |
+
"checkpoint": "/runs/simple-llm-code60-fresh-5b-20261019-r02-s02/step-000043000.pt",
|
| 100 |
+
"weights_sha256": "295bd44a6291f07bce58820fd1f28eb36866a471ace2f3fb62ef98be46a8d01a",
|
| 101 |
+
"weight_roundtrip_exact": true,
|
| 102 |
+
"cpu_logits_roundtrip_exact": true,
|
| 103 |
+
"loss_token_exposures": {
|
| 104 |
+
"fineweb_edu": 186105340,
|
| 105 |
+
"code_python": 1954110000,
|
| 106 |
+
"finemath_4plus": 372217046,
|
| 107 |
+
"pes2o": 186107595,
|
| 108 |
+
"pubmed": 55834285,
|
| 109 |
+
"arxiv": 37225670,
|
| 110 |
+
"pressbooks": 60453682,
|
| 111 |
+
"libretexts": 7290853,
|
| 112 |
+
"doab": 83692548,
|
| 113 |
+
"gutenberg": 104618112,
|
| 114 |
+
"pre_1929_books": 146460538,
|
| 115 |
+
"library_of_congress": 62773786
|
| 116 |
+
}
|
| 117 |
+
}
|