jfrick commited on
Commit
9350d7d
·
verified ·
1 Parent(s): bf8d706

Archive step-000043000: 3,256,889,455 phase targets

Browse files
checkpoints/step-000043000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:295bd44a6291f07bce58820fd1f28eb36866a471ace2f3fb62ef98be46a8d01a
3
+ size 598238584
checkpoints/step-000043000/provenance.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "scholarly-ddp-v1",
3
+ "config": {
4
+ "world_size": 8,
5
+ "global_windows": 32,
6
+ "microbatch": 2,
7
+ "crop_padding": true,
8
+ "loss_chunk_size": 0,
9
+ "compile_model": true,
10
+ "cache_rope": true,
11
+ "context": 8192,
12
+ "target_tokens": 5000000000,
13
+ "warmup_tokens": 1,
14
+ "peak_lr": 3e-05,
15
+ "min_lr_ratio": 1.0,
16
+ "weight_decay": 0.1,
17
+ "seed": 42,
18
+ "checkpoint_every": 1000,
19
+ "evaluate_every": 500,
20
+ "pilot": false,
21
+ "experimental_data": true,
22
+ "prefetch_batches": 4,
23
+ "n_layer": 14,
24
+ "n_head": 12,
25
+ "n_embd": 768
26
+ },
27
+ "data_identity": {
28
+ "manifest_sha256": "16fe8f60a1185260c291341ac87b49b94d59eac5004ad2b048d781ddaf82ce8f",
29
+ "context": 8192,
30
+ "split": "train",
31
+ "tokenizer_sha256": "add5b2c7542e4efded329735825ecdcb94df1432617c3d3e4b7508aa459432f9"
32
+ },
33
+ "source_sha256": {
34
+ "trainer": "0e117622b1305b52d82e4d46544512f692a213c864ee85f67a881c98432f9949",
35
+ "config": "035cb07e7194ed6559f66c1d12425833904cd571dd76f4090384b1e1f7511c4f",
36
+ "model": "2c978432e84cad6b2f661c63e7e62bccfc393a2aeb9f0519f7ca3db6775ed326",
37
+ "loader": "7821e13e765a9f2c0e0a94744bc7f2df8eea647f26d7aa0069f62556618a0ef2",
38
+ "raw_loader": "fe7952d3adbd33666875117bd6a9ac10ee0f06dea2eb5ecc4b1f201584ef2a3f",
39
+ "prefetch": "1786b1423f5881b23223ff0db47284a522020237d77e3d7fa28d542deea5c218",
40
+ "live_loader": "64457d0e60f1f1b664d3b174183c6e6f3d0c618973fe56712f9e72cd2a118a75",
41
+ "tokenizer": "00e32adc13dab74f5a029c63304b4591bd48f7cd6b086b22d4624d244667b4da"
42
+ },
43
+ "torch_version": "2.6.0+cu124",
44
+ "gpu": "NVIDIA H100 80GB HBM3",
45
+ "parameters": 124389632,
46
+ "resume_from": "/runs/simple-llm-code60-fresh-5b-20261019-r02-s01/step-000000004.pt",
47
+ "lineage": {
48
+ "parent_checkpoint": "/runs/simple-llm-124m-eu-onepass-20261018-s02/step-000003838.pt",
49
+ "parent_sha256": "a9b4ff39352fae4aa163433df7130dbede1437657bee7c44fc26a6eb020af565",
50
+ "parent_tokens": 5878591538,
51
+ "immediate_parent_phase_tokens": 878610362,
52
+ "immediate_parent_step": 3838,
53
+ "parent_source_sha256": {
54
+ "trainer": "0e117622b1305b52d82e4d46544512f692a213c864ee85f67a881c98432f9949",
55
+ "config": "035cb07e7194ed6559f66c1d12425833904cd571dd76f4090384b1e1f7511c4f",
56
+ "model": "2c978432e84cad6b2f661c63e7e62bccfc393a2aeb9f0519f7ca3db6775ed326",
57
+ "loader": "7821e13e765a9f2c0e0a94744bc7f2df8eea647f26d7aa0069f62556618a0ef2",
58
+ "raw_loader": "fe7952d3adbd33666875117bd6a9ac10ee0f06dea2eb5ecc4b1f201584ef2a3f",
59
+ "prefetch": "1786b1423f5881b23223ff0db47284a522020237d77e3d7fa28d542deea5c218",
60
+ "live_loader": "64457d0e60f1f1b664d3b174183c6e6f3d0c618973fe56712f9e72cd2a118a75",
61
+ "tokenizer": "00e32adc13dab74f5a029c63304b4591bd48f7cd6b086b22d4624d244667b4da"
62
+ },
63
+ "prior_lineage": {
64
+ "parent_checkpoint": "/runs/simple-llm-124m-8k-5b-20261018-s03/step-000064227.pt",
65
+ "parent_sha256": "ce26b11aed843008fb89893a5e90219b1a5c96e44ef32d42eb9188e427df1876",
66
+ "parent_tokens": 4999981176,
67
+ "parent_step": 64227,
68
+ "parent_source_sha256": {
69
+ "trainer": "51e57355cb0a58a826c61b8d293647f15f9bea94a8fe312ed3751e17b5ec0f8f",
70
+ "config": "96e64243a30c5c4379d8493ce085f92338d0522daddfbd1916358aaafa21408b",
71
+ "model": "96bd95eb51ff5b2aa3d4cd9fc07073a6d21dc9003b36bbe9461fa5780a76882d",
72
+ "loader": "7821e13e765a9f2c0e0a94744bc7f2df8eea647f26d7aa0069f62556618a0ef2",
73
+ "raw_loader": "b5cc8623d84b3916c610faf15d248824423085f2d4070b095df33ac8e80c749c",
74
+ "prefetch": "1786b1423f5881b23223ff0db47284a522020237d77e3d7fa28d542deea5c218",
75
+ "live_loader": "64457d0e60f1f1b664d3b174183c6e6f3d0c618973fe56712f9e72cd2a118a75",
76
+ "tokenizer": "00e32adc13dab74f5a029c63304b4591bd48f7cd6b086b22d4624d244667b4da"
77
+ },
78
+ "optimizer": "preserved Adam moments and optimizer update counts",
79
+ "schedule": "new phase, constant parent terminal LR; new data cursor",
80
+ "phase_tokens_start": 0
81
+ },
82
+ "phase_request": {
83
+ "fresh_only": true,
84
+ "coding_share": 60,
85
+ "target_tokens": 5000000000,
86
+ "book_allocation": "fresh-book-pool-v1"
87
+ },
88
+ "optimizer": "Adam moments/update counts preserved",
89
+ "schedule": {
90
+ "peak_lr": 3e-05,
91
+ "min_lr_ratio": 1.0,
92
+ "warmup_tokens": 1,
93
+ "counter": "phase loss tokens"
94
+ }
95
+ },
96
+ "step": 43000,
97
+ "tokens": 3256889455,
98
+ "run_id": "simple-llm-code60-fresh-5b-20261019-r02",
99
+ "checkpoint": "/runs/simple-llm-code60-fresh-5b-20261019-r02-s02/step-000043000.pt",
100
+ "weights_sha256": "295bd44a6291f07bce58820fd1f28eb36866a471ace2f3fb62ef98be46a8d01a",
101
+ "weight_roundtrip_exact": true,
102
+ "cpu_logits_roundtrip_exact": true,
103
+ "loss_token_exposures": {
104
+ "fineweb_edu": 186105340,
105
+ "code_python": 1954110000,
106
+ "finemath_4plus": 372217046,
107
+ "pes2o": 186107595,
108
+ "pubmed": 55834285,
109
+ "arxiv": 37225670,
110
+ "pressbooks": 60453682,
111
+ "libretexts": 7290853,
112
+ "doab": 83692548,
113
+ "gutenberg": 104618112,
114
+ "pre_1929_books": 146460538,
115
+ "library_of_congress": 62773786
116
+ }
117
+ }