Hugging Face's logo

DedeProGames
/
Wisp-5M

Text Generation
Transformers
Safetensors
English
llama
tiny
small-language-model
pretrained
wisp
text-generation-inference
Model card Files
xet
 Community
DedeProGames commited on
Commit
fddf996
·
verified ·
1 Parent(s): 0945f1d

Wisp-5M: 3.00B tokens

Browse files
Files changed (8) hide show
  1. config.json +28 -0
  2. generation_config.json +10 -0
  3. model.safetensors +3 -0
  4. special_tokens_map.json +9 -0
  5. tokenizer.json +0 -0
  6. tokenizer_config.json +13 -0
  7. training_log.jsonl +241 -0
  8. training_summary.json +41 -0
config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LlamaForCausalLM"
4
+ ],
5
+ "model_type": "llama",
6
+ "vocab_size": 8192,
7
+ "hidden_size": 192,
8
+ "intermediate_size": 512,
9
+ "num_hidden_layers": 9,
10
+ "num_attention_heads": 6,
11
+ "num_key_value_heads": 2,
12
+ "head_dim": 32,
13
+ "hidden_act": "silu",
14
+ "max_position_embeddings": 2048,
15
+ "rope_theta": 10000.0,
16
+ "rope_scaling": null,
17
+ "rms_norm_eps": 1e-05,
18
+ "initializer_range": 0.02,
19
+ "attention_bias": false,
20
+ "attention_dropout": 0.0,
21
+ "mlp_bias": false,
22
+ "tie_word_embeddings": true,
23
+ "bos_token_id": 0,
24
+ "eos_token_id": 0,
25
+ "pad_token_id": 1,
26
+ "use_cache": true,
27
+ "torch_dtype": "float32"
28
+ }
generation_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 0,
3
+ "eos_token_id": 0,
4
+ "pad_token_id": 1,
5
+ "do_sample": true,
6
+ "temperature": 0.8,
7
+ "top_p": 0.95,
8
+ "repetition_penalty": 1.1,
9
+ "max_new_tokens": 128
10
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51bde3adc1001450a2491179459ec445d5ccaacb85d6eaed5fc72ee051eb0e2a
3
+ size 20470784
special_tokens_map.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|pad|>",
5
+ "additional_special_tokens": [
6
+ "<|im_start|>",
7
+ "<|im_end|>"
8
+ ]
9
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "PreTrainedTokenizerFast",
3
+ "bos_token": "<|endoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "pad_token": "<|pad|>",
6
+ "unk_token": null,
7
+ "additional_special_tokens": [
8
+ "<|im_start|>",
9
+ "<|im_end|>"
10
+ ],
11
+ "model_max_length": 2048,
12
+ "clean_up_tokenization_spaces": false
13
+ }
training_log.jsonl ADDED
@@ -0,0 +1,241 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"step": 50, "tokens": 13107200, "loss": 6.2863, "lr": 0.002632, "gnorm": 1.794, "tok_s": 112970, "elapsed_min": 1.93, "eta_min": 172.3, "cost_usd": 0.025}
2
+ {"step": 100, "tokens": 26214400, "loss": 5.3439, "lr": 0.005263, "gnorm": 0.522, "tok_s": 148170, "elapsed_min": 3.41, "eta_min": 170.8, "cost_usd": 0.035}
3
+ {"step": 150, "tokens": 39321600, "loss": 5.004, "lr": 0.007895, "gnorm": 0.332, "tok_s": 147414, "elapsed_min": 4.89, "eta_min": 170.0, "cost_usd": 0.044}
4
+ {"step": 200, "tokens": 52428800, "loss": 4.787, "lr": 0.01, "gnorm": 0.376, "tok_s": 147402, "elapsed_min": 6.37, "eta_min": 168.7, "cost_usd": 0.054}
5
+ {"step": 250, "tokens": 65536000, "loss": 4.5665, "lr": 0.01, "gnorm": 0.309, "tok_s": 147478, "elapsed_min": 7.85, "eta_min": 166.6, "cost_usd": 0.064}
6
+ {"step": 300, "tokens": 78643200, "loss": 4.2811, "lr": 0.01, "gnorm": 0.283, "tok_s": 147505, "elapsed_min": 9.33, "eta_min": 164.8, "cost_usd": 0.074}
7
+ {"step": 350, "tokens": 91750400, "loss": 4.0843, "lr": 0.01, "gnorm": 0.293, "tok_s": 147582, "elapsed_min": 10.81, "eta_min": 163.3, "cost_usd": 0.084}
8
+ {"step": 400, "tokens": 104857600, "loss": 3.9369, "lr": 0.01, "gnorm": 0.274, "tok_s": 147644, "elapsed_min": 12.29, "eta_min": 161.9, "cost_usd": 0.094}
9
+ {"step": 450, "tokens": 117964800, "loss": 3.7318, "lr": 0.01, "gnorm": 0.23, "tok_s": 147666, "elapsed_min": 13.77, "eta_min": 160.4, "cost_usd": 0.104}
10
+ {"step": 500, "tokens": 131072000, "loss": 3.6983, "lr": 0.01, "gnorm": 0.231, "tok_s": 147745, "elapsed_min": 15.25, "eta_min": 158.9, "cost_usd": 0.113}
11
+ {"step": 550, "tokens": 144179200, "loss": 3.7879, "lr": 0.01, "gnorm": 0.238, "tok_s": 147778, "elapsed_min": 16.73, "eta_min": 157.4, "cost_usd": 0.123}
12
+ {"step": 600, "tokens": 157286400, "loss": 3.6755, "lr": 0.01, "gnorm": 0.216, "tok_s": 147836, "elapsed_min": 18.21, "eta_min": 155.9, "cost_usd": 0.133}
13
+ {"step": 650, "tokens": 170393600, "loss": 3.7285, "lr": 0.01, "gnorm": 0.243, "tok_s": 147795, "elapsed_min": 19.69, "eta_min": 154.5, "cost_usd": 0.143}
14
+ {"step": 700, "tokens": 183500800, "loss": 3.6543, "lr": 0.01, "gnorm": 0.235, "tok_s": 147825, "elapsed_min": 21.16, "eta_min": 153.0, "cost_usd": 0.153}
15
+ {"step": 750, "tokens": 196608000, "loss": 3.6663, "lr": 0.01, "gnorm": 0.242, "tok_s": 147914, "elapsed_min": 22.64, "eta_min": 151.5, "cost_usd": 0.163}
16
+ {"step": 800, "tokens": 209715200, "loss": 3.6651, "lr": 0.01, "gnorm": 0.204, "tok_s": 147840, "elapsed_min": 24.12, "eta_min": 150.1, "cost_usd": 0.173}
17
+ {"step": 850, "tokens": 222822400, "loss": 3.5437, "lr": 0.01, "gnorm": 0.188, "tok_s": 147859, "elapsed_min": 25.6, "eta_min": 148.6, "cost_usd": 0.182}
18
+ {"step": 900, "tokens": 235929600, "loss": 3.345, "lr": 0.01, "gnorm": 0.232, "tok_s": 147809, "elapsed_min": 27.07, "eta_min": 147.2, "cost_usd": 0.192}
19
+ {"step": 950, "tokens": 249036800, "loss": 3.5323, "lr": 0.01, "gnorm": 0.189, "tok_s": 147815, "elapsed_min": 28.55, "eta_min": 145.7, "cost_usd": 0.202}
20
+ {"step": 1000, "tokens": 262144000, "loss": 3.5837, "lr": 0.01, "gnorm": 0.216, "tok_s": 147895, "elapsed_min": 30.03, "eta_min": 144.2, "cost_usd": 0.212}
21
+ {"step": 1000, "tokens": 262144000, "val": {"dclm": 3.8555, "finemath": 3.0666, "fineweb": 3.6721, "stack": 2.3637, "avg": 3.2395}}
22
+ {"step": 1050, "tokens": 275251200, "loss": 3.5221, "lr": 0.01, "gnorm": 0.184, "tok_s": 144654, "elapsed_min": 31.54, "eta_min": 142.6, "cost_usd": 0.222}
23
+ {"step": 1100, "tokens": 288358400, "loss": 3.5472, "lr": 0.01, "gnorm": 0.208, "tok_s": 147937, "elapsed_min": 33.02, "eta_min": 141.2, "cost_usd": 0.232}
24
+ {"step": 1150, "tokens": 301465600, "loss": 3.5529, "lr": 0.01, "gnorm": 0.213, "tok_s": 148044, "elapsed_min": 34.49, "eta_min": 139.7, "cost_usd": 0.242}
25
+ {"step": 1200, "tokens": 314572800, "loss": 3.4755, "lr": 0.01, "gnorm": 0.209, "tok_s": 147977, "elapsed_min": 35.97, "eta_min": 138.2, "cost_usd": 0.252}
26
+ {"step": 1250, "tokens": 327680000, "loss": 3.5551, "lr": 0.01, "gnorm": 0.249, "tok_s": 147978, "elapsed_min": 37.44, "eta_min": 301.0, "cost_usd": 0.261}
27
+ {"step": 1300, "tokens": 340787200, "loss": 3.4471, "lr": 0.01, "gnorm": 0.192, "tok_s": 147877, "elapsed_min": 38.92, "eta_min": 299.6, "cost_usd": 0.271}
28
+ {"step": 1350, "tokens": 353894400, "loss": 3.5183, "lr": 0.01, "gnorm": 0.217, "tok_s": 147818, "elapsed_min": 40.4, "eta_min": 298.3, "cost_usd": 0.281}
29
+ {"step": 1400, "tokens": 367001600, "loss": 3.3952, "lr": 0.01, "gnorm": 0.188, "tok_s": 147939, "elapsed_min": 41.88, "eta_min": 296.7, "cost_usd": 0.291}
30
+ {"step": 1450, "tokens": 380108800, "loss": 3.5208, "lr": 0.01, "gnorm": 0.194, "tok_s": 147940, "elapsed_min": 43.35, "eta_min": 295.1, "cost_usd": 0.301}
31
+ {"step": 1500, "tokens": 393216000, "loss": 3.3653, "lr": 0.01, "gnorm": 0.199, "tok_s": 148061, "elapsed_min": 44.83, "eta_min": 293.5, "cost_usd": 0.311}
32
+ {"step": 1550, "tokens": 406323200, "loss": 3.5222, "lr": 0.01, "gnorm": 0.202, "tok_s": 148126, "elapsed_min": 46.3, "eta_min": 291.9, "cost_usd": 0.32}
33
+ {"step": 1600, "tokens": 419430400, "loss": 3.3801, "lr": 0.01, "gnorm": 0.201, "tok_s": 148038, "elapsed_min": 47.78, "eta_min": 290.4, "cost_usd": 0.33}
34
+ {"step": 1650, "tokens": 432537600, "loss": 3.5274, "lr": 0.01, "gnorm": 0.201, "tok_s": 148017, "elapsed_min": 49.25, "eta_min": 289.1, "cost_usd": 0.34}
35
+ {"step": 1700, "tokens": 445644800, "loss": 3.4234, "lr": 0.01, "gnorm": 0.187, "tok_s": 148019, "elapsed_min": 50.73, "eta_min": 287.6, "cost_usd": 0.35}
36
+ {"step": 1750, "tokens": 458752000, "loss": 3.4477, "lr": 0.01, "gnorm": 0.197, "tok_s": 148080, "elapsed_min": 52.2, "eta_min": 286.1, "cost_usd": 0.36}
37
+ {"step": 1800, "tokens": 471859200, "loss": 3.4446, "lr": 0.01, "gnorm": 0.23, "tok_s": 148063, "elapsed_min": 53.68, "eta_min": 284.5, "cost_usd": 0.37}
38
+ {"step": 1850, "tokens": 484966400, "loss": 3.4018, "lr": 0.01, "gnorm": 0.228, "tok_s": 148006, "elapsed_min": 55.16, "eta_min": 283.1, "cost_usd": 0.38}
39
+ {"step": 1900, "tokens": 498073600, "loss": 3.3678, "lr": 0.01, "gnorm": 0.175, "tok_s": 148047, "elapsed_min": 56.63, "eta_min": 281.7, "cost_usd": 0.389}
40
+ {"step": 1950, "tokens": 511180800, "loss": 3.4723, "lr": 0.01, "gnorm": 0.18, "tok_s": 147974, "elapsed_min": 58.11, "eta_min": 280.2, "cost_usd": 0.399}
41
+ {"step": 2000, "tokens": 524288000, "loss": 3.3465, "lr": 0.01, "gnorm": 0.179, "tok_s": 147959, "elapsed_min": 59.58, "eta_min": 278.8, "cost_usd": 0.409}
42
+ {"step": 2000, "tokens": 524288000, "val": {"dclm": 3.7534, "finemath": 2.9212, "fineweb": 3.5602, "stack": 2.198, "avg": 3.1082}}
43
+ {"step": 2050, "tokens": 537395200, "loss": 3.3937, "lr": 0.01, "gnorm": 0.207, "tok_s": 144681, "elapsed_min": 61.09, "eta_min": 277.6, "cost_usd": 0.419}
44
+ {"step": 2100, "tokens": 550502400, "loss": 3.365, "lr": 0.01, "gnorm": 0.201, "tok_s": 147722, "elapsed_min": 62.57, "eta_min": 276.4, "cost_usd": 0.429}
45
+ {"step": 2150, "tokens": 563609600, "loss": 3.3761, "lr": 0.01, "gnorm": 0.17, "tok_s": 147808, "elapsed_min": 64.05, "eta_min": 274.8, "cost_usd": 0.439}
46
+ {"step": 2200, "tokens": 576716800, "loss": 3.4626, "lr": 0.01, "gnorm": 0.207, "tok_s": 147780, "elapsed_min": 65.53, "eta_min": 273.3, "cost_usd": 0.449}
47
+ {"step": 2250, "tokens": 589824000, "loss": 3.4681, "lr": 0.01, "gnorm": 0.19, "tok_s": 147735, "elapsed_min": 67.01, "eta_min": 271.8, "cost_usd": 0.459}
48
+ {"step": 2300, "tokens": 602931200, "loss": 3.4398, "lr": 0.01, "gnorm": 0.192, "tok_s": 147742, "elapsed_min": 68.49, "eta_min": 270.4, "cost_usd": 0.468}
49
+ {"step": 2350, "tokens": 616038400, "loss": 3.4247, "lr": 0.01, "gnorm": 0.21, "tok_s": 147808, "elapsed_min": 69.96, "eta_min": 268.9, "cost_usd": 0.478}
50
+ {"step": 2400, "tokens": 629145600, "loss": 3.4823, "lr": 0.01, "gnorm": 0.192, "tok_s": 147796, "elapsed_min": 71.44, "eta_min": 267.3, "cost_usd": 0.488}
51
+ {"step": 2450, "tokens": 642252800, "loss": 3.4252, "lr": 0.01, "gnorm": 0.171, "tok_s": 147709, "elapsed_min": 72.92, "eta_min": 265.9, "cost_usd": 0.498}
52
+ {"step": 2500, "tokens": 655360000, "loss": 3.4286, "lr": 0.01, "gnorm": 0.21, "tok_s": 147672, "elapsed_min": 74.4, "eta_min": 264.6, "cost_usd": 0.508}
53
+ {"step": 2550, "tokens": 668467200, "loss": 3.4775, "lr": 0.01, "gnorm": 0.189, "tok_s": 147681, "elapsed_min": 75.88, "eta_min": 263.1, "cost_usd": 0.518}
54
+ {"step": 2600, "tokens": 681574400, "loss": 3.3503, "lr": 0.01, "gnorm": 0.204, "tok_s": 147755, "elapsed_min": 77.36, "eta_min": 261.6, "cost_usd": 0.528}
55
+ {"step": 2650, "tokens": 694681600, "loss": 3.3824, "lr": 0.01, "gnorm": 0.19, "tok_s": 147719, "elapsed_min": 78.84, "eta_min": 260.1, "cost_usd": 0.537}
56
+ {"step": 2700, "tokens": 707788800, "loss": 3.351, "lr": 0.01, "gnorm": 0.201, "tok_s": 147694, "elapsed_min": 80.32, "eta_min": 258.6, "cost_usd": 0.547}
57
+ {"step": 2750, "tokens": 720896000, "loss": 3.4293, "lr": 0.01, "gnorm": 0.18, "tok_s": 147741, "elapsed_min": 81.8, "eta_min": 257.1, "cost_usd": 0.557}
58
+ {"step": 2800, "tokens": 734003200, "loss": 3.3068, "lr": 0.01, "gnorm": 0.182, "tok_s": 147842, "elapsed_min": 83.27, "eta_min": 255.5, "cost_usd": 0.567}
59
+ {"step": 2850, "tokens": 747110400, "loss": 3.4236, "lr": 0.01, "gnorm": 0.201, "tok_s": 147828, "elapsed_min": 84.75, "eta_min": 254.0, "cost_usd": 0.577}
60
+ {"step": 2900, "tokens": 760217600, "loss": 3.3502, "lr": 0.01, "gnorm": 0.191, "tok_s": 147785, "elapsed_min": 86.23, "eta_min": 252.5, "cost_usd": 0.587}
61
+ {"step": 2950, "tokens": 773324800, "loss": 3.3504, "lr": 0.01, "gnorm": 0.208, "tok_s": 147820, "elapsed_min": 87.71, "eta_min": 251.1, "cost_usd": 0.597}
62
+ {"step": 3000, "tokens": 786432000, "loss": 3.3578, "lr": 0.01, "gnorm": 0.213, "tok_s": 147847, "elapsed_min": 89.18, "eta_min": 249.5, "cost_usd": 0.606}
63
+ {"step": 3000, "tokens": 786432000, "val": {"dclm": 3.7156, "finemath": 2.879, "fineweb": 3.5191, "stack": 2.167, "avg": 3.0702}}
64
+ {"step": 3050, "tokens": 799539200, "loss": 3.297, "lr": 0.01, "gnorm": 0.18, "tok_s": 144781, "elapsed_min": 90.69, "eta_min": 248.1, "cost_usd": 0.616}
65
+ {"step": 3100, "tokens": 812646400, "loss": 3.4379, "lr": 0.01, "gnorm": 0.194, "tok_s": 147759, "elapsed_min": 92.17, "eta_min": 246.7, "cost_usd": 0.626}
66
+ {"step": 3150, "tokens": 825753600, "loss": 3.3068, "lr": 0.01, "gnorm": 0.166, "tok_s": 147703, "elapsed_min": 93.65, "eta_min": 245.3, "cost_usd": 0.636}
67
+ {"step": 3200, "tokens": 838860800, "loss": 3.2842, "lr": 0.01, "gnorm": 0.21, "tok_s": 147659, "elapsed_min": 95.13, "eta_min": 243.9, "cost_usd": 0.646}
68
+ {"step": 3250, "tokens": 851968000, "loss": 3.3575, "lr": 0.01, "gnorm": 0.179, "tok_s": 147802, "elapsed_min": 96.61, "eta_min": 242.3, "cost_usd": 0.656}
69
+ {"step": 3300, "tokens": 865075200, "loss": 3.5455, "lr": 0.01, "gnorm": 0.207, "tok_s": 147740, "elapsed_min": 98.09, "eta_min": 240.8, "cost_usd": 0.666}
70
+ {"step": 3350, "tokens": 878182400, "loss": 3.3867, "lr": 0.01, "gnorm": 0.202, "tok_s": 147855, "elapsed_min": 99.56, "eta_min": 239.3, "cost_usd": 0.676}
71
+ {"step": 3400, "tokens": 891289600, "loss": 3.3779, "lr": 0.01, "gnorm": 0.192, "tok_s": 147883, "elapsed_min": 101.04, "eta_min": 237.7, "cost_usd": 0.685}
72
+ {"step": 3450, "tokens": 904396800, "loss": 3.1981, "lr": 0.01, "gnorm": 0.173, "tok_s": 147821, "elapsed_min": 102.52, "eta_min": 236.2, "cost_usd": 0.695}
73
+ {"step": 3500, "tokens": 917504000, "loss": 3.3697, "lr": 0.01, "gnorm": 0.191, "tok_s": 147917, "elapsed_min": 104.0, "eta_min": 234.7, "cost_usd": 0.705}
74
+ {"step": 3550, "tokens": 930611200, "loss": 3.3621, "lr": 0.01, "gnorm": 0.229, "tok_s": 147874, "elapsed_min": 105.47, "eta_min": 233.2, "cost_usd": 0.715}
75
+ {"step": 3600, "tokens": 943718400, "loss": 3.4412, "lr": 0.01, "gnorm": 0.212, "tok_s": 147812, "elapsed_min": 106.95, "eta_min": 231.8, "cost_usd": 0.725}
76
+ {"step": 3650, "tokens": 956825600, "loss": 3.3536, "lr": 0.01, "gnorm": 0.181, "tok_s": 147846, "elapsed_min": 108.43, "eta_min": 230.3, "cost_usd": 0.735}
77
+ {"step": 3700, "tokens": 969932800, "loss": 3.3173, "lr": 0.01, "gnorm": 0.199, "tok_s": 147863, "elapsed_min": 109.91, "eta_min": 228.8, "cost_usd": 0.745}
78
+ {"step": 3750, "tokens": 983040000, "loss": 3.4453, "lr": 0.01, "gnorm": 0.182, "tok_s": 147873, "elapsed_min": 111.38, "eta_min": 227.3, "cost_usd": 0.754}
79
+ {"step": 3800, "tokens": 996147200, "loss": 3.3188, "lr": 0.01, "gnorm": 0.21, "tok_s": 147855, "elapsed_min": 112.86, "eta_min": 225.9, "cost_usd": 0.764}
80
+ {"step": 3850, "tokens": 1009254400, "loss": 3.3027, "lr": 0.01, "gnorm": 0.183, "tok_s": 147762, "elapsed_min": 114.34, "eta_min": 224.5, "cost_usd": 0.774}
81
+ {"step": 3900, "tokens": 1022361600, "loss": 3.4229, "lr": 0.01, "gnorm": 0.192, "tok_s": 147734, "elapsed_min": 115.82, "eta_min": 223.1, "cost_usd": 0.784}
82
+ {"step": 3950, "tokens": 1035468800, "loss": 3.426, "lr": 0.01, "gnorm": 0.191, "tok_s": 147755, "elapsed_min": 117.3, "eta_min": 221.6, "cost_usd": 0.794}
83
+ {"step": 4000, "tokens": 1048576000, "loss": 3.3845, "lr": 0.01, "gnorm": 0.227, "tok_s": 147821, "elapsed_min": 118.77, "eta_min": 220.1, "cost_usd": 0.804}
84
+ {"step": 4000, "tokens": 1048576000, "val": {"dclm": 3.6992, "finemath": 2.8594, "fineweb": 3.4942, "stack": 2.1285, "avg": 3.0453}}
85
+ {"step": 4050, "tokens": 1061683200, "loss": 3.2994, "lr": 0.01, "gnorm": 0.192, "tok_s": 144597, "elapsed_min": 120.29, "eta_min": 218.7, "cost_usd": 0.814}
86
+ {"step": 4100, "tokens": 1074790400, "loss": 3.3568, "lr": 0.01, "gnorm": 0.212, "tok_s": 147569, "elapsed_min": 121.77, "eta_min": 217.4, "cost_usd": 0.824}
87
+ {"step": 4150, "tokens": 1087897600, "loss": 3.3982, "lr": 0.01, "gnorm": 0.192, "tok_s": 147526, "elapsed_min": 123.25, "eta_min": 216.0, "cost_usd": 0.833}
88
+ {"step": 4200, "tokens": 1101004800, "loss": 3.2352, "lr": 0.01, "gnorm": 0.183, "tok_s": 147589, "elapsed_min": 124.73, "eta_min": 214.5, "cost_usd": 0.843}
89
+ {"step": 4250, "tokens": 1114112000, "loss": 3.3169, "lr": 0.01, "gnorm": 0.192, "tok_s": 147585, "elapsed_min": 126.21, "eta_min": 213.0, "cost_usd": 0.853}
90
+ {"step": 4300, "tokens": 1127219200, "loss": 3.3761, "lr": 0.01, "gnorm": 0.214, "tok_s": 147504, "elapsed_min": 127.69, "eta_min": 211.5, "cost_usd": 0.863}
91
+ {"step": 4350, "tokens": 1140326400, "loss": 3.3804, "lr": 0.01, "gnorm": 0.213, "tok_s": 147566, "elapsed_min": 129.17, "eta_min": 210.1, "cost_usd": 0.873}
92
+ {"step": 4400, "tokens": 1153433600, "loss": 3.2996, "lr": 0.01, "gnorm": 0.212, "tok_s": 147580, "elapsed_min": 130.65, "eta_min": 208.5, "cost_usd": 0.883}
93
+ {"step": 4450, "tokens": 1166540800, "loss": 3.3682, "lr": 0.01, "gnorm": 0.208, "tok_s": 147646, "elapsed_min": 132.13, "eta_min": 207.0, "cost_usd": 0.893}
94
+ {"step": 4500, "tokens": 1179648000, "loss": 3.4237, "lr": 0.01, "gnorm": 0.181, "tok_s": 147647, "elapsed_min": 133.61, "eta_min": 205.5, "cost_usd": 0.903}
95
+ {"step": 4550, "tokens": 1192755200, "loss": 3.3648, "lr": 0.01, "gnorm": 0.206, "tok_s": 147621, "elapsed_min": 135.09, "eta_min": 204.0, "cost_usd": 0.912}
96
+ {"step": 4600, "tokens": 1205862400, "loss": 3.3873, "lr": 0.01, "gnorm": 0.203, "tok_s": 147700, "elapsed_min": 136.57, "eta_min": 202.5, "cost_usd": 0.922}
97
+ {"step": 4650, "tokens": 1218969600, "loss": 3.5371, "lr": 0.01, "gnorm": 0.211, "tok_s": 147713, "elapsed_min": 138.05, "eta_min": 201.0, "cost_usd": 0.932}
98
+ {"step": 4700, "tokens": 1232076800, "loss": 3.367, "lr": 0.01, "gnorm": 0.207, "tok_s": 147719, "elapsed_min": 139.52, "eta_min": 199.5, "cost_usd": 0.942}
99
+ {"step": 4750, "tokens": 1245184000, "loss": 3.3518, "lr": 0.01, "gnorm": 0.201, "tok_s": 147713, "elapsed_min": 141.0, "eta_min": 198.0, "cost_usd": 0.952}
100
+ {"step": 4800, "tokens": 1258291200, "loss": 3.4905, "lr": 0.01, "gnorm": 0.221, "tok_s": 147663, "elapsed_min": 142.48, "eta_min": 196.5, "cost_usd": 0.962}
101
+ {"step": 4850, "tokens": 1271398400, "loss": 3.4146, "lr": 0.01, "gnorm": 0.18, "tok_s": 147707, "elapsed_min": 143.96, "eta_min": 195.1, "cost_usd": 0.972}
102
+ {"step": 4900, "tokens": 1284505600, "loss": 3.3662, "lr": 0.01, "gnorm": 0.214, "tok_s": 147645, "elapsed_min": 145.44, "eta_min": 193.6, "cost_usd": 0.981}
103
+ {"step": 4950, "tokens": 1297612800, "loss": 3.4718, "lr": 0.01, "gnorm": 0.207, "tok_s": 147518, "elapsed_min": 146.92, "eta_min": 192.2, "cost_usd": 0.991}
104
+ {"step": 5000, "tokens": 1310720000, "loss": 3.296, "lr": 0.01, "gnorm": 0.202, "tok_s": 147529, "elapsed_min": 148.4, "eta_min": 190.8, "cost_usd": 1.001}
105
+ {"step": 5000, "tokens": 1310720000, "val": {"dclm": 3.6777, "finemath": 2.8429, "fineweb": 3.482, "stack": 2.0981, "avg": 3.0252}}
106
+ {"step": 5050, "tokens": 1323827200, "loss": 3.3277, "lr": 0.01, "gnorm": 0.204, "tok_s": 144633, "elapsed_min": 149.91, "eta_min": 189.4, "cost_usd": 1.011}
107
+ {"step": 5100, "tokens": 1336934400, "loss": 3.2564, "lr": 0.01, "gnorm": 0.213, "tok_s": 147219, "elapsed_min": 151.4, "eta_min": 188.0, "cost_usd": 1.021}
108
+ {"step": 5150, "tokens": 1350041600, "loss": 3.2842, "lr": 0.01, "gnorm": 0.201, "tok_s": 147299, "elapsed_min": 152.88, "eta_min": 186.6, "cost_usd": 1.031}
109
+ {"step": 5200, "tokens": 1363148800, "loss": 3.4201, "lr": 0.01, "gnorm": 0.194, "tok_s": 147469, "elapsed_min": 154.36, "eta_min": 185.1, "cost_usd": 1.041}
110
+ {"step": 5250, "tokens": 1376256000, "loss": 3.3604, "lr": 0.01, "gnorm": 0.182, "tok_s": 147488, "elapsed_min": 155.84, "eta_min": 183.5, "cost_usd": 1.051}
111
+ {"step": 5300, "tokens": 1389363200, "loss": 3.3905, "lr": 0.01, "gnorm": 0.198, "tok_s": 147477, "elapsed_min": 157.32, "eta_min": 182.0, "cost_usd": 1.061}
112
+ {"step": 5350, "tokens": 1402470400, "loss": 3.291, "lr": 0.01, "gnorm": 0.176, "tok_s": 147416, "elapsed_min": 158.81, "eta_min": 180.6, "cost_usd": 1.07}
113
+ {"step": 5400, "tokens": 1415577600, "loss": 3.3489, "lr": 0.01, "gnorm": 0.192, "tok_s": 147506, "elapsed_min": 160.29, "eta_min": 179.1, "cost_usd": 1.08}
114
+ {"step": 5450, "tokens": 1428684800, "loss": 3.4236, "lr": 0.01, "gnorm": 0.205, "tok_s": 147559, "elapsed_min": 161.77, "eta_min": 177.5, "cost_usd": 1.09}
115
+ {"step": 5500, "tokens": 1441792000, "loss": 3.3177, "lr": 0.01, "gnorm": 0.186, "tok_s": 147599, "elapsed_min": 163.25, "eta_min": 176.0, "cost_usd": 1.1}
116
+ {"step": 5550, "tokens": 1454899200, "loss": 3.3136, "lr": 0.01, "gnorm": 0.22, "tok_s": 147587, "elapsed_min": 164.73, "eta_min": 174.5, "cost_usd": 1.11}
117
+ {"step": 5600, "tokens": 1468006400, "loss": 3.3891, "lr": 0.01, "gnorm": 0.187, "tok_s": 147644, "elapsed_min": 166.21, "eta_min": 173.0, "cost_usd": 1.12}
118
+ {"step": 5650, "tokens": 1481113600, "loss": 3.3442, "lr": 0.01, "gnorm": 0.212, "tok_s": 147646, "elapsed_min": 167.69, "eta_min": 171.4, "cost_usd": 1.13}
119
+ {"step": 5700, "tokens": 1494220800, "loss": 3.324, "lr": 0.01, "gnorm": 0.187, "tok_s": 147685, "elapsed_min": 169.17, "eta_min": 169.9, "cost_usd": 1.14}
120
+ {"step": 5750, "tokens": 1507328000, "loss": 3.3094, "lr": 0.01, "gnorm": 0.219, "tok_s": 147636, "elapsed_min": 170.65, "eta_min": 168.5, "cost_usd": 1.149}
121
+ {"step": 5800, "tokens": 1520435200, "loss": 3.2733, "lr": 0.01, "gnorm": 0.198, "tok_s": 147633, "elapsed_min": 172.12, "eta_min": 167.0, "cost_usd": 1.159}
122
+ {"step": 5850, "tokens": 1533542400, "loss": 3.3937, "lr": 0.01, "gnorm": 0.204, "tok_s": 147600, "elapsed_min": 173.6, "eta_min": 165.6, "cost_usd": 1.169}
123
+ {"step": 5900, "tokens": 1546649600, "loss": 3.2834, "lr": 0.01, "gnorm": 0.211, "tok_s": 147601, "elapsed_min": 175.08, "eta_min": 164.1, "cost_usd": 1.179}
124
+ {"step": 5950, "tokens": 1559756800, "loss": 3.3333, "lr": 0.01, "gnorm": 0.198, "tok_s": 147635, "elapsed_min": 176.56, "eta_min": 162.6, "cost_usd": 1.189}
125
+ {"step": 6000, "tokens": 1572864000, "loss": 3.2617, "lr": 0.01, "gnorm": 0.188, "tok_s": 147732, "elapsed_min": 178.04, "eta_min": 161.0, "cost_usd": 1.199}
126
+ {"step": 6000, "tokens": 1572864000, "val": {"dclm": 3.6692, "finemath": 2.8182, "fineweb": 3.4566, "stack": 2.0984, "avg": 3.0106}}
127
+ {"step": 6050, "tokens": 1585971200, "loss": 3.383, "lr": 0.01, "gnorm": 0.211, "tok_s": 144705, "elapsed_min": 179.55, "eta_min": 159.6, "cost_usd": 1.209}
128
+ {"step": 6100, "tokens": 1599078400, "loss": 3.365, "lr": 0.01, "gnorm": 0.218, "tok_s": 147334, "elapsed_min": 181.04, "eta_min": 158.3, "cost_usd": 1.219}
129
+ {"step": 6150, "tokens": 1612185600, "loss": 3.3964, "lr": 0.01, "gnorm": 0.207, "tok_s": 147385, "elapsed_min": 182.52, "eta_min": 156.9, "cost_usd": 1.229}
130
+ {"step": 6200, "tokens": 1625292800, "loss": 3.3997, "lr": 0.01, "gnorm": 0.176, "tok_s": 147367, "elapsed_min": 184.0, "eta_min": 155.5, "cost_usd": 1.238}
131
+ {"step": 6250, "tokens": 1638400000, "loss": 3.3697, "lr": 0.01, "gnorm": 0.192, "tok_s": 147364, "elapsed_min": 185.48, "eta_min": 154.0, "cost_usd": 1.248}
132
+ {"step": 6300, "tokens": 1651507200, "loss": 3.269, "lr": 0.01, "gnorm": 0.223, "tok_s": 147283, "elapsed_min": 186.97, "eta_min": 152.5, "cost_usd": 1.258}
133
+ {"step": 6350, "tokens": 1664614400, "loss": 3.4083, "lr": 0.01, "gnorm": 0.212, "tok_s": 147345, "elapsed_min": 188.45, "eta_min": 151.1, "cost_usd": 1.268}
134
+ {"step": 6400, "tokens": 1677721600, "loss": 3.3175, "lr": 0.01, "gnorm": 0.205, "tok_s": 147354, "elapsed_min": 189.93, "eta_min": 149.6, "cost_usd": 1.278}
135
+ {"step": 6450, "tokens": 1690828800, "loss": 3.3714, "lr": 0.01, "gnorm": 0.188, "tok_s": 147519, "elapsed_min": 191.41, "eta_min": 148.0, "cost_usd": 1.288}
136
+ {"step": 6500, "tokens": 1703936000, "loss": 3.3093, "lr": 0.01, "gnorm": 0.206, "tok_s": 147457, "elapsed_min": 192.89, "eta_min": 146.4, "cost_usd": 1.298}
137
+ {"step": 6550, "tokens": 1717043200, "loss": 3.2643, "lr": 0.01, "gnorm": 0.224, "tok_s": 147430, "elapsed_min": 194.38, "eta_min": 145.0, "cost_usd": 1.308}
138
+ {"step": 6600, "tokens": 1730150400, "loss": 3.3451, "lr": 0.01, "gnorm": 0.212, "tok_s": 147482, "elapsed_min": 195.86, "eta_min": 143.5, "cost_usd": 1.318}
139
+ {"step": 6650, "tokens": 1743257600, "loss": 3.4278, "lr": 0.01, "gnorm": 0.187, "tok_s": 147413, "elapsed_min": 197.34, "eta_min": 142.0, "cost_usd": 1.327}
140
+ {"step": 6700, "tokens": 1756364800, "loss": 3.4111, "lr": 0.01, "gnorm": 0.231, "tok_s": 147451, "elapsed_min": 198.82, "eta_min": 140.6, "cost_usd": 1.337}
141
+ {"step": 6750, "tokens": 1769472000, "loss": 3.3189, "lr": 0.01, "gnorm": 0.183, "tok_s": 147484, "elapsed_min": 200.3, "eta_min": 139.1, "cost_usd": 1.347}
142
+ {"step": 6800, "tokens": 1782579200, "loss": 3.3474, "lr": 0.01, "gnorm": 0.217, "tok_s": 147424, "elapsed_min": 201.78, "eta_min": 137.6, "cost_usd": 1.357}
143
+ {"step": 6850, "tokens": 1795686400, "loss": 3.325, "lr": 0.01, "gnorm": 0.191, "tok_s": 147397, "elapsed_min": 203.26, "eta_min": 136.2, "cost_usd": 1.367}
144
+ {"step": 6900, "tokens": 1808793600, "loss": 3.3576, "lr": 0.01, "gnorm": 0.185, "tok_s": 147481, "elapsed_min": 204.75, "eta_min": 134.6, "cost_usd": 1.377}
145
+ {"step": 6950, "tokens": 1821900800, "loss": 3.3322, "lr": 0.01, "gnorm": 0.194, "tok_s": 147506, "elapsed_min": 206.23, "eta_min": 133.1, "cost_usd": 1.387}
146
+ {"step": 7000, "tokens": 1835008000, "loss": 3.4271, "lr": 0.01, "gnorm": 0.19, "tok_s": 147393, "elapsed_min": 207.71, "eta_min": 131.7, "cost_usd": 1.397}
147
+ {"step": 7000, "tokens": 1835008000, "val": {"dclm": 3.6657, "finemath": 2.8171, "fineweb": 3.4583, "stack": 2.1101, "avg": 3.0128}}
148
+ {"step": 7050, "tokens": 1848115200, "loss": 3.3185, "lr": 0.01, "gnorm": 0.188, "tok_s": 144534, "elapsed_min": 209.22, "eta_min": 130.3, "cost_usd": 1.407}
149
+ {"step": 7100, "tokens": 1861222400, "loss": 3.3442, "lr": 0.01, "gnorm": 0.204, "tok_s": 147195, "elapsed_min": 210.7, "eta_min": 128.8, "cost_usd": 1.416}
150
+ {"step": 7150, "tokens": 1874329600, "loss": 3.336, "lr": 0.01, "gnorm": 0.229, "tok_s": 147385, "elapsed_min": 212.19, "eta_min": 127.3, "cost_usd": 1.426}
151
+ {"step": 7200, "tokens": 1887436800, "loss": 3.3662, "lr": 0.01, "gnorm": 0.218, "tok_s": 147421, "elapsed_min": 213.67, "eta_min": 125.8, "cost_usd": 1.436}
152
+ {"step": 7250, "tokens": 1900544000, "loss": 3.2873, "lr": 0.01, "gnorm": 0.23, "tok_s": 147224, "elapsed_min": 215.15, "eta_min": 124.4, "cost_usd": 1.446}
153
+ {"step": 7300, "tokens": 1913651200, "loss": 3.3363, "lr": 0.01, "gnorm": 0.196, "tok_s": 147272, "elapsed_min": 216.64, "eta_min": 123.0, "cost_usd": 1.456}
154
+ {"step": 7350, "tokens": 1926758400, "loss": 3.2998, "lr": 0.01, "gnorm": 0.238, "tok_s": 147237, "elapsed_min": 218.12, "eta_min": 121.5, "cost_usd": 1.466}
155
+ {"step": 7400, "tokens": 1939865600, "loss": 3.4038, "lr": 0.01, "gnorm": 0.221, "tok_s": 147172, "elapsed_min": 219.6, "eta_min": 120.0, "cost_usd": 1.476}
156
+ {"step": 7450, "tokens": 1952972800, "loss": 3.3251, "lr": 0.01, "gnorm": 0.184, "tok_s": 147135, "elapsed_min": 221.09, "eta_min": 118.6, "cost_usd": 1.486}
157
+ {"step": 7500, "tokens": 1966080000, "loss": 3.3991, "lr": 0.01, "gnorm": 0.19, "tok_s": 147239, "elapsed_min": 222.57, "eta_min": 117.1, "cost_usd": 1.496}
158
+ {"step": 7550, "tokens": 1979187200, "loss": 3.3152, "lr": 0.01, "gnorm": 0.22, "tok_s": 147216, "elapsed_min": 224.06, "eta_min": 115.6, "cost_usd": 1.506}
159
+ {"step": 7600, "tokens": 1992294400, "loss": 3.3676, "lr": 0.01, "gnorm": 0.199, "tok_s": 147205, "elapsed_min": 225.54, "eta_min": 114.1, "cost_usd": 1.515}
160
+ {"step": 7650, "tokens": 2005401600, "loss": 3.3384, "lr": 0.01, "gnorm": 0.213, "tok_s": 147225, "elapsed_min": 227.02, "eta_min": 112.6, "cost_usd": 1.525}
161
+ {"step": 7700, "tokens": 2018508800, "loss": 3.2688, "lr": 0.01, "gnorm": 0.211, "tok_s": 147127, "elapsed_min": 228.51, "eta_min": 111.1, "cost_usd": 1.535}
162
+ {"step": 7750, "tokens": 2031616000, "loss": 3.2679, "lr": 0.01, "gnorm": 0.204, "tok_s": 147049, "elapsed_min": 229.99, "eta_min": 109.7, "cost_usd": 1.545}
163
+ {"step": 7800, "tokens": 2044723200, "loss": 3.3241, "lr": 0.01, "gnorm": 0.224, "tok_s": 147223, "elapsed_min": 231.48, "eta_min": 108.2, "cost_usd": 1.555}
164
+ {"step": 7850, "tokens": 2057830400, "loss": 3.3268, "lr": 0.01, "gnorm": 0.196, "tok_s": 147375, "elapsed_min": 232.96, "eta_min": 106.6, "cost_usd": 1.565}
165
+ {"step": 7900, "tokens": 2070937600, "loss": 3.3042, "lr": 0.01, "gnorm": 0.215, "tok_s": 147361, "elapsed_min": 234.44, "eta_min": 105.1, "cost_usd": 1.575}
166
+ {"step": 7950, "tokens": 2084044800, "loss": 3.3191, "lr": 0.01, "gnorm": 0.238, "tok_s": 147325, "elapsed_min": 235.93, "eta_min": 103.6, "cost_usd": 1.585}
167
+ {"step": 8000, "tokens": 2097152000, "loss": 3.2666, "lr": 0.01, "gnorm": 0.194, "tok_s": 147275, "elapsed_min": 237.41, "eta_min": 102.1, "cost_usd": 1.595}
168
+ {"step": 8000, "tokens": 2097152000, "val": {"dclm": 3.6568, "finemath": 2.8131, "fineweb": 3.4508, "stack": 2.0667, "avg": 2.9968}}
169
+ {"step": 8050, "tokens": 2110259200, "loss": 3.2709, "lr": 0.01, "gnorm": 0.196, "tok_s": 144217, "elapsed_min": 238.92, "eta_min": 100.8, "cost_usd": 1.605}
170
+ {"step": 8100, "tokens": 2123366400, "loss": 3.3756, "lr": 0.01, "gnorm": 0.196, "tok_s": 146857, "elapsed_min": 240.41, "eta_min": 99.4, "cost_usd": 1.615}
171
+ {"step": 8150, "tokens": 2136473600, "loss": 3.2967, "lr": 0.01, "gnorm": 0.463, "tok_s": 147145, "elapsed_min": 241.9, "eta_min": 97.8, "cost_usd": 1.624}
172
+ {"step": 8200, "tokens": 2149580800, "loss": 3.3779, "lr": 0.01, "gnorm": 0.198, "tok_s": 147083, "elapsed_min": 243.38, "eta_min": 96.3, "cost_usd": 1.634}
173
+ {"step": 8250, "tokens": 2162688000, "loss": 3.4325, "lr": 0.01, "gnorm": 0.2, "tok_s": 146965, "elapsed_min": 244.87, "eta_min": 94.9, "cost_usd": 1.644}
174
+ {"step": 8300, "tokens": 2175795200, "loss": 3.2128, "lr": 0.01, "gnorm": 0.211, "tok_s": 146952, "elapsed_min": 246.35, "eta_min": 93.5, "cost_usd": 1.654}
175
+ {"step": 8350, "tokens": 2188902400, "loss": 3.2421, "lr": 0.01, "gnorm": 0.197, "tok_s": 146971, "elapsed_min": 247.84, "eta_min": 92.0, "cost_usd": 1.664}
176
+ {"step": 8400, "tokens": 2202009600, "loss": 3.223, "lr": 0.01, "gnorm": 0.239, "tok_s": 147013, "elapsed_min": 249.33, "eta_min": 90.5, "cost_usd": 1.674}
177
+ {"step": 8450, "tokens": 2215116800, "loss": 3.2426, "lr": 0.01, "gnorm": 0.21, "tok_s": 146981, "elapsed_min": 250.81, "eta_min": 89.0, "cost_usd": 1.684}
178
+ {"step": 8500, "tokens": 2228224000, "loss": 3.2874, "lr": 0.01, "gnorm": 0.205, "tok_s": 147028, "elapsed_min": 252.3, "eta_min": 87.5, "cost_usd": 1.694}
179
+ {"step": 8550, "tokens": 2241331200, "loss": 3.2613, "lr": 0.01, "gnorm": 0.189, "tok_s": 146976, "elapsed_min": 253.78, "eta_min": 86.0, "cost_usd": 1.704}
180
+ {"step": 8600, "tokens": 2254438400, "loss": 3.4587, "lr": 0.01, "gnorm": 0.21, "tok_s": 147006, "elapsed_min": 255.27, "eta_min": 84.5, "cost_usd": 1.714}
181
+ {"step": 8650, "tokens": 2267545600, "loss": 3.3467, "lr": 0.01, "gnorm": 0.212, "tok_s": 146999, "elapsed_min": 256.76, "eta_min": 83.0, "cost_usd": 1.724}
182
+ {"step": 8700, "tokens": 2280652800, "loss": 3.375, "lr": 0.01, "gnorm": 0.209, "tok_s": 146970, "elapsed_min": 258.24, "eta_min": 81.6, "cost_usd": 1.733}
183
+ {"step": 8750, "tokens": 2293760000, "loss": 3.3718, "lr": 0.01, "gnorm": 0.189, "tok_s": 146956, "elapsed_min": 259.73, "eta_min": 80.1, "cost_usd": 1.743}
184
+ {"step": 8800, "tokens": 2306867200, "loss": 3.4497, "lr": 0.01, "gnorm": 0.205, "tok_s": 147128, "elapsed_min": 261.21, "eta_min": 78.6, "cost_usd": 1.753}
185
+ {"step": 8850, "tokens": 2319974400, "loss": 3.3151, "lr": 0.01, "gnorm": 0.227, "tok_s": 147095, "elapsed_min": 262.7, "eta_min": 77.0, "cost_usd": 1.763}
186
+ {"step": 8900, "tokens": 2333081600, "loss": 3.3449, "lr": 0.01, "gnorm": 0.215, "tok_s": 147040, "elapsed_min": 264.19, "eta_min": 75.6, "cost_usd": 1.773}
187
+ {"step": 8950, "tokens": 2346188800, "loss": 3.2643, "lr": 0.01, "gnorm": 0.19, "tok_s": 146962, "elapsed_min": 265.67, "eta_min": 74.1, "cost_usd": 1.783}
188
+ {"step": 9000, "tokens": 2359296000, "loss": 3.3343, "lr": 0.01, "gnorm": 0.201, "tok_s": 146906, "elapsed_min": 267.16, "eta_min": 72.7, "cost_usd": 1.793}
189
+ {"step": 9000, "tokens": 2359296000, "val": {"dclm": 3.6591, "finemath": 2.7944, "fineweb": 3.4467, "stack": 2.0713, "avg": 2.9929}}
190
+ {"step": 9050, "tokens": 2372403200, "loss": 3.4646, "lr": 0.01, "gnorm": 0.249, "tok_s": 144128, "elapsed_min": 268.67, "eta_min": 71.2, "cost_usd": 1.803}
191
+ {"step": 9100, "tokens": 2385510400, "loss": 3.3362, "lr": 0.01, "gnorm": 0.197, "tok_s": 147023, "elapsed_min": 270.16, "eta_min": 69.7, "cost_usd": 1.813}
192
+ {"step": 9150, "tokens": 2398617600, "loss": 3.2802, "lr": 0.01, "gnorm": 0.171, "tok_s": 146871, "elapsed_min": 271.65, "eta_min": 68.2, "cost_usd": 1.823}
193
+ {"step": 9200, "tokens": 2411724800, "loss": 3.351, "lr": 0.009808, "gnorm": 0.222, "tok_s": 146995, "elapsed_min": 273.13, "eta_min": 66.7, "cost_usd": 1.833}
194
+ {"step": 9250, "tokens": 2424832000, "loss": 3.3431, "lr": 0.009589, "gnorm": 0.198, "tok_s": 147086, "elapsed_min": 274.62, "eta_min": 65.2, "cost_usd": 1.843}
195
+ {"step": 9300, "tokens": 2437939200, "loss": 3.3836, "lr": 0.009371, "gnorm": 0.202, "tok_s": 147170, "elapsed_min": 276.1, "eta_min": 63.7, "cost_usd": 1.852}
196
+ {"step": 9350, "tokens": 2451046400, "loss": 3.3469, "lr": 0.009152, "gnorm": 0.196, "tok_s": 147143, "elapsed_min": 277.59, "eta_min": 62.2, "cost_usd": 1.862}
197
+ {"step": 9400, "tokens": 2464153600, "loss": 3.3016, "lr": 0.008934, "gnorm": 0.194, "tok_s": 147179, "elapsed_min": 279.07, "eta_min": 60.7, "cost_usd": 1.872}
198
+ {"step": 9450, "tokens": 2477260800, "loss": 3.3088, "lr": 0.008716, "gnorm": 0.172, "tok_s": 147070, "elapsed_min": 280.56, "eta_min": 59.2, "cost_usd": 1.882}
199
+ {"step": 9500, "tokens": 2490368000, "loss": 3.2286, "lr": 0.008497, "gnorm": 0.2, "tok_s": 147080, "elapsed_min": 282.04, "eta_min": 57.7, "cost_usd": 1.892}
200
+ {"step": 9550, "tokens": 2503475200, "loss": 3.3034, "lr": 0.008279, "gnorm": 0.191, "tok_s": 147211, "elapsed_min": 283.53, "eta_min": 56.2, "cost_usd": 1.902}
201
+ {"step": 9600, "tokens": 2516582400, "loss": 3.361, "lr": 0.00806, "gnorm": 0.196, "tok_s": 147228, "elapsed_min": 285.01, "eta_min": 54.7, "cost_usd": 1.912}
202
+ {"step": 9650, "tokens": 2529689600, "loss": 3.2527, "lr": 0.007842, "gnorm": 0.192, "tok_s": 147138, "elapsed_min": 286.5, "eta_min": 53.3, "cost_usd": 1.922}
203
+ {"step": 9700, "tokens": 2542796800, "loss": 3.2002, "lr": 0.007623, "gnorm": 0.181, "tok_s": 147214, "elapsed_min": 287.98, "eta_min": 51.8, "cost_usd": 1.932}
204
+ {"step": 9750, "tokens": 2555904000, "loss": 3.2591, "lr": 0.007405, "gnorm": 0.205, "tok_s": 147270, "elapsed_min": 289.46, "eta_min": 50.3, "cost_usd": 1.942}
205
+ {"step": 9800, "tokens": 2569011200, "loss": 3.1652, "lr": 0.007187, "gnorm": 0.184, "tok_s": 147312, "elapsed_min": 290.95, "eta_min": 48.8, "cost_usd": 1.951}
206
+ {"step": 9850, "tokens": 2582118400, "loss": 3.1451, "lr": 0.006968, "gnorm": 0.185, "tok_s": 147233, "elapsed_min": 292.43, "eta_min": 47.3, "cost_usd": 1.961}
207
+ {"step": 9900, "tokens": 2595225600, "loss": 3.2472, "lr": 0.00675, "gnorm": 0.186, "tok_s": 147200, "elapsed_min": 293.91, "eta_min": 45.8, "cost_usd": 1.971}
208
+ {"step": 9950, "tokens": 2608332800, "loss": 3.2923, "lr": 0.006531, "gnorm": 0.167, "tok_s": 147218, "elapsed_min": 295.4, "eta_min": 44.3, "cost_usd": 1.981}
209
+ {"step": 10000, "tokens": 2621440000, "loss": 3.2263, "lr": 0.006313, "gnorm": 0.173, "tok_s": 147266, "elapsed_min": 296.88, "eta_min": 42.8, "cost_usd": 1.991}
210
+ {"step": 10000, "tokens": 2621440000, "val": {"dclm": 3.5842, "finemath": 2.7228, "fineweb": 3.3733, "stack": 1.9698, "avg": 2.9125}}
211
+ {"step": 10050, "tokens": 2634547200, "loss": 3.2422, "lr": 0.006094, "gnorm": 0.199, "tok_s": 144218, "elapsed_min": 298.4, "eta_min": 41.4, "cost_usd": 2.001}
212
+ {"step": 10100, "tokens": 2647654400, "loss": 3.3557, "lr": 0.005876, "gnorm": 0.198, "tok_s": 147091, "elapsed_min": 299.88, "eta_min": 39.9, "cost_usd": 2.011}
213
+ {"step": 10150, "tokens": 2660761600, "loss": 3.351, "lr": 0.005657, "gnorm": 0.195, "tok_s": 146844, "elapsed_min": 301.37, "eta_min": 38.4, "cost_usd": 2.021}
214
+ {"step": 10200, "tokens": 2673868800, "loss": 3.3251, "lr": 0.005439, "gnorm": 0.199, "tok_s": 147023, "elapsed_min": 302.85, "eta_min": 37.0, "cost_usd": 2.031}
215
+ {"step": 10250, "tokens": 2686976000, "loss": 3.2239, "lr": 0.005221, "gnorm": 0.226, "tok_s": 147141, "elapsed_min": 304.34, "eta_min": 35.5, "cost_usd": 2.041}
216
+ {"step": 10300, "tokens": 2700083200, "loss": 3.2378, "lr": 0.005002, "gnorm": 0.229, "tok_s": 147271, "elapsed_min": 305.82, "eta_min": 34.0, "cost_usd": 2.051}
217
+ {"step": 10350, "tokens": 2713190400, "loss": 3.2707, "lr": 0.004784, "gnorm": 0.192, "tok_s": 147308, "elapsed_min": 307.31, "eta_min": 32.4, "cost_usd": 2.06}
218
+ {"step": 10400, "tokens": 2726297600, "loss": 3.1847, "lr": 0.004565, "gnorm": 0.191, "tok_s": 147390, "elapsed_min": 308.79, "eta_min": 31.0, "cost_usd": 2.07}
219
+ {"step": 10450, "tokens": 2739404800, "loss": 3.2019, "lr": 0.004347, "gnorm": 0.202, "tok_s": 147309, "elapsed_min": 310.27, "eta_min": 29.5, "cost_usd": 2.08}
220
+ {"step": 10500, "tokens": 2752512000, "loss": 3.259, "lr": 0.004128, "gnorm": 0.18, "tok_s": 147329, "elapsed_min": 311.75, "eta_min": 28.0, "cost_usd": 2.09}
221
+ {"step": 10550, "tokens": 2765619200, "loss": 3.1176, "lr": 0.00391, "gnorm": 0.195, "tok_s": 147343, "elapsed_min": 313.24, "eta_min": 26.5, "cost_usd": 2.1}
222
+ {"step": 10600, "tokens": 2778726400, "loss": 3.2894, "lr": 0.003692, "gnorm": 0.196, "tok_s": 147356, "elapsed_min": 314.72, "eta_min": 25.0, "cost_usd": 2.11}
223
+ {"step": 10650, "tokens": 2791833600, "loss": 3.1497, "lr": 0.003473, "gnorm": 0.197, "tok_s": 147307, "elapsed_min": 316.2, "eta_min": 23.5, "cost_usd": 2.12}
224
+ {"step": 10700, "tokens": 2804940800, "loss": 3.2232, "lr": 0.003255, "gnorm": 0.187, "tok_s": 147436, "elapsed_min": 317.68, "eta_min": 22.1, "cost_usd": 2.13}
225
+ {"step": 10750, "tokens": 2818048000, "loss": 3.1616, "lr": 0.003036, "gnorm": 0.176, "tok_s": 147414, "elapsed_min": 319.16, "eta_min": 20.6, "cost_usd": 2.14}
226
+ {"step": 10800, "tokens": 2831155200, "loss": 3.1348, "lr": 0.002818, "gnorm": 0.203, "tok_s": 147361, "elapsed_min": 320.65, "eta_min": 19.1, "cost_usd": 2.149}
227
+ {"step": 10850, "tokens": 2844262400, "loss": 3.1003, "lr": 0.002599, "gnorm": 0.174, "tok_s": 147355, "elapsed_min": 322.13, "eta_min": 17.6, "cost_usd": 2.159}
228
+ {"step": 10900, "tokens": 2857369600, "loss": 3.0696, "lr": 0.002381, "gnorm": 0.149, "tok_s": 147174, "elapsed_min": 323.61, "eta_min": 16.1, "cost_usd": 2.169}
229
+ {"step": 10950, "tokens": 2870476800, "loss": 3.0913, "lr": 0.002163, "gnorm": 0.19, "tok_s": 147325, "elapsed_min": 325.1, "eta_min": 14.7, "cost_usd": 2.179}
230
+ {"step": 11000, "tokens": 2883584000, "loss": 3.132, "lr": 0.001944, "gnorm": 0.166, "tok_s": 147234, "elapsed_min": 326.58, "eta_min": 13.2, "cost_usd": 2.189}
231
+ {"step": 11000, "tokens": 2883584000, "val": {"dclm": 3.4975, "finemath": 2.623, "fineweb": 3.2736, "stack": 1.8298, "avg": 2.806}}
232
+ {"step": 11050, "tokens": 2896691200, "loss": 3.2088, "lr": 0.001726, "gnorm": 0.146, "tok_s": 144409, "elapsed_min": 328.09, "eta_min": 11.7, "cost_usd": 2.199}
233
+ {"step": 11100, "tokens": 2909798400, "loss": 3.0317, "lr": 0.001507, "gnorm": 0.132, "tok_s": 147325, "elapsed_min": 329.58, "eta_min": 10.2, "cost_usd": 2.209}
234
+ {"step": 11150, "tokens": 2922905600, "loss": 3.0893, "lr": 0.001289, "gnorm": 0.137, "tok_s": 147189, "elapsed_min": 331.06, "eta_min": 8.7, "cost_usd": 2.219}
235
+ {"step": 11200, "tokens": 2936012800, "loss": 3.2157, "lr": 0.00107, "gnorm": 0.124, "tok_s": 147411, "elapsed_min": 332.54, "eta_min": 7.2, "cost_usd": 2.229}
236
+ {"step": 11250, "tokens": 2949120000, "loss": 3.1137, "lr": 0.000852, "gnorm": 0.122, "tok_s": 147329, "elapsed_min": 334.02, "eta_min": 5.8, "cost_usd": 2.239}
237
+ {"step": 11300, "tokens": 2962227200, "loss": 3.179, "lr": 0.000633, "gnorm": 0.132, "tok_s": 147356, "elapsed_min": 335.51, "eta_min": 4.3, "cost_usd": 2.249}
238
+ {"step": 11350, "tokens": 2975334400, "loss": 3.0909, "lr": 0.000415, "gnorm": 0.123, "tok_s": 147325, "elapsed_min": 336.99, "eta_min": 2.8, "cost_usd": 2.258}
239
+ {"step": 11400, "tokens": 2988441600, "loss": 2.9855, "lr": 0.000197, "gnorm": 0.109, "tok_s": 147397, "elapsed_min": 338.47, "eta_min": 1.3, "cost_usd": 2.268}
240
+ {"step": 11444, "tokens": 2999975936, "loss": 3.1066, "lr": 4e-06, "gnorm": 0.117, "tok_s": 147429, "elapsed_min": 339.78, "eta_min": 0.0, "cost_usd": 2.277}
241
+ {"step": 11444, "tokens": 2999975936, "val": {"dclm": 3.4559, "finemath": 2.5755, "fineweb": 3.2282, "stack": 1.7677, "avg": 2.7568}}
training_summary.json ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "size": "5M",
3
+ "params": 5115456,
4
+ "config": {
5
+ "vocab_size": 8192,
6
+ "hidden_size": 192,
7
+ "intermediate_size": 512,
8
+ "num_hidden_layers": 9,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "max_position_embeddings": 2048,
12
+ "rope_theta": 10000.0,
13
+ "rms_norm_eps": 1e-05,
14
+ "initializer_range": 0.02
15
+ },
16
+ "tokens": 2999975936,
17
+ "steps": 11444,
18
+ "batch_tokens": 262144,
19
+ "seq_len": 2048,
20
+ "opt": "muon",
21
+ "lr": 0.01,
22
+ "wd": 0.1,
23
+ "warmup_frac": 0.01,
24
+ "decay_frac": 0.2,
25
+ "gpu": "Tesla T4",
26
+ "train_minutes": 339.8,
27
+ "final_val": {
28
+ "dclm": 3.4559,
29
+ "finemath": 2.5755,
30
+ "fineweb": 3.2282,
31
+ "stack": 1.7677,
32
+ "avg": 2.7568
33
+ },
34
+ "samples": [
35
+ "The meaning of life is the same as the meaning of life. The meaning of life is the same as the meaning of life.\nThe meaning of life is the same as the meaning of life. It means that we know what we are doing and how we can do",
36
+ "Once upon a time, the world was not as much of an issue. The world had to be more than just a single day and a week.\nThe world was very different from what we have today. We are all in our own homes. It is a great",
37
+ "def is_prime(n): 10\n# 2. The number of digits in the first digit is 3, and the last digit is 4.\n# 3. The number of digits in the second digit is 5, and the last digit",
38
+ "The derivative of x^2 is the product of its value. The derivative of y^2 is the product of its value.\n\n## 1. What are the derivatives?\n\n### 3. How do you find the derivative of x^2?\n\n### "
39
+ ],
40
+ "parity_max_diff": 1.621246337890625e-05
41
+ }