Deep35 continual checkpoint step 50000

last-checkpoint/config.json

@@ -5,7 +5,7 @@
   "attention_bias": false,
   "attention_dropout": 0.0,
   "bos_token_id": 1,
-  "dtype": "float32",
+  "dtype": "bfloat16",
   "eos_token_id": 2,
   "head_dim": 64,
   "hidden_act": "silu",
@@ -18,12 +18,13 @@
   "num_attention_heads": 6,
   "num_hidden_layers": 15,
   "num_key_value_heads": 1,
+  "pad_token_id": 2,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": null,
   "rope_theta": 10000.0,
   "tie_word_embeddings": true,
   "transformers_version": "4.57.6",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 32000
 }

last-checkpoint/continual_state.json

@@ -0,0 +1,37 @@
+{
+  "global_step": 50000,
+  "sequence_tokens_emitted": 6553731072,
+  "mix": [
+    0.9,
+    0.05,
+    0.03,
+    0.0175,
+    0.0025
+  ],
+  "fineweb": "HuggingFaceFW/fineweb-edu:default",
+  "ultra_multi": "openbmb/Ultra-FineWeb-L3:Ultra-FineWeb-L3-en-Multi-Style-Synthetic",
+  "ultra_qa": "openbmb/Ultra-FineWeb-L3:Ultra-FineWeb-L3-en-QA-Synthetic",
+  "rewrite": "appvoid/rewrite6",
+  "noprompt": "appvoid/no-prompt-75k",
+  "early_mix": [
+    0.9,
+    0.05,
+    0.03,
+    0.0175,
+    0.0025
+  ],
+  "late_mix": [
+    0.8,
+    0.09,
+    0.05,
+    0.05,
+    0.01
+  ],
+  "formatter_version": 5,
+  "rewrite6_format": "join non-empty instruction/text/output; empty components valid",
+  "noprompt_format": "stored text field exactly; empty rows skipped",
+  "transition_global_step": 49000,
+  "tokens_at_transition": 6422528000,
+  "legacy_tokens_per_update": 131072,
+  "continual_tokens_per_update": 294912
+}

last-checkpoint/data_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48240d82fb4ec513e553f4160185e307330919d819490b3ea8c78f24bf43d62c
+size 44927

last-checkpoint/generation_config.json

@@ -4,5 +4,6 @@
   "eos_token_id": [
     2
   ],
+  "pad_token_id": 2,
   "transformers_version": "4.57.6"
 }

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04b88033e1475ded1b606254082140ba945ee741465a4048f79714c67a15c125
-size 140637408
+oid sha256:db8bbfa71dd2c2786bb4ff3d73fce3c17be04939a5d04af2a34e8de87f948a02
+size 70326256

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76242aeb26df7d2635494c9a4457ee49fc3ab69778623f44dce5b289df7e2652
-size 281362507
+oid sha256:e304a2907e9eea97ee87fd6c8a8def9ea396d910ed695aa387c188a301d2ae11
+size 140740171

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05d83c3ee3c8e95e538ce7ad3336285b6895c62c33531f37b7797db24b42b5d5
+oid sha256:7585603e8a9c01c60e20cbe4cd2940208638c80511bb33e00a30bdc219d4888a
 size 14645

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:12f85f2f5e67e5d6880ff3e7c7ccbf88dff86e32833c0b30555d5a8922acc43f
+oid sha256:3a76536743b0948938ff1099639f0d5a5c75d477b200f6e590e96e6a6203065f
 size 1465

last-checkpoint/special_tokens_map.json

@@ -13,6 +13,7 @@
     "rstrip": false,
     "single_word": false
   },
+  "pad_token": "</s>",
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

last-checkpoint/tokenizer_config.json

@@ -33,8 +33,8 @@
   "eos_token": "</s>",
   "extra_special_tokens": {},
   "legacy": false,
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": null,
+  "model_max_length": 1000000000000000000000000000000,
+  "pad_token": "</s>",
   "padding_side": "right",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6422523396335229,
+  "epoch": 5e-07,
   "eval_steps": 500,
-  "global_step": 49000,
+  "global_step": 50000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -35190,10 +35190,726 @@
       "eval_samples_per_second": 31.826,
       "eval_steps_per_second": 0.509,
       "step": 49000
+    },
+    {
+      "epoch": 5e-09,
+      "grad_norm": 0.7265625,
+      "learning_rate": 0.00010745841306752502,
+      "loss": 3.1778,
+      "step": 49010
+    },
+    {
+      "epoch": 1e-08,
+      "grad_norm": 0.63671875,
+      "learning_rate": 0.00010734454578537477,
+      "loss": 3.1465,
+      "step": 49020
+    },
+    {
+      "epoch": 1.5e-08,
+      "grad_norm": 0.609375,
+      "learning_rate": 0.00010723072864920421,
+      "loss": 3.1177,
+      "step": 49030
+    },
+    {
+      "epoch": 2e-08,
+      "grad_norm": 0.625,
+      "learning_rate": 0.00010711696175800602,
+      "loss": 3.1645,
+      "step": 49040
+    },
+    {
+      "epoch": 2.5e-08,
+      "grad_norm": 0.63671875,
+      "learning_rate": 0.0001070032452107293,
+      "loss": 3.1674,
+      "step": 49050
+    },
+    {
+      "epoch": 3e-08,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0001068895791062792,
+      "loss": 3.1582,
+      "step": 49060
+    },
+    {
+      "epoch": 3.5e-08,
+      "grad_norm": 0.62890625,
+      "learning_rate": 0.00010677596354351713,
+      "loss": 3.1465,
+      "step": 49070
+    },
+    {
+      "epoch": 4e-08,
+      "grad_norm": 0.6953125,
+      "learning_rate": 0.00010666239862126061,
+      "loss": 3.1316,
+      "step": 49080
+    },
+    {
+      "epoch": 4.5e-08,
+      "grad_norm": 0.66015625,
+      "learning_rate": 0.00010654888443828301,
+      "loss": 3.1577,
+      "step": 49090
+    },
+    {
+      "epoch": 5e-08,
+      "grad_norm": 0.6953125,
+      "learning_rate": 0.00010643542109331357,
+      "loss": 3.1044,
+      "step": 49100
+    },
+    {
+      "epoch": 5.5e-08,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.00010632200868503724,
+      "loss": 3.1189,
+      "step": 49110
+    },
+    {
+      "epoch": 6e-08,
+      "grad_norm": 0.59765625,
+      "learning_rate": 0.00010620864731209483,
+      "loss": 3.0972,
+      "step": 49120
+    },
+    {
+      "epoch": 6.5e-08,
+      "grad_norm": 0.6171875,
+      "learning_rate": 0.00010609533707308268,
+      "loss": 3.1177,
+      "step": 49130
+    },
+    {
+      "epoch": 7e-08,
+      "grad_norm": 0.6328125,
+      "learning_rate": 0.00010598207806655264,
+      "loss": 3.1101,
+      "step": 49140
+    },
+    {
+      "epoch": 7.5e-08,
+      "grad_norm": 0.6953125,
+      "learning_rate": 0.00010586887039101204,
+      "loss": 3.1387,
+      "step": 49150
+    },
+    {
+      "epoch": 8e-08,
+      "grad_norm": 0.6796875,
+      "learning_rate": 0.00010575571414492357,
+      "loss": 3.1128,
+      "step": 49160
+    },
+    {
+      "epoch": 8.5e-08,
+      "grad_norm": 0.65625,
+      "learning_rate": 0.0001056426094267051,
+      "loss": 3.1292,
+      "step": 49170
+    },
+    {
+      "epoch": 9e-08,
+      "grad_norm": 0.64453125,
+      "learning_rate": 0.0001055295563347298,
+      "loss": 3.1134,
+      "step": 49180
+    },
+    {
+      "epoch": 9.5e-08,
+      "grad_norm": 0.65234375,
+      "learning_rate": 0.0001054165549673259,
+      "loss": 3.0817,
+      "step": 49190
+    },
+    {
+      "epoch": 1e-07,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.00010530360542277661,
+      "loss": 3.1199,
+      "step": 49200
+    },
+    {
+      "epoch": 1.05e-07,
+      "grad_norm": 0.62890625,
+      "learning_rate": 0.00010519070779932005,
+      "loss": 3.1137,
+      "step": 49210
+    },
+    {
+      "epoch": 1.1e-07,
+      "grad_norm": 0.76171875,
+      "learning_rate": 0.00010507786219514923,
+      "loss": 3.0613,
+      "step": 49220
+    },
+    {
+      "epoch": 1.15e-07,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.00010496506870841195,
+      "loss": 3.1172,
+      "step": 49230
+    },
+    {
+      "epoch": 1.2e-07,
+      "grad_norm": 0.62890625,
+      "learning_rate": 0.00010485232743721069,
+      "loss": 3.0776,
+      "step": 49240
+    },
+    {
+      "epoch": 1.25e-07,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.0001047396384796023,
+      "loss": 3.1018,
+      "step": 49250
+    },
+    {
+      "epoch": 1.3e-07,
+      "grad_norm": 0.6328125,
+      "learning_rate": 0.00010462700193359841,
+      "loss": 3.0794,
+      "step": 49260
+    },
+    {
+      "epoch": 1.35e-07,
+      "grad_norm": 0.578125,
+      "learning_rate": 0.00010451441789716487,
+      "loss": 3.1154,
+      "step": 49270
+    },
+    {
+      "epoch": 1.4e-07,
+      "grad_norm": 0.64453125,
+      "learning_rate": 0.00010440188646822201,
+      "loss": 3.1138,
+      "step": 49280
+    },
+    {
+      "epoch": 1.45e-07,
+      "grad_norm": 0.6015625,
+      "learning_rate": 0.00010428940774464427,
+      "loss": 3.1024,
+      "step": 49290
+    },
+    {
+      "epoch": 1.5e-07,
+      "grad_norm": 0.66796875,
+      "learning_rate": 0.00010417698182426035,
+      "loss": 3.0942,
+      "step": 49300
+    },
+    {
+      "epoch": 1.55e-07,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.00010406460880485295,
+      "loss": 3.0977,
+      "step": 49310
+    },
+    {
+      "epoch": 1.6e-07,
+      "grad_norm": 0.60546875,
+      "learning_rate": 0.00010395228878415881,
+      "loss": 3.0713,
+      "step": 49320
+    },
+    {
+      "epoch": 1.65e-07,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.00010384002185986863,
+      "loss": 3.1061,
+      "step": 49330
+    },
+    {
+      "epoch": 1.7e-07,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.00010372780812962673,
+      "loss": 3.0984,
+      "step": 49340
+    },
+    {
+      "epoch": 1.75e-07,
+      "grad_norm": 0.640625,
+      "learning_rate": 0.00010361564769103133,
+      "loss": 3.1019,
+      "step": 49350
+    },
+    {
+      "epoch": 1.8e-07,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.00010350354064163419,
+      "loss": 3.0661,
+      "step": 49360
+    },
+    {
+      "epoch": 1.85e-07,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.00010339148707894089,
+      "loss": 3.1016,
+      "step": 49370
+    },
+    {
+      "epoch": 1.9e-07,
+      "grad_norm": 0.81640625,
+      "learning_rate": 0.00010327948710041024,
+      "loss": 3.0893,
+      "step": 49380
+    },
+    {
+      "epoch": 1.95e-07,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.00010316754080345442,
+      "loss": 3.1009,
+      "step": 49390
+    },
+    {
+      "epoch": 2e-07,
+      "grad_norm": 0.6328125,
+      "learning_rate": 0.00010305564828543907,
+      "loss": 3.1064,
+      "step": 49400
+    },
+    {
+      "epoch": 2.05e-07,
+      "grad_norm": 0.5859375,
+      "learning_rate": 0.00010294380964368298,
+      "loss": 3.065,
+      "step": 49410
+    },
+    {
+      "epoch": 2.1e-07,
+      "grad_norm": 0.6171875,
+      "learning_rate": 0.00010283202497545811,
+      "loss": 3.0552,
+      "step": 49420
+    },
+    {
+      "epoch": 2.15e-07,
+      "grad_norm": 0.671875,
+      "learning_rate": 0.00010272029437798943,
+      "loss": 3.0983,
+      "step": 49430
+    },
+    {
+      "epoch": 2.2e-07,
+      "grad_norm": 0.8671875,
+      "learning_rate": 0.00010260861794845497,
+      "loss": 3.1077,
+      "step": 49440
+    },
+    {
+      "epoch": 2.25e-07,
+      "grad_norm": 0.62890625,
+      "learning_rate": 0.00010249699578398557,
+      "loss": 3.0824,
+      "step": 49450
+    },
+    {
+      "epoch": 2.3e-07,
+      "grad_norm": 0.6171875,
+      "learning_rate": 0.00010238542798166487,
+      "loss": 3.0985,
+      "step": 49460
+    },
+    {
+      "epoch": 2.35e-07,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.0001022739146385293,
+      "loss": 3.1138,
+      "step": 49470
+    },
+    {
+      "epoch": 2.4e-07,
+      "grad_norm": 0.87890625,
+      "learning_rate": 0.00010216245585156784,
+      "loss": 3.1316,
+      "step": 49480
+    },
+    {
+      "epoch": 2.45e-07,
+      "grad_norm": 0.63671875,
+      "learning_rate": 0.00010205105171772204,
+      "loss": 3.0916,
+      "step": 49490
+    },
+    {
+      "epoch": 2.5e-07,
+      "grad_norm": 0.703125,
+      "learning_rate": 0.00010193970233388597,
+      "loss": 3.1142,
+      "step": 49500
+    },
+    {
+      "epoch": 2.5e-07,
+      "eval_loss": 3.164301872253418,
+      "eval_runtime": 0.2006,
+      "eval_samples_per_second": 159.558,
+      "eval_steps_per_second": 4.986,
+      "step": 49500
+    },
+    {
+      "epoch": 2.55e-07,
+      "grad_norm": 0.7578125,
+      "learning_rate": 0.00010182840779690595,
+      "loss": 3.0552,
+      "step": 49510
+    },
+    {
+      "epoch": 2.6e-07,
+      "grad_norm": 0.6015625,
+      "learning_rate": 0.0001017171682035809,
+      "loss": 3.0932,
+      "step": 49520
+    },
+    {
+      "epoch": 2.65e-07,
+      "grad_norm": 0.63671875,
+      "learning_rate": 0.00010160598365066155,
+      "loss": 3.1036,
+      "step": 49530
+    },
+    {
+      "epoch": 2.7e-07,
+      "grad_norm": 0.6796875,
+      "learning_rate": 0.000101494854234851,
+      "loss": 3.0786,
+      "step": 49540
+    },
+    {
+      "epoch": 2.75e-07,
+      "grad_norm": 0.62890625,
+      "learning_rate": 0.00010138378005280439,
+      "loss": 3.0714,
+      "step": 49550
+    },
+    {
+      "epoch": 2.8e-07,
+      "grad_norm": 0.68359375,
+      "learning_rate": 0.00010127276120112878,
+      "loss": 3.0573,
+      "step": 49560
+    },
+    {
+      "epoch": 2.85e-07,
+      "grad_norm": 0.64453125,
+      "learning_rate": 0.00010116179777638298,
+      "loss": 3.0492,
+      "step": 49570
+    },
+    {
+      "epoch": 2.9e-07,
+      "grad_norm": 0.765625,
+      "learning_rate": 0.00010105088987507782,
+      "loss": 3.1214,
+      "step": 49580
+    },
+    {
+      "epoch": 2.95e-07,
+      "grad_norm": 0.67578125,
+      "learning_rate": 0.00010094003759367574,
+      "loss": 3.0896,
+      "step": 49590
+    },
+    {
+      "epoch": 3e-07,
+      "grad_norm": 0.59375,
+      "learning_rate": 0.00010082924102859073,
+      "loss": 3.1125,
+      "step": 49600
+    },
+    {
+      "epoch": 3.05e-07,
+      "grad_norm": 0.578125,
+      "learning_rate": 0.00010071850027618852,
+      "loss": 3.0446,
+      "step": 49610
+    },
+    {
+      "epoch": 3.1e-07,
+      "grad_norm": 0.6328125,
+      "learning_rate": 0.000100607815432786,
+      "loss": 3.1128,
+      "step": 49620
+    },
+    {
+      "epoch": 3.15e-07,
+      "grad_norm": 0.7265625,
+      "learning_rate": 0.00010049718659465167,
+      "loss": 3.1234,
+      "step": 49630
+    },
+    {
+      "epoch": 3.2e-07,
+      "grad_norm": 0.62890625,
+      "learning_rate": 0.00010038661385800525,
+      "loss": 3.0634,
+      "step": 49640
+    },
+    {
+      "epoch": 3.25e-07,
+      "grad_norm": 0.62890625,
+      "learning_rate": 0.00010027609731901759,
+      "loss": 3.0568,
+      "step": 49650
+    },
+    {
+      "epoch": 3.3e-07,
+      "grad_norm": 0.6328125,
+      "learning_rate": 0.00010016563707381093,
+      "loss": 3.0853,
+      "step": 49660
+    },
+    {
+      "epoch": 3.35e-07,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.00010005523321845814,
+      "loss": 3.1028,
+      "step": 49670
+    },
+    {
+      "epoch": 3.4e-07,
+      "grad_norm": 0.59375,
+      "learning_rate": 9.994488584898336e-05,
+      "loss": 3.0982,
+      "step": 49680
+    },
+    {
+      "epoch": 3.45e-07,
+      "grad_norm": 0.87890625,
+      "learning_rate": 9.98345950613615e-05,
+      "loss": 3.1122,
+      "step": 49690
+    },
+    {
+      "epoch": 3.5e-07,
+      "grad_norm": 0.64453125,
+      "learning_rate": 9.972436095151826e-05,
+      "loss": 3.1003,
+      "step": 49700
+    },
+    {
+      "epoch": 3.55e-07,
+      "grad_norm": 0.6328125,
+      "learning_rate": 9.961418361532994e-05,
+      "loss": 3.1298,
+      "step": 49710
+    },
+    {
+      "epoch": 3.6e-07,
+      "grad_norm": 0.63671875,
+      "learning_rate": 9.950406314862365e-05,
+      "loss": 3.0694,
+      "step": 49720
+    },
+    {
+      "epoch": 3.65e-07,
+      "grad_norm": 0.6875,
+      "learning_rate": 9.939399964717698e-05,
+      "loss": 3.08,
+      "step": 49730
+    },
+    {
+      "epoch": 3.7e-07,
+      "grad_norm": 0.71875,
+      "learning_rate": 9.928399320671787e-05,
+      "loss": 3.0896,
+      "step": 49740
+    },
+    {
+      "epoch": 3.75e-07,
+      "grad_norm": 0.62109375,
+      "learning_rate": 9.91740439229248e-05,
+      "loss": 3.0944,
+      "step": 49750
+    },
+    {
+      "epoch": 3.8e-07,
+      "grad_norm": 0.59375,
+      "learning_rate": 9.906415189142632e-05,
+      "loss": 3.0935,
+      "step": 49760
+    },
+    {
+      "epoch": 3.85e-07,
+      "grad_norm": 0.63671875,
+      "learning_rate": 9.895431720780137e-05,
+      "loss": 3.0593,
+      "step": 49770
+    },
+    {
+      "epoch": 3.9e-07,
+      "grad_norm": 0.61328125,
+      "learning_rate": 9.884453996757894e-05,
+      "loss": 3.0615,
+      "step": 49780
+    },
+    {
+      "epoch": 3.95e-07,
+      "grad_norm": 0.66796875,
+      "learning_rate": 9.873482026623804e-05,
+      "loss": 3.066,
+      "step": 49790
+    },
+    {
+      "epoch": 4e-07,
+      "grad_norm": 0.65625,
+      "learning_rate": 9.862515819920776e-05,
+      "loss": 3.0987,
+      "step": 49800
+    },
+    {
+      "epoch": 4.05e-07,
+      "grad_norm": 0.67578125,
+      "learning_rate": 9.851555386186687e-05,
+      "loss": 3.0693,
+      "step": 49810
+    },
+    {
+      "epoch": 4.1e-07,
+      "grad_norm": 0.66015625,
+      "learning_rate": 9.840600734954405e-05,
+      "loss": 3.1125,
+      "step": 49820
+    },
+    {
+      "epoch": 4.15e-07,
+      "grad_norm": 0.609375,
+      "learning_rate": 9.829651875751764e-05,
+      "loss": 3.0705,
+      "step": 49830
+    },
+    {
+      "epoch": 4.2e-07,
+      "grad_norm": 0.67578125,
+      "learning_rate": 9.818708818101574e-05,
+      "loss": 3.0935,
+      "step": 49840
+    },
+    {
+      "epoch": 4.25e-07,
+      "grad_norm": 0.6328125,
+      "learning_rate": 9.807771571521565e-05,
+      "loss": 3.0583,
+      "step": 49850
+    },
+    {
+      "epoch": 4.3e-07,
+      "grad_norm": 0.62109375,
+      "learning_rate": 9.796840145524456e-05,
+      "loss": 3.0835,
+      "step": 49860
+    },
+    {
+      "epoch": 4.35e-07,
+      "grad_norm": 0.60546875,
+      "learning_rate": 9.785914549617873e-05,
+      "loss": 3.0431,
+      "step": 49870
+    },
+    {
+      "epoch": 4.4e-07,
+      "grad_norm": 0.65234375,
+      "learning_rate": 9.774994793304384e-05,
+      "loss": 3.0697,
+      "step": 49880
+    },
+    {
+      "epoch": 4.45e-07,
+      "grad_norm": 0.7265625,
+      "learning_rate": 9.764080886081484e-05,
+      "loss": 3.0997,
+      "step": 49890
+    },
+    {
+      "epoch": 4.5e-07,
+      "grad_norm": 0.7578125,
+      "learning_rate": 9.753172837441554e-05,
+      "loss": 3.0633,
+      "step": 49900
+    },
+    {
+      "epoch": 4.55e-07,
+      "grad_norm": 0.6484375,
+      "learning_rate": 9.742270656871906e-05,
+      "loss": 3.0752,
+      "step": 49910
+    },
+    {
+      "epoch": 4.6e-07,
+      "grad_norm": 0.69140625,
+      "learning_rate": 9.731374353854742e-05,
+      "loss": 3.0803,
+      "step": 49920
+    },
+    {
+      "epoch": 4.65e-07,
+      "grad_norm": 0.5859375,
+      "learning_rate": 9.720483937867147e-05,
+      "loss": 3.073,
+      "step": 49930
+    },
+    {
+      "epoch": 4.7e-07,
+      "grad_norm": 0.61328125,
+      "learning_rate": 9.709599418381087e-05,
+      "loss": 3.0693,
+      "step": 49940
+    },
+    {
+      "epoch": 4.75e-07,
+      "grad_norm": 0.640625,
+      "learning_rate": 9.698720804863402e-05,
+      "loss": 3.124,
+      "step": 49950
+    },
+    {
+      "epoch": 4.8e-07,
+      "grad_norm": 0.60546875,
+      "learning_rate": 9.687848106775797e-05,
+      "loss": 3.09,
+      "step": 49960
+    },
+    {
+      "epoch": 4.85e-07,
+      "grad_norm": 0.68359375,
+      "learning_rate": 9.676981333574823e-05,
+      "loss": 3.0619,
+      "step": 49970
+    },
+    {
+      "epoch": 4.9e-07,
+      "grad_norm": 0.625,
+      "learning_rate": 9.666120494711896e-05,
+      "loss": 3.0917,
+      "step": 49980
+    },
+    {
+      "epoch": 4.95e-07,
+      "grad_norm": 0.64453125,
+      "learning_rate": 9.655265599633246e-05,
+      "loss": 3.0923,
+      "step": 49990
+    },
+    {
+      "epoch": 5e-07,
+      "grad_norm": 0.61328125,
+      "learning_rate": 9.644416657779942e-05,
+      "loss": 3.1021,
+      "step": 50000
+    },
+    {
+      "epoch": 5e-07,
+      "eval_loss": 3.1472666263580322,
+      "eval_runtime": 0.0603,
+      "eval_samples_per_second": 530.541,
+      "eval_steps_per_second": 16.579,
+      "step": 50000
     }
   ],
   "logging_steps": 10,
-  "max_steps": 76294,
+  "max_steps": 2000000000,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 9223372036854775807,
   "save_steps": 500,
@@ -35209,7 +35925,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 8.81206191194112e+17,
+  "total_flos": 8.991899910144e+17,
   "train_batch_size": 128,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d543c88f7fef36ef68b6d45e6baf53067caa045005adb7fff867ae9be65dfe2
-size 5905
+oid sha256:a90e75c7346f69e1be727fa5f245a2e5173b1ff2d4b0864e74ef7e71c0f330f5
+size 5969