tzar z commited on
Commit
07fd794
·
verified ·
1 Parent(s): 46d4f39

Sovereign Genesis - checkpoint-5500 - Weights Only Search

Browse files
Files changed (4) hide show
  1. optimizer.pt +1 -1
  2. rng_state.pth +1 -1
  3. scheduler.pt +1 -1
  4. trainer_state.json +503 -3
optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6d0fd3f5da4145bf8440cb1b5e631e654e5c728b3173a4059a32e283ea65c405
3
  size 156887749
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd4450a5c9ccb7099015cecad40e69b2b1bcb6d9b9114960bf8aebdc8e44d551
3
  size 156887749
rng_state.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:767245398d4ac7b0c9fa3efa7bf8a9f296c1c6beaf69f0ea4dd88b5290cc410f
3
  size 14645
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5fdd0e030d6c42ee39eee0431577f1d5f0914c0c71064c24cda015c0e8f9f5cc
3
  size 14645
scheduler.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:37f13459914a2e3c5a2f6374e452b23d937bfcd8e3b09a21cea063e99f85843b
3
  size 1465
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:521f6e223c5d47e087261619f0a3f38092b0e8ed1f73265aaed7e468af493598
3
  size 1465
trainer_state.json CHANGED
@@ -2,9 +2,9 @@
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
- "epoch": 2.3618327822390173,
6
  "eval_steps": 500,
7
- "global_step": 5000,
8
  "is_hyper_param_search": false,
9
  "is_local_process_zero": true,
10
  "is_world_process_zero": true,
@@ -5008,6 +5008,506 @@
5008
  "mean_token_accuracy": 0.9155340820550919,
5009
  "num_tokens": 69777845.0,
5010
  "step": 5000
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5011
  }
5012
  ],
5013
  "logging_steps": 10,
@@ -5027,7 +5527,7 @@
5027
  "attributes": {}
5028
  }
5029
  },
5030
- "total_flos": 1.0520056670651228e+18,
5031
  "train_batch_size": 4,
5032
  "trial_name": null,
5033
  "trial_params": null
 
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
+ "epoch": 2.5980160604629194,
6
  "eval_steps": 500,
7
+ "global_step": 5500,
8
  "is_hyper_param_search": false,
9
  "is_local_process_zero": true,
10
  "is_world_process_zero": true,
 
5008
  "mean_token_accuracy": 0.9155340820550919,
5009
  "num_tokens": 69777845.0,
5010
  "step": 5000
5011
+ },
5012
+ {
5013
+ "entropy": 0.3717569367960095,
5014
+ "epoch": 2.3665564478034957,
5015
+ "grad_norm": 0.06603619456291199,
5016
+ "learning_rate": 0.00017400514141928722,
5017
+ "loss": 0.32091450691223145,
5018
+ "mean_token_accuracy": 0.9150391787290573,
5019
+ "num_tokens": 69915165.0,
5020
+ "step": 5010
5021
+ },
5022
+ {
5023
+ "entropy": 0.3680802728049457,
5024
+ "epoch": 2.3712801133679737,
5025
+ "grad_norm": 0.09668844193220139,
5026
+ "learning_rate": 0.00017390501795961017,
5027
+ "loss": 0.34298737049102784,
5028
+ "mean_token_accuracy": 0.9123553864657878,
5029
+ "num_tokens": 70056805.0,
5030
+ "step": 5020
5031
+ },
5032
+ {
5033
+ "entropy": 0.35295952931046487,
5034
+ "epoch": 2.3760037789324517,
5035
+ "grad_norm": 0.06308913230895996,
5036
+ "learning_rate": 0.00017380473097432814,
5037
+ "loss": 0.3172333240509033,
5038
+ "mean_token_accuracy": 0.9175053678452969,
5039
+ "num_tokens": 70198087.0,
5040
+ "step": 5030
5041
+ },
5042
+ {
5043
+ "entropy": 0.36961612179875375,
5044
+ "epoch": 2.3807274444969297,
5045
+ "grad_norm": 0.0788240060210228,
5046
+ "learning_rate": 0.0001737042806853407,
5047
+ "loss": 0.3349175930023193,
5048
+ "mean_token_accuracy": 0.9132102608680726,
5049
+ "num_tokens": 70335404.0,
5050
+ "step": 5040
5051
+ },
5052
+ {
5053
+ "entropy": 0.3711566047742963,
5054
+ "epoch": 2.3854511100614078,
5055
+ "grad_norm": 0.07890364527702332,
5056
+ "learning_rate": 0.00017360366731490877,
5057
+ "loss": 0.3386693000793457,
5058
+ "mean_token_accuracy": 0.9134459957480431,
5059
+ "num_tokens": 70477392.0,
5060
+ "step": 5050
5061
+ },
5062
+ {
5063
+ "entropy": 0.35097449999302627,
5064
+ "epoch": 2.3901747756258858,
5065
+ "grad_norm": 0.06498616188764572,
5066
+ "learning_rate": 0.00017350289108565404,
5067
+ "loss": 0.3155566930770874,
5068
+ "mean_token_accuracy": 0.9186516053974628,
5069
+ "num_tokens": 70613392.0,
5070
+ "step": 5060
5071
+ },
5072
+ {
5073
+ "entropy": 0.3575643849559128,
5074
+ "epoch": 2.3948984411903638,
5075
+ "grad_norm": 0.07433422654867172,
5076
+ "learning_rate": 0.00017340195222055854,
5077
+ "loss": 0.3291230440139771,
5078
+ "mean_token_accuracy": 0.9150498561561108,
5079
+ "num_tokens": 70761404.0,
5080
+ "step": 5070
5081
+ },
5082
+ {
5083
+ "entropy": 0.3681374185718596,
5084
+ "epoch": 2.3996221067548418,
5085
+ "grad_norm": 0.0771808996796608,
5086
+ "learning_rate": 0.00017330085094296422,
5087
+ "loss": 0.3228792190551758,
5088
+ "mean_token_accuracy": 0.9155564263463021,
5089
+ "num_tokens": 70902964.0,
5090
+ "step": 5080
5091
+ },
5092
+ {
5093
+ "entropy": 0.3403599739074707,
5094
+ "epoch": 2.40434577231932,
5095
+ "grad_norm": 0.061653293669223785,
5096
+ "learning_rate": 0.0001731995874765724,
5097
+ "loss": 0.3083616256713867,
5098
+ "mean_token_accuracy": 0.918988062441349,
5099
+ "num_tokens": 71042549.0,
5100
+ "step": 5090
5101
+ },
5102
+ {
5103
+ "entropy": 0.36283190539106724,
5104
+ "epoch": 2.409069437883798,
5105
+ "grad_norm": 0.08139196038246155,
5106
+ "learning_rate": 0.00017309816204544317,
5107
+ "loss": 0.3336886644363403,
5108
+ "mean_token_accuracy": 0.9147178888320923,
5109
+ "num_tokens": 71180384.0,
5110
+ "step": 5100
5111
+ },
5112
+ {
5113
+ "entropy": 0.34851250536739825,
5114
+ "epoch": 2.413793103448276,
5115
+ "grad_norm": 0.06935746222734451,
5116
+ "learning_rate": 0.0001729965748739951,
5117
+ "loss": 0.317404580116272,
5118
+ "mean_token_accuracy": 0.9191467575728893,
5119
+ "num_tokens": 71318781.0,
5120
+ "step": 5110
5121
+ },
5122
+ {
5123
+ "entropy": 0.35370422853156924,
5124
+ "epoch": 2.418516769012754,
5125
+ "grad_norm": 0.0793750062584877,
5126
+ "learning_rate": 0.00017289482618700462,
5127
+ "loss": 0.30998620986938474,
5128
+ "mean_token_accuracy": 0.917963957041502,
5129
+ "num_tokens": 71451944.0,
5130
+ "step": 5120
5131
+ },
5132
+ {
5133
+ "entropy": 0.3418350827880204,
5134
+ "epoch": 2.423240434577232,
5135
+ "grad_norm": 0.07875291258096695,
5136
+ "learning_rate": 0.00017279291620960543,
5137
+ "loss": 0.3185269832611084,
5138
+ "mean_token_accuracy": 0.9166646711528301,
5139
+ "num_tokens": 71593532.0,
5140
+ "step": 5130
5141
+ },
5142
+ {
5143
+ "entropy": 0.36766334623098373,
5144
+ "epoch": 2.42796410014171,
5145
+ "grad_norm": 0.0857739970088005,
5146
+ "learning_rate": 0.0001726908451672882,
5147
+ "loss": 0.32974236011505126,
5148
+ "mean_token_accuracy": 0.9142980873584747,
5149
+ "num_tokens": 71730615.0,
5150
+ "step": 5140
5151
+ },
5152
+ {
5153
+ "entropy": 0.3898682286031544,
5154
+ "epoch": 2.432687765706188,
5155
+ "grad_norm": 0.08818437159061432,
5156
+ "learning_rate": 0.0001725886132859,
5157
+ "loss": 0.36099967956542967,
5158
+ "mean_token_accuracy": 0.9105031929910183,
5159
+ "num_tokens": 71871723.0,
5160
+ "step": 5150
5161
+ },
5162
+ {
5163
+ "entropy": 0.3895622887648642,
5164
+ "epoch": 2.4374114312706663,
5165
+ "grad_norm": 0.0812084749341011,
5166
+ "learning_rate": 0.0001724862207916437,
5167
+ "loss": 0.362603497505188,
5168
+ "mean_token_accuracy": 0.9069844730198383,
5169
+ "num_tokens": 72012227.0,
5170
+ "step": 5160
5171
+ },
5172
+ {
5173
+ "entropy": 0.3616925870999694,
5174
+ "epoch": 2.442135096835144,
5175
+ "grad_norm": 0.06379372626543045,
5176
+ "learning_rate": 0.0001723836679110776,
5177
+ "loss": 0.32681663036346437,
5178
+ "mean_token_accuracy": 0.9173739403486252,
5179
+ "num_tokens": 72155955.0,
5180
+ "step": 5170
5181
+ },
5182
+ {
5183
+ "entropy": 0.34351073727011683,
5184
+ "epoch": 2.4468587623996223,
5185
+ "grad_norm": 0.0815335288643837,
5186
+ "learning_rate": 0.00017228095487111487,
5187
+ "loss": 0.31263697147369385,
5188
+ "mean_token_accuracy": 0.9207109160721302,
5189
+ "num_tokens": 72296105.0,
5190
+ "step": 5180
5191
+ },
5192
+ {
5193
+ "entropy": 0.367705793119967,
5194
+ "epoch": 2.4515824279641003,
5195
+ "grad_norm": 0.07983652502298355,
5196
+ "learning_rate": 0.00017217808189902306,
5197
+ "loss": 0.34312000274658205,
5198
+ "mean_token_accuracy": 0.9133535690605641,
5199
+ "num_tokens": 72427639.0,
5200
+ "step": 5190
5201
+ },
5202
+ {
5203
+ "entropy": 0.3358869240619242,
5204
+ "epoch": 2.4563060935285783,
5205
+ "grad_norm": 0.07612093538045883,
5206
+ "learning_rate": 0.0001720750492224236,
5207
+ "loss": 0.28635563850402834,
5208
+ "mean_token_accuracy": 0.9230332836508751,
5209
+ "num_tokens": 72576973.0,
5210
+ "step": 5200
5211
+ },
5212
+ {
5213
+ "entropy": 0.3458953768014908,
5214
+ "epoch": 2.4610297590930563,
5215
+ "grad_norm": 0.07325061410665512,
5216
+ "learning_rate": 0.00017197185706929128,
5217
+ "loss": 0.30686020851135254,
5218
+ "mean_token_accuracy": 0.9192182451486588,
5219
+ "num_tokens": 72720494.0,
5220
+ "step": 5210
5221
+ },
5222
+ {
5223
+ "entropy": 0.3425209357403219,
5224
+ "epoch": 2.4657534246575343,
5225
+ "grad_norm": 0.06628164649009705,
5226
+ "learning_rate": 0.00017186850566795373,
5227
+ "loss": 0.3105634689331055,
5228
+ "mean_token_accuracy": 0.9185604088008403,
5229
+ "num_tokens": 72862211.0,
5230
+ "step": 5220
5231
+ },
5232
+ {
5233
+ "entropy": 0.3979892102070153,
5234
+ "epoch": 2.4704770902220123,
5235
+ "grad_norm": 0.07804671674966812,
5236
+ "learning_rate": 0.00017176499524709096,
5237
+ "loss": 0.3673126220703125,
5238
+ "mean_token_accuracy": 0.9062325954437256,
5239
+ "num_tokens": 73005958.0,
5240
+ "step": 5230
5241
+ },
5242
+ {
5243
+ "entropy": 0.3278886295855045,
5244
+ "epoch": 2.4752007557864903,
5245
+ "grad_norm": 0.07158853858709335,
5246
+ "learning_rate": 0.00017166132603573485,
5247
+ "loss": 0.28293752670288086,
5248
+ "mean_token_accuracy": 0.9247597858309746,
5249
+ "num_tokens": 73150446.0,
5250
+ "step": 5240
5251
+ },
5252
+ {
5253
+ "entropy": 0.33812597766518593,
5254
+ "epoch": 2.4799244213509684,
5255
+ "grad_norm": 0.06800093501806259,
5256
+ "learning_rate": 0.00017155749826326864,
5257
+ "loss": 0.30004396438598635,
5258
+ "mean_token_accuracy": 0.9229036450386048,
5259
+ "num_tokens": 73290485.0,
5260
+ "step": 5250
5261
+ },
5262
+ {
5263
+ "entropy": 0.32100099362432954,
5264
+ "epoch": 2.4846480869154464,
5265
+ "grad_norm": 0.06439868360757828,
5266
+ "learning_rate": 0.00017145351215942632,
5267
+ "loss": 0.2929414987564087,
5268
+ "mean_token_accuracy": 0.9239821515977382,
5269
+ "num_tokens": 73431354.0,
5270
+ "step": 5260
5271
+ },
5272
+ {
5273
+ "entropy": 0.35717023378238083,
5274
+ "epoch": 2.4893717524799244,
5275
+ "grad_norm": 0.06693439930677414,
5276
+ "learning_rate": 0.0001713493679542923,
5277
+ "loss": 0.32613224983215333,
5278
+ "mean_token_accuracy": 0.9171266071498394,
5279
+ "num_tokens": 73570275.0,
5280
+ "step": 5270
5281
+ },
5282
+ {
5283
+ "entropy": 0.3441743317991495,
5284
+ "epoch": 2.4940954180444024,
5285
+ "grad_norm": 0.0831870511174202,
5286
+ "learning_rate": 0.00017124506587830083,
5287
+ "loss": 0.31802730560302733,
5288
+ "mean_token_accuracy": 0.9191960535943509,
5289
+ "num_tokens": 73701904.0,
5290
+ "step": 5280
5291
+ },
5292
+ {
5293
+ "entropy": 0.3742942173033953,
5294
+ "epoch": 2.4988190836088804,
5295
+ "grad_norm": 0.08274650573730469,
5296
+ "learning_rate": 0.00017114060616223536,
5297
+ "loss": 0.3411454916000366,
5298
+ "mean_token_accuracy": 0.9124328427016735,
5299
+ "num_tokens": 73836277.0,
5300
+ "step": 5290
5301
+ },
5302
+ {
5303
+ "entropy": 0.34059149408712985,
5304
+ "epoch": 2.5035427491733584,
5305
+ "grad_norm": 0.0695531889796257,
5306
+ "learning_rate": 0.00017103598903722827,
5307
+ "loss": 0.3107386350631714,
5308
+ "mean_token_accuracy": 0.9196738213300705,
5309
+ "num_tokens": 73977700.0,
5310
+ "step": 5300
5311
+ },
5312
+ {
5313
+ "entropy": 0.3607602405361831,
5314
+ "epoch": 2.5082664147378364,
5315
+ "grad_norm": 0.08559349924325943,
5316
+ "learning_rate": 0.00017093121473476015,
5317
+ "loss": 0.3217759370803833,
5318
+ "mean_token_accuracy": 0.9155218534171581,
5319
+ "num_tokens": 74120486.0,
5320
+ "step": 5310
5321
+ },
5322
+ {
5323
+ "entropy": 0.3522417671047151,
5324
+ "epoch": 2.5129900803023144,
5325
+ "grad_norm": 0.06628310680389404,
5326
+ "learning_rate": 0.00017082628348665934,
5327
+ "loss": 0.3156892776489258,
5328
+ "mean_token_accuracy": 0.9205105111002923,
5329
+ "num_tokens": 74257237.0,
5330
+ "step": 5320
5331
+ },
5332
+ {
5333
+ "entropy": 0.37255304008722306,
5334
+ "epoch": 2.517713745866793,
5335
+ "grad_norm": 0.07685009390115738,
5336
+ "learning_rate": 0.00017072119552510157,
5337
+ "loss": 0.3570106029510498,
5338
+ "mean_token_accuracy": 0.9101258270442486,
5339
+ "num_tokens": 74394555.0,
5340
+ "step": 5330
5341
+ },
5342
+ {
5343
+ "entropy": 0.3417865173891187,
5344
+ "epoch": 2.5224374114312704,
5345
+ "grad_norm": 0.06182515248656273,
5346
+ "learning_rate": 0.0001706159510826092,
5347
+ "loss": 0.30006103515625,
5348
+ "mean_token_accuracy": 0.9223997719585896,
5349
+ "num_tokens": 74535607.0,
5350
+ "step": 5340
5351
+ },
5352
+ {
5353
+ "entropy": 0.3596333073452115,
5354
+ "epoch": 2.527161076995749,
5355
+ "grad_norm": 0.08518214523792267,
5356
+ "learning_rate": 0.00017051055039205086,
5357
+ "loss": 0.3307522773742676,
5358
+ "mean_token_accuracy": 0.9149652808904648,
5359
+ "num_tokens": 74674135.0,
5360
+ "step": 5350
5361
+ },
5362
+ {
5363
+ "entropy": 0.3565245810896158,
5364
+ "epoch": 2.531884742560227,
5365
+ "grad_norm": 0.07785555720329285,
5366
+ "learning_rate": 0.00017040499368664095,
5367
+ "loss": 0.3165716648101807,
5368
+ "mean_token_accuracy": 0.916690094769001,
5369
+ "num_tokens": 74807533.0,
5370
+ "step": 5360
5371
+ },
5372
+ {
5373
+ "entropy": 0.37183555755764247,
5374
+ "epoch": 2.536608408124705,
5375
+ "grad_norm": 0.0807003527879715,
5376
+ "learning_rate": 0.00017029928119993904,
5377
+ "loss": 0.34868154525756834,
5378
+ "mean_token_accuracy": 0.9119903385639191,
5379
+ "num_tokens": 74948566.0,
5380
+ "step": 5370
5381
+ },
5382
+ {
5383
+ "entropy": 0.3737936247140169,
5384
+ "epoch": 2.541332073689183,
5385
+ "grad_norm": 0.057071540504693985,
5386
+ "learning_rate": 0.00017019341316584934,
5387
+ "loss": 0.3513514995574951,
5388
+ "mean_token_accuracy": 0.9121352203190327,
5389
+ "num_tokens": 75087320.0,
5390
+ "step": 5380
5391
+ },
5392
+ {
5393
+ "entropy": 0.3535418752580881,
5394
+ "epoch": 2.546055739253661,
5395
+ "grad_norm": 0.07253091782331467,
5396
+ "learning_rate": 0.0001700873898186203,
5397
+ "loss": 0.3125487804412842,
5398
+ "mean_token_accuracy": 0.919706829637289,
5399
+ "num_tokens": 75225167.0,
5400
+ "step": 5390
5401
+ },
5402
+ {
5403
+ "entropy": 0.3378803776577115,
5404
+ "epoch": 2.550779404818139,
5405
+ "grad_norm": 0.08137330412864685,
5406
+ "learning_rate": 0.00016998121139284405,
5407
+ "loss": 0.30165958404541016,
5408
+ "mean_token_accuracy": 0.9213343054056168,
5409
+ "num_tokens": 75364306.0,
5410
+ "step": 5400
5411
+ },
5412
+ {
5413
+ "entropy": 0.3566731529310346,
5414
+ "epoch": 2.555503070382617,
5415
+ "grad_norm": 0.07202183455228806,
5416
+ "learning_rate": 0.00016987487812345574,
5417
+ "loss": 0.3214185476303101,
5418
+ "mean_token_accuracy": 0.9166874349117279,
5419
+ "num_tokens": 75504073.0,
5420
+ "step": 5410
5421
+ },
5422
+ {
5423
+ "entropy": 0.35006280736997725,
5424
+ "epoch": 2.560226735947095,
5425
+ "grad_norm": 0.07671020925045013,
5426
+ "learning_rate": 0.0001697683902457332,
5427
+ "loss": 0.32263405323028566,
5428
+ "mean_token_accuracy": 0.9176624983549118,
5429
+ "num_tokens": 75651688.0,
5430
+ "step": 5420
5431
+ },
5432
+ {
5433
+ "entropy": 0.3596495855599642,
5434
+ "epoch": 2.564950401511573,
5435
+ "grad_norm": 0.057611897587776184,
5436
+ "learning_rate": 0.0001696617479952964,
5437
+ "loss": 0.31757681369781493,
5438
+ "mean_token_accuracy": 0.9163362935185433,
5439
+ "num_tokens": 75792454.0,
5440
+ "step": 5430
5441
+ },
5442
+ {
5443
+ "entropy": 0.3428363500162959,
5444
+ "epoch": 2.569674067076051,
5445
+ "grad_norm": 0.06537020951509476,
5446
+ "learning_rate": 0.00016955495160810683,
5447
+ "loss": 0.3163311004638672,
5448
+ "mean_token_accuracy": 0.9194522321224212,
5449
+ "num_tokens": 75928302.0,
5450
+ "step": 5440
5451
+ },
5452
+ {
5453
+ "entropy": 0.33568228837102654,
5454
+ "epoch": 2.574397732640529,
5455
+ "grad_norm": 0.08116862922906876,
5456
+ "learning_rate": 0.000169448001320467,
5457
+ "loss": 0.2961144924163818,
5458
+ "mean_token_accuracy": 0.9231125563383102,
5459
+ "num_tokens": 76071503.0,
5460
+ "step": 5450
5461
+ },
5462
+ {
5463
+ "entropy": 0.334136747661978,
5464
+ "epoch": 2.579121398205007,
5465
+ "grad_norm": 0.0786835104227066,
5466
+ "learning_rate": 0.00016934089736902002,
5467
+ "loss": 0.3097526550292969,
5468
+ "mean_token_accuracy": 0.9204525671899318,
5469
+ "num_tokens": 76205030.0,
5470
+ "step": 5460
5471
+ },
5472
+ {
5473
+ "entropy": 0.3176261980086565,
5474
+ "epoch": 2.583845063769485,
5475
+ "grad_norm": 0.05822472646832466,
5476
+ "learning_rate": 0.0001692336399907489,
5477
+ "loss": 0.2668977975845337,
5478
+ "mean_token_accuracy": 0.9287205383181572,
5479
+ "num_tokens": 76346970.0,
5480
+ "step": 5470
5481
+ },
5482
+ {
5483
+ "entropy": 0.33487942814826965,
5484
+ "epoch": 2.5885687293339634,
5485
+ "grad_norm": 0.08300083875656128,
5486
+ "learning_rate": 0.00016912622942297627,
5487
+ "loss": 0.303263258934021,
5488
+ "mean_token_accuracy": 0.9212611213326454,
5489
+ "num_tokens": 76488488.0,
5490
+ "step": 5480
5491
+ },
5492
+ {
5493
+ "entropy": 0.33707099203020335,
5494
+ "epoch": 2.593292394898441,
5495
+ "grad_norm": 0.08029662817716599,
5496
+ "learning_rate": 0.00016901866590336362,
5497
+ "loss": 0.2941617965698242,
5498
+ "mean_token_accuracy": 0.9243578106164932,
5499
+ "num_tokens": 76628143.0,
5500
+ "step": 5490
5501
+ },
5502
+ {
5503
+ "entropy": 0.3697511538863182,
5504
+ "epoch": 2.5980160604629194,
5505
+ "grad_norm": 0.07310616225004196,
5506
+ "learning_rate": 0.00016891094966991083,
5507
+ "loss": 0.33811867237091064,
5508
+ "mean_token_accuracy": 0.9146531477570534,
5509
+ "num_tokens": 76770868.0,
5510
+ "step": 5500
5511
  }
5512
  ],
5513
  "logging_steps": 10,
 
5527
  "attributes": {}
5528
  }
5529
  },
5530
+ "total_flos": 1.157857056941396e+18,
5531
  "train_batch_size": 4,
5532
  "trial_name": null,
5533
  "trial_params": null