tzar z commited on
Commit
bf48b49
·
verified ·
1 Parent(s): de3e492

Sovereign Genesis - checkpoint-5000 - Weights Only Search

Browse files
Files changed (4) hide show
  1. optimizer.pt +1 -1
  2. rng_state.pth +1 -1
  3. scheduler.pt +1 -1
  4. trainer_state.json +503 -3
optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:858a1f73fd0e89967feb1974fccbe6c557a574a1e22e574faf8a969f2d80931a
3
  size 156887749
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d0fd3f5da4145bf8440cb1b5e631e654e5c728b3173a4059a32e283ea65c405
3
  size 156887749
rng_state.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:725d6511296fd506b58c9700b79cb85857c863e8247661661608ff0c2ecf8011
3
  size 14645
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:767245398d4ac7b0c9fa3efa7bf8a9f296c1c6beaf69f0ea4dd88b5290cc410f
3
  size 14645
scheduler.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5e85206bb202ec8428f1708e0c37a928ce020685f3cd8564bdc8ef834b0e4f3a
3
  size 1465
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:37f13459914a2e3c5a2f6374e452b23d937bfcd8e3b09a21cea063e99f85843b
3
  size 1465
trainer_state.json CHANGED
@@ -2,9 +2,9 @@
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
- "epoch": 2.1256495040151155,
6
  "eval_steps": 500,
7
- "global_step": 4500,
8
  "is_hyper_param_search": false,
9
  "is_local_process_zero": true,
10
  "is_world_process_zero": true,
@@ -4508,6 +4508,506 @@
4508
  "mean_token_accuracy": 0.9049822241067886,
4509
  "num_tokens": 62812870.0,
4510
  "step": 4500
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4511
  }
4512
  ],
4513
  "logging_steps": 10,
@@ -4527,7 +5027,7 @@
4527
  "attributes": {}
4528
  }
4529
  },
4530
- "total_flos": 9.466898051393004e+17,
4531
  "train_batch_size": 4,
4532
  "trial_name": null,
4533
  "trial_params": null
 
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
+ "epoch": 2.3618327822390173,
6
  "eval_steps": 500,
7
+ "global_step": 5000,
8
  "is_hyper_param_search": false,
9
  "is_local_process_zero": true,
10
  "is_world_process_zero": true,
 
4508
  "mean_token_accuracy": 0.9049822241067886,
4509
  "num_tokens": 62812870.0,
4510
  "step": 4500
4511
+ },
4512
+ {
4513
+ "entropy": 0.3644891042262316,
4514
+ "epoch": 2.1303731695795936,
4515
+ "grad_norm": 0.07077588140964508,
4516
+ "learning_rate": 0.00017879802261613068,
4517
+ "loss": 0.32347466945648196,
4518
+ "mean_token_accuracy": 0.9153170421719551,
4519
+ "num_tokens": 62950453.0,
4520
+ "step": 4510
4521
+ },
4522
+ {
4523
+ "entropy": 0.3753189181908965,
4524
+ "epoch": 2.135096835144072,
4525
+ "grad_norm": 0.07934729754924774,
4526
+ "learning_rate": 0.00017870635023170802,
4527
+ "loss": 0.35052378177642823,
4528
+ "mean_token_accuracy": 0.9095086582005024,
4529
+ "num_tokens": 63086813.0,
4530
+ "step": 4520
4531
+ },
4532
+ {
4533
+ "entropy": 0.344531574100256,
4534
+ "epoch": 2.13982050070855,
4535
+ "grad_norm": 0.06132613494992256,
4536
+ "learning_rate": 0.0001786145036980342,
4537
+ "loss": 0.31072587966918946,
4538
+ "mean_token_accuracy": 0.9187924608588218,
4539
+ "num_tokens": 63222413.0,
4540
+ "step": 4530
4541
+ },
4542
+ {
4543
+ "entropy": 0.3367819916456938,
4544
+ "epoch": 2.144544166273028,
4545
+ "grad_norm": 0.07123655825853348,
4546
+ "learning_rate": 0.000178522483218333,
4547
+ "loss": 0.3007503509521484,
4548
+ "mean_token_accuracy": 0.9209801577031612,
4549
+ "num_tokens": 63360839.0,
4550
+ "step": 4540
4551
+ },
4552
+ {
4553
+ "entropy": 0.37355535849928856,
4554
+ "epoch": 2.149267831837506,
4555
+ "grad_norm": 0.07354898750782013,
4556
+ "learning_rate": 0.00017843028899621315,
4557
+ "loss": 0.3450171947479248,
4558
+ "mean_token_accuracy": 0.9104658022522927,
4559
+ "num_tokens": 63502321.0,
4560
+ "step": 4550
4561
+ },
4562
+ {
4563
+ "entropy": 0.3626279551535845,
4564
+ "epoch": 2.153991497401984,
4565
+ "grad_norm": 0.07379408180713654,
4566
+ "learning_rate": 0.00017833792123566778,
4567
+ "loss": 0.3310466051101685,
4568
+ "mean_token_accuracy": 0.9146706216037274,
4569
+ "num_tokens": 63646863.0,
4570
+ "step": 4560
4571
+ },
4572
+ {
4573
+ "entropy": 0.3701190158724785,
4574
+ "epoch": 2.158715162966462,
4575
+ "grad_norm": 0.06171635165810585,
4576
+ "learning_rate": 0.00017824538014107403,
4577
+ "loss": 0.3444772005081177,
4578
+ "mean_token_accuracy": 0.9130326814949512,
4579
+ "num_tokens": 63788236.0,
4580
+ "step": 4570
4581
+ },
4582
+ {
4583
+ "entropy": 0.3738078149035573,
4584
+ "epoch": 2.16343882853094,
4585
+ "grad_norm": 0.05464085564017296,
4586
+ "learning_rate": 0.00017815266591719245,
4587
+ "loss": 0.35484113693237307,
4588
+ "mean_token_accuracy": 0.9113606236875057,
4589
+ "num_tokens": 63929307.0,
4590
+ "step": 4580
4591
+ },
4592
+ {
4593
+ "entropy": 0.3798240454867482,
4594
+ "epoch": 2.168162494095418,
4595
+ "grad_norm": 0.08261693269014359,
4596
+ "learning_rate": 0.00017805977876916685,
4597
+ "loss": 0.34636573791503905,
4598
+ "mean_token_accuracy": 0.9096949256956577,
4599
+ "num_tokens": 64065655.0,
4600
+ "step": 4590
4601
+ },
4602
+ {
4603
+ "entropy": 0.3755222076550126,
4604
+ "epoch": 2.172886159659896,
4605
+ "grad_norm": 0.06726128607988358,
4606
+ "learning_rate": 0.0001779667189025235,
4607
+ "loss": 0.341137433052063,
4608
+ "mean_token_accuracy": 0.912428331375122,
4609
+ "num_tokens": 64202678.0,
4610
+ "step": 4600
4611
+ },
4612
+ {
4613
+ "entropy": 0.32989654783159494,
4614
+ "epoch": 2.177609825224374,
4615
+ "grad_norm": 0.0664137676358223,
4616
+ "learning_rate": 0.00017787348652317092,
4617
+ "loss": 0.29708251953125,
4618
+ "mean_token_accuracy": 0.922088798135519,
4619
+ "num_tokens": 64343569.0,
4620
+ "step": 4610
4621
+ },
4622
+ {
4623
+ "entropy": 0.3934741189703345,
4624
+ "epoch": 2.182333490788852,
4625
+ "grad_norm": 0.08352284133434296,
4626
+ "learning_rate": 0.00017778008183739932,
4627
+ "loss": 0.35492236614227296,
4628
+ "mean_token_accuracy": 0.9071983322501183,
4629
+ "num_tokens": 64487045.0,
4630
+ "step": 4620
4631
+ },
4632
+ {
4633
+ "entropy": 0.34776588194072244,
4634
+ "epoch": 2.18705715635333,
4635
+ "grad_norm": 0.08185728639364243,
4636
+ "learning_rate": 0.0001776865050518801,
4637
+ "loss": 0.3108149528503418,
4638
+ "mean_token_accuracy": 0.9175512231886387,
4639
+ "num_tokens": 64628896.0,
4640
+ "step": 4630
4641
+ },
4642
+ {
4643
+ "entropy": 0.3408706252463162,
4644
+ "epoch": 2.191780821917808,
4645
+ "grad_norm": 0.07090574502944946,
4646
+ "learning_rate": 0.00017759275637366554,
4647
+ "loss": 0.3044494390487671,
4648
+ "mean_token_accuracy": 0.9201794132590294,
4649
+ "num_tokens": 64770622.0,
4650
+ "step": 4640
4651
+ },
4652
+ {
4653
+ "entropy": 0.38265586011111735,
4654
+ "epoch": 2.196504487482286,
4655
+ "grad_norm": 0.0640915110707283,
4656
+ "learning_rate": 0.0001774988360101883,
4657
+ "loss": 0.3519306182861328,
4658
+ "mean_token_accuracy": 0.9097047552466393,
4659
+ "num_tokens": 64914683.0,
4660
+ "step": 4650
4661
+ },
4662
+ {
4663
+ "entropy": 0.3646923953667283,
4664
+ "epoch": 2.201228153046764,
4665
+ "grad_norm": 0.05122111737728119,
4666
+ "learning_rate": 0.00017740474416926073,
4667
+ "loss": 0.34943673610687254,
4668
+ "mean_token_accuracy": 0.912996233254671,
4669
+ "num_tokens": 65046171.0,
4670
+ "step": 4660
4671
+ },
4672
+ {
4673
+ "entropy": 0.3709303382784128,
4674
+ "epoch": 2.205951818611242,
4675
+ "grad_norm": 0.07036547362804413,
4676
+ "learning_rate": 0.0001773104810590748,
4677
+ "loss": 0.3317831516265869,
4678
+ "mean_token_accuracy": 0.9141067914664746,
4679
+ "num_tokens": 65187824.0,
4680
+ "step": 4670
4681
+ },
4682
+ {
4683
+ "entropy": 0.37796775698661805,
4684
+ "epoch": 2.21067548417572,
4685
+ "grad_norm": 0.06316936761140823,
4686
+ "learning_rate": 0.00017721604688820133,
4687
+ "loss": 0.34329471588134763,
4688
+ "mean_token_accuracy": 0.9125158511102199,
4689
+ "num_tokens": 65326878.0,
4690
+ "step": 4680
4691
+ },
4692
+ {
4693
+ "entropy": 0.36794558390974996,
4694
+ "epoch": 2.2153991497401986,
4695
+ "grad_norm": 0.07478772103786469,
4696
+ "learning_rate": 0.00017712144186558972,
4697
+ "loss": 0.34176435470581057,
4698
+ "mean_token_accuracy": 0.9126577869057655,
4699
+ "num_tokens": 65472098.0,
4700
+ "step": 4690
4701
+ },
4702
+ {
4703
+ "entropy": 0.3847030168399215,
4704
+ "epoch": 2.2201228153046766,
4705
+ "grad_norm": 0.053054895251989365,
4706
+ "learning_rate": 0.00017702666620056726,
4707
+ "loss": 0.3531605005264282,
4708
+ "mean_token_accuracy": 0.9097660399973393,
4709
+ "num_tokens": 65607840.0,
4710
+ "step": 4700
4711
+ },
4712
+ {
4713
+ "entropy": 0.3529300346970558,
4714
+ "epoch": 2.2248464808691546,
4715
+ "grad_norm": 0.07372882217168808,
4716
+ "learning_rate": 0.00017693172010283892,
4717
+ "loss": 0.3232670783996582,
4718
+ "mean_token_accuracy": 0.9168261006474495,
4719
+ "num_tokens": 65740150.0,
4720
+ "step": 4710
4721
+ },
4722
+ {
4723
+ "entropy": 0.3606896920129657,
4724
+ "epoch": 2.2295701464336326,
4725
+ "grad_norm": 0.06723502278327942,
4726
+ "learning_rate": 0.00017683660378248678,
4727
+ "loss": 0.32462348937988283,
4728
+ "mean_token_accuracy": 0.9145806074142456,
4729
+ "num_tokens": 65877155.0,
4730
+ "step": 4720
4731
+ },
4732
+ {
4733
+ "entropy": 0.3608708480373025,
4734
+ "epoch": 2.2342938119981106,
4735
+ "grad_norm": 0.07947982847690582,
4736
+ "learning_rate": 0.0001767413174499695,
4737
+ "loss": 0.337389874458313,
4738
+ "mean_token_accuracy": 0.9128516048192978,
4739
+ "num_tokens": 66020895.0,
4740
+ "step": 4730
4741
+ },
4742
+ {
4743
+ "entropy": 0.3662800621241331,
4744
+ "epoch": 2.2390174775625886,
4745
+ "grad_norm": 0.07876905053853989,
4746
+ "learning_rate": 0.000176645861316122,
4747
+ "loss": 0.33322911262512206,
4748
+ "mean_token_accuracy": 0.9142575584352016,
4749
+ "num_tokens": 66167145.0,
4750
+ "step": 4740
4751
+ },
4752
+ {
4753
+ "entropy": 0.35156021658331155,
4754
+ "epoch": 2.2437411431270666,
4755
+ "grad_norm": 0.07309263199567795,
4756
+ "learning_rate": 0.00017655023559215484,
4757
+ "loss": 0.3214474439620972,
4758
+ "mean_token_accuracy": 0.9149869471788407,
4759
+ "num_tokens": 66306859.0,
4760
+ "step": 4750
4761
+ },
4762
+ {
4763
+ "entropy": 0.39834485165774824,
4764
+ "epoch": 2.2484648086915446,
4765
+ "grad_norm": 0.08221690356731415,
4766
+ "learning_rate": 0.00017645444048965388,
4767
+ "loss": 0.35496699810028076,
4768
+ "mean_token_accuracy": 0.9076141364872455,
4769
+ "num_tokens": 66445152.0,
4770
+ "step": 4760
4771
+ },
4772
+ {
4773
+ "entropy": 0.3521524195559323,
4774
+ "epoch": 2.2531884742560226,
4775
+ "grad_norm": 0.08107949793338776,
4776
+ "learning_rate": 0.00017635847622057965,
4777
+ "loss": 0.32071890830993655,
4778
+ "mean_token_accuracy": 0.9162487290799618,
4779
+ "num_tokens": 66578229.0,
4780
+ "step": 4770
4781
+ },
4782
+ {
4783
+ "entropy": 0.3437899772077799,
4784
+ "epoch": 2.2579121398205007,
4785
+ "grad_norm": 0.0685676783323288,
4786
+ "learning_rate": 0.00017626234299726715,
4787
+ "loss": 0.30377700328826907,
4788
+ "mean_token_accuracy": 0.9203708678483963,
4789
+ "num_tokens": 66719507.0,
4790
+ "step": 4780
4791
+ },
4792
+ {
4793
+ "entropy": 0.3620788909494877,
4794
+ "epoch": 2.2626358053849787,
4795
+ "grad_norm": 0.08134099096059799,
4796
+ "learning_rate": 0.00017616604103242502,
4797
+ "loss": 0.33079211711883544,
4798
+ "mean_token_accuracy": 0.9154773041605949,
4799
+ "num_tokens": 66861243.0,
4800
+ "step": 4790
4801
+ },
4802
+ {
4803
+ "entropy": 0.3565347112715244,
4804
+ "epoch": 2.2673594709494567,
4805
+ "grad_norm": 0.058874648064374924,
4806
+ "learning_rate": 0.00017606957053913544,
4807
+ "loss": 0.32880067825317383,
4808
+ "mean_token_accuracy": 0.9145106978714466,
4809
+ "num_tokens": 66999811.0,
4810
+ "step": 4800
4811
+ },
4812
+ {
4813
+ "entropy": 0.36329170744866135,
4814
+ "epoch": 2.2720831365139347,
4815
+ "grad_norm": 0.07846896350383759,
4816
+ "learning_rate": 0.00017597293173085343,
4817
+ "loss": 0.3312691688537598,
4818
+ "mean_token_accuracy": 0.9129531547427178,
4819
+ "num_tokens": 67131276.0,
4820
+ "step": 4810
4821
+ },
4822
+ {
4823
+ "entropy": 0.35137215442955494,
4824
+ "epoch": 2.2768068020784127,
4825
+ "grad_norm": 0.08317407965660095,
4826
+ "learning_rate": 0.00017587612482140635,
4827
+ "loss": 0.3243239164352417,
4828
+ "mean_token_accuracy": 0.9175820186734199,
4829
+ "num_tokens": 67266674.0,
4830
+ "step": 4820
4831
+ },
4832
+ {
4833
+ "entropy": 0.39138262122869494,
4834
+ "epoch": 2.2815304676428907,
4835
+ "grad_norm": 0.07166536897420883,
4836
+ "learning_rate": 0.00017577915002499358,
4837
+ "loss": 0.36102991104125975,
4838
+ "mean_token_accuracy": 0.9072135731577873,
4839
+ "num_tokens": 67410593.0,
4840
+ "step": 4830
4841
+ },
4842
+ {
4843
+ "entropy": 0.3518522471189499,
4844
+ "epoch": 2.286254133207369,
4845
+ "grad_norm": 0.07341722398996353,
4846
+ "learning_rate": 0.000175682007556186,
4847
+ "loss": 0.3146039009094238,
4848
+ "mean_token_accuracy": 0.9182766884565353,
4849
+ "num_tokens": 67552721.0,
4850
+ "step": 4840
4851
+ },
4852
+ {
4853
+ "entropy": 0.35175929674878714,
4854
+ "epoch": 2.2909777987718467,
4855
+ "grad_norm": 0.06283942610025406,
4856
+ "learning_rate": 0.00017558469762992549,
4857
+ "loss": 0.30020389556884763,
4858
+ "mean_token_accuracy": 0.9206199988722801,
4859
+ "num_tokens": 67688880.0,
4860
+ "step": 4850
4861
+ },
4862
+ {
4863
+ "entropy": 0.36785907223820685,
4864
+ "epoch": 2.295701464336325,
4865
+ "grad_norm": 0.07479214668273926,
4866
+ "learning_rate": 0.00017548722046152433,
4867
+ "loss": 0.3509315252304077,
4868
+ "mean_token_accuracy": 0.9113536290824413,
4869
+ "num_tokens": 67821502.0,
4870
+ "step": 4860
4871
+ },
4872
+ {
4873
+ "entropy": 0.36734033189713955,
4874
+ "epoch": 2.300425129900803,
4875
+ "grad_norm": 0.07622663676738739,
4876
+ "learning_rate": 0.000175389576266665,
4877
+ "loss": 0.33639395236968994,
4878
+ "mean_token_accuracy": 0.9118621356785297,
4879
+ "num_tokens": 67952323.0,
4880
+ "step": 4870
4881
+ },
4882
+ {
4883
+ "entropy": 0.38060117568820717,
4884
+ "epoch": 2.305148795465281,
4885
+ "grad_norm": 0.06672433763742447,
4886
+ "learning_rate": 0.00017529176526139944,
4887
+ "loss": 0.3438107013702393,
4888
+ "mean_token_accuracy": 0.9105553299188613,
4889
+ "num_tokens": 68089099.0,
4890
+ "step": 4880
4891
+ },
4892
+ {
4893
+ "entropy": 0.3727780839428306,
4894
+ "epoch": 2.309872461029759,
4895
+ "grad_norm": 0.07081436365842819,
4896
+ "learning_rate": 0.00017519378766214878,
4897
+ "loss": 0.3472867965698242,
4898
+ "mean_token_accuracy": 0.9104488752782345,
4899
+ "num_tokens": 68227401.0,
4900
+ "step": 4890
4901
+ },
4902
+ {
4903
+ "entropy": 0.3444125018082559,
4904
+ "epoch": 2.314596126594237,
4905
+ "grad_norm": 0.07869017869234085,
4906
+ "learning_rate": 0.0001750956436857027,
4907
+ "loss": 0.3122610092163086,
4908
+ "mean_token_accuracy": 0.9187357731163501,
4909
+ "num_tokens": 68372503.0,
4910
+ "step": 4900
4911
+ },
4912
+ {
4913
+ "entropy": 0.38529773708432913,
4914
+ "epoch": 2.319319792158715,
4915
+ "grad_norm": 0.06087201088666916,
4916
+ "learning_rate": 0.00017499733354921898,
4917
+ "loss": 0.35583522319793703,
4918
+ "mean_token_accuracy": 0.9096518360078335,
4919
+ "num_tokens": 68523575.0,
4920
+ "step": 4910
4921
+ },
4922
+ {
4923
+ "entropy": 0.329516320489347,
4924
+ "epoch": 2.324043457723193,
4925
+ "grad_norm": 0.06620240211486816,
4926
+ "learning_rate": 0.00017489885747022318,
4927
+ "loss": 0.2991222381591797,
4928
+ "mean_token_accuracy": 0.923500907421112,
4929
+ "num_tokens": 68662446.0,
4930
+ "step": 4920
4931
+ },
4932
+ {
4933
+ "entropy": 0.3849239324219525,
4934
+ "epoch": 2.328767123287671,
4935
+ "grad_norm": 0.08974209427833557,
4936
+ "learning_rate": 0.00017480021566660788,
4937
+ "loss": 0.3524339199066162,
4938
+ "mean_token_accuracy": 0.9093660332262516,
4939
+ "num_tokens": 68810406.0,
4940
+ "step": 4930
4941
+ },
4942
+ {
4943
+ "entropy": 0.3772364977747202,
4944
+ "epoch": 2.3334907888521492,
4945
+ "grad_norm": 0.07428587973117828,
4946
+ "learning_rate": 0.00017470140835663248,
4947
+ "loss": 0.35045742988586426,
4948
+ "mean_token_accuracy": 0.9099268049001694,
4949
+ "num_tokens": 68951698.0,
4950
+ "step": 4940
4951
+ },
4952
+ {
4953
+ "entropy": 0.3782703010365367,
4954
+ "epoch": 2.3382144544166272,
4955
+ "grad_norm": 0.08954386413097382,
4956
+ "learning_rate": 0.0001746024357589225,
4957
+ "loss": 0.3453653573989868,
4958
+ "mean_token_accuracy": 0.9120040215551853,
4959
+ "num_tokens": 69086778.0,
4960
+ "step": 4950
4961
+ },
4962
+ {
4963
+ "entropy": 0.3368293071165681,
4964
+ "epoch": 2.3429381199811052,
4965
+ "grad_norm": 0.06562545150518417,
4966
+ "learning_rate": 0.0001745032980924692,
4967
+ "loss": 0.30047316551208497,
4968
+ "mean_token_accuracy": 0.9200038239359856,
4969
+ "num_tokens": 69223879.0,
4970
+ "step": 4960
4971
+ },
4972
+ {
4973
+ "entropy": 0.33657576572149994,
4974
+ "epoch": 2.3476617855455832,
4975
+ "grad_norm": 0.0857587456703186,
4976
+ "learning_rate": 0.00017440399557662912,
4977
+ "loss": 0.2978757619857788,
4978
+ "mean_token_accuracy": 0.9229826048016548,
4979
+ "num_tokens": 69361696.0,
4980
+ "step": 4970
4981
+ },
4982
+ {
4983
+ "entropy": 0.35648726038634776,
4984
+ "epoch": 2.3523854511100613,
4985
+ "grad_norm": 0.06771888583898544,
4986
+ "learning_rate": 0.0001743045284311235,
4987
+ "loss": 0.3391841173171997,
4988
+ "mean_token_accuracy": 0.9125791676342487,
4989
+ "num_tokens": 69497700.0,
4990
+ "step": 4980
4991
+ },
4992
+ {
4993
+ "entropy": 0.377038012444973,
4994
+ "epoch": 2.3571091166745393,
4995
+ "grad_norm": 0.05512234941124916,
4996
+ "learning_rate": 0.0001742048968760379,
4997
+ "loss": 0.329133939743042,
4998
+ "mean_token_accuracy": 0.913535013794899,
4999
+ "num_tokens": 69638549.0,
5000
+ "step": 4990
5001
+ },
5002
+ {
5003
+ "entropy": 0.34691223232075574,
5004
+ "epoch": 2.3618327822390173,
5005
+ "grad_norm": 0.07337567210197449,
5006
+ "learning_rate": 0.0001741051011318216,
5007
+ "loss": 0.326473593711853,
5008
+ "mean_token_accuracy": 0.9155340820550919,
5009
+ "num_tokens": 69777845.0,
5010
+ "step": 5000
5011
  }
5012
  ],
5013
  "logging_steps": 10,
 
5027
  "attributes": {}
5028
  }
5029
  },
5030
+ "total_flos": 1.0520056670651228e+18,
5031
  "train_batch_size": 4,
5032
  "trial_name": null,
5033
  "trial_params": null