amirali1985 commited on
Commit
ce00ad3
·
verified ·
1 Parent(s): f5f3d63

Upload add_sub_sorl_v1_abs30_K1_10K

Browse files
add_sub_sorl_v1_abs30_K1_10K/metrics.json CHANGED
@@ -63,377 +63,377 @@
63
  3133
64
  ],
65
  "loss": [
66
- 7.419157981872559,
67
- 4.167179584503174,
68
- 2.757251739501953,
69
- 2.7242889404296875,
70
- 2.4392311573028564,
71
- 2.12624454498291,
72
- 1.7021116018295288,
73
- 0.1742163896560669,
74
- -4.171255588531494,
75
- -7.291606426239014,
76
- -7.83780574798584,
77
- -4.589130401611328,
78
- -2.949950695037842,
79
- -1.528172254562378,
80
- -0.9651578664779663,
81
- -0.8742300271987915,
82
- -0.550557017326355,
83
- -0.38707196712493896,
84
- -0.5141674876213074,
85
- -0.6698199510574341,
86
- -0.7084141373634338,
87
- -0.47245967388153076,
88
- -0.31419694423675537,
89
- -0.31762227416038513,
90
- -0.24917703866958618,
91
- -0.5615812540054321,
92
- -0.11299914121627808,
93
- -0.3741247057914734,
94
- -0.1392630785703659,
95
- -0.4355674088001251,
96
- -0.7360162138938904,
97
- -0.5835363268852234,
98
- -0.5749945640563965,
99
- -0.4588763415813446,
100
- -0.5538105964660645,
101
- -0.009306885302066803,
102
- -0.4344196915626526,
103
- -0.23470139503479004,
104
- -0.4863656759262085,
105
- -0.6077032685279846,
106
- -0.41746845841407776,
107
- -0.4240177273750305,
108
- -0.17653952538967133,
109
- -0.39587831497192383,
110
- -0.1691063493490219,
111
- -0.12267603725194931,
112
- -0.3334910571575165,
113
- -0.11227956414222717,
114
- -0.2049349546432495,
115
- -0.3078603148460388,
116
- -0.17479164898395538,
117
- -0.25693076848983765,
118
- -0.22230546176433563,
119
- -0.24037127196788788,
120
- -0.07592999190092087,
121
- -0.35476192831993103,
122
- -0.3569074273109436,
123
- -0.22312912344932556,
124
- -0.22871972620487213,
125
- -0.16561374068260193
126
  ],
127
  "base_loss": [
128
- 7.4173455238342285,
129
- 3.6480441093444824,
130
- 1.9713058471679688,
131
- 1.8219451904296875,
132
- 1.865543007850647,
133
- 1.8542261123657227,
134
- 1.805886149406433,
135
- 1.7817356586456299,
136
- 1.8499161005020142,
137
- 1.6429988145828247,
138
- 1.4622584581375122,
139
- 0.8687496185302734,
140
- 0.5731961131095886,
141
- 0.35015660524368286,
142
- 0.2690192759037018,
143
- 0.22296881675720215,
144
- 0.1571018397808075,
145
- 0.16091905534267426,
146
- 0.1413668543100357,
147
- 0.14242176711559296,
148
- 0.15262103080749512,
149
- 0.11749549955129623,
150
- 0.1065789982676506,
151
- 0.08413096517324448,
152
- 0.07932066917419434,
153
- 0.11860263347625732,
154
- 0.07409992069005966,
155
- 0.07803652435541153,
156
- 0.047815799713134766,
157
- 0.10229728370904922,
158
- 0.11566775292158127,
159
- 0.10984303057193756,
160
- 0.09876441955566406,
161
- 0.0688941702246666,
162
- 0.08261072635650635,
163
- 0.053319063037633896,
164
- 0.06365398317575455,
165
- 0.0505535863339901,
166
- 0.06858514994382858,
167
- 0.07909546047449112,
168
- 0.060411665588617325,
169
- 0.0625351145863533,
170
- 0.032719288021326065,
171
- 0.05432973429560661,
172
- 0.045225996524095535,
173
- 0.021227678284049034,
174
- 0.050432708114385605,
175
- 0.027083681896328926,
176
- 0.031196508556604385,
177
- 0.04615318775177002,
178
- 0.027639741078019142,
179
- 0.05231175199151039,
180
- 0.03158637136220932,
181
- 0.032559193670749664,
182
- 0.01280138734728098,
183
- 0.04558839276432991,
184
- 0.04613020643591881,
185
- 0.03275144845247269,
186
- 0.03132258728146553,
187
- 0.024606991559267044
188
  ],
189
  "info_loss": [
190
- -0.5185031890869141,
191
- -0.09349203109741211,
192
- -0.03490471839904785,
193
- -0.02159249782562256,
194
- -0.054520368576049805,
195
- -0.08344614505767822,
196
- -0.1214907169342041,
197
- -0.27150094509124756,
198
- -0.7121474742889404,
199
- -1.0041959285736084,
200
- -1.0389212369918823,
201
- -0.6523160934448242,
202
- -0.45543068647384644,
203
- -0.28470802307128906,
204
- -0.21153677999973297,
205
- -0.1875189244747162,
206
- -0.14062973856925964,
207
- -0.12236915528774261,
208
- -0.12718062102794647,
209
- -0.1365828663110733,
210
- -0.13193942606449127,
211
- -0.10466521978378296,
212
- -0.08482377231121063,
213
- -0.0746171772480011,
214
- -0.07218538224697113,
215
- -0.10219592601060867,
216
- -0.04982271045446396,
217
- -0.07324571162462234,
218
- -0.038586750626564026,
219
- -0.07255075871944427,
220
- -0.10315456241369247,
221
- -0.08851015567779541,
222
- -0.08371994644403458,
223
- -0.06724002212285995,
224
- -0.0802496150135994,
225
- -0.021388381719589233,
226
- -0.06261564046144485,
227
- -0.0424460805952549,
228
- -0.06758096814155579,
229
- -0.07665054500102997,
230
- -0.058085739612579346,
231
- -0.05829145759344101,
232
- -0.030241522938013077,
233
- -0.053621694445610046,
234
- -0.03214117884635925,
235
- -0.020935092121362686,
236
- -0.04635276272892952,
237
- -0.02176225744187832,
238
- -0.03096679225564003,
239
- -0.04063677042722702,
240
- -0.02714877389371395,
241
- -0.036504216492176056,
242
- -0.031483057886362076,
243
- -0.032049935311079025,
244
- -0.012561824172735214,
245
- -0.045204248279333115,
246
- -0.04523665830492973,
247
- -0.0305631123483181,
248
- -0.0310970451682806,
249
- -0.023466313257813454
250
  ],
251
  "abs_loss": [
252
- 3.381772994995117,
253
- 2.774198293685913,
254
- 2.777165412902832,
255
- 2.7333767414093018,
256
- 2.7525675296783447,
257
- 2.701953649520874,
258
- 2.742412567138672,
259
- 2.6781795024871826,
260
- 2.65073823928833,
261
- 2.6559343338012695,
262
- 2.530367612838745,
263
- 2.4332215785980225,
264
- 2.2735555171966553,
265
- 1.9705966711044312,
266
- 1.622453212738037,
267
- 1.4155868291854858,
268
- 1.1430540084838867,
269
- 0.9845269322395325,
270
- 0.9937983155250549,
271
- 0.802861750125885,
272
- 0.7566664814949036,
273
- 0.6577289700508118,
274
- 0.7418577075004578,
275
- 0.5929294228553772,
276
- 0.5775941014289856,
277
- 0.5590793490409851,
278
- 0.6108124256134033,
279
- 0.4465348720550537,
280
- 0.393932968378067,
281
- 0.3919040262699127,
282
- 0.41778573393821716,
283
- 0.4128030240535736,
284
- 0.36697685718536377,
285
- 0.3254868686199188,
286
- 0.3693109452724457,
287
- 0.3870414197444916,
288
- 0.2671542167663574,
289
- 0.23766420781612396,
290
- 0.27502405643463135,
291
- 0.19808150827884674,
292
- 0.2262984663248062,
293
- 0.24667154252529144,
294
- 0.24236536026000977,
295
- 0.20162977278232574,
296
- 0.2073269635438919,
297
- 0.18350571393966675,
298
- 0.1846071034669876,
299
- 0.15442465245723724,
300
- 0.1817246526479721,
301
- 0.14412124454975128,
302
- 0.14306403696537018,
303
- 0.12268459051847458,
304
- 0.13687384128570557,
305
- 0.11778446286916733,
306
- 0.11796697229146957,
307
- 0.10396292060613632,
308
- 0.11473483592271805,
309
- 0.10343429446220398,
310
- 0.07652456313371658,
311
- 0.10590269416570663
312
  ],
313
  "zipf_loss": [
314
- 4.848667144775391,
315
- 1.176635980606079,
316
- 0.8572765588760376,
317
- 0.8449309468269348,
318
- 0.8436350226402283,
319
- 0.8362844586372375,
320
- 0.8368913531303406,
321
- 0.8396722078323364,
322
- 0.8352292776107788,
323
- 0.8417611122131348,
324
- 0.836111843585968,
325
- 0.821958601474762,
326
- 0.8038046956062317,
327
- 0.7716918587684631,
328
- 0.7189455032348633,
329
- 0.6364318132400513,
330
- 0.5843331813812256,
331
- 0.5772478580474854,
332
- 0.51689213514328,
333
- 0.4733008146286011,
334
- 0.3826923966407776,
335
- 0.3909241557121277,
336
- 0.35327601432800293,
337
- 0.28512564301490784,
338
- 0.3355967402458191,
339
- 0.28586745262145996,
340
- 0.25004681944847107,
341
- 0.23564238846302032,
342
- 0.15939532220363617,
343
- 0.148452490568161,
344
- 0.13808311522006989,
345
- 0.15044189989566803,
346
- 0.12674278020858765,
347
- 0.11208108067512512,
348
- 0.12914372980594635,
349
- 0.11255372315645218,
350
- 0.10136730968952179,
351
- 0.11543937772512436,
352
- 0.09335644543170929,
353
- 0.05989854037761688,
354
- 0.08034739643335342,
355
- 0.07169462740421295,
356
- 0.06891988217830658,
357
- 0.06584595888853073,
358
- 0.08634673058986664,
359
- 0.04709663242101669,
360
- 0.061143141239881516,
361
- 0.06281687319278717,
362
- 0.055363979190588,
363
- 0.037942074239254,
364
- 0.054749950766563416,
365
- 0.04353116825222969,
366
- 0.04725135862827301,
367
- 0.03579045459628105,
368
- 0.02509017288684845,
369
- 0.041295863687992096,
370
- 0.03785546123981476,
371
- 0.03940713033080101,
372
- 0.04327566549181938,
373
- 0.03385213017463684
374
  ],
375
  "denoise_loss": [],
376
  "ortho_loss": [
377
- 0.3230772912502289,
378
- 0.1277298629283905,
379
- 0.10143587738275528,
380
- 0.08668191730976105,
381
- 0.09006865322589874,
382
- 0.07957521826028824,
383
- 0.09151509404182434,
384
- 0.09135851263999939,
385
- 0.1040191501379013,
386
- 0.118146151304245,
387
- 0.12644976377487183,
388
- 0.13958248496055603,
389
- 0.15548494458198547,
390
- 0.16445577144622803,
391
- 0.17663978040218353,
392
- 0.17986539006233215,
393
- 0.1787496656179428,
394
- 0.18172365427017212,
395
- 0.184878870844841,
396
- 0.18665480613708496,
397
- 0.18443602323532104,
398
- 0.17972944676876068,
399
- 0.1847168207168579,
400
- 0.18398721516132355,
401
- 0.18395553529262543,
402
- 0.1859062761068344,
403
- 0.1844593584537506,
404
- 0.19130879640579224,
405
- 0.18957658112049103,
406
- 0.18175643682479858,
407
- 0.18003104627132416,
408
- 0.18403305113315582,
409
- 0.1801728755235672,
410
- 0.17507442831993103,
411
- 0.18054728209972382,
412
- 0.17856059968471527,
413
- 0.1795228123664856,
414
- 0.17617978155612946,
415
- 0.17519906163215637,
416
- 0.17287763953208923,
417
- 0.1746804416179657,
418
- 0.1751648634672165,
419
- 0.17409953474998474,
420
- 0.17315584421157837,
421
- 0.17171329259872437,
422
- 0.17051003873348236,
423
- 0.17108182609081268,
424
- 0.1729588657617569,
425
- 0.16980983316898346,
426
- 0.1726285070180893,
427
- 0.1682920753955841,
428
- 0.1723119467496872,
429
- 0.1724519282579422,
430
- 0.17354536056518555,
431
- 0.17300596833229065,
432
- 0.17255157232284546,
433
- 0.17369846999645233,
434
- 0.17459796369075775,
435
- 0.17422601580619812,
436
- 0.17440278828144073
437
  ],
438
  "lr": [
439
  3.9200000000000004e-05,
@@ -521,6 +521,8 @@
521
  3083
522
  ],
523
  "eval_accuracy": [
 
 
524
  0.01,
525
  0.0,
526
  0.0,
@@ -538,474 +540,472 @@
538
  0.0,
539
  0.0,
540
  0.0,
541
- 0.0,
542
- 0.0,
543
  0.0
544
  ]
545
  },
546
- "final_accuracy": 0.96125,
547
  "sft_eval": {
548
  "config": {
549
  "ops": "add_sub",
550
  "K": null,
551
  "mode": "sft",
552
  "n_digits": 6,
553
- "n_per_split": 50
554
  },
555
  "splits": {
556
  "add_S0": {
557
- "full_accuracy": 0.94,
558
- "digit_accuracy": 0.9914285714285714,
559
- "n_examples": 50,
560
  "per_subtask": {
561
  "SA": {
562
- "accuracy": 0.9898305084745763,
563
- "count": 295
564
  },
565
  "SS": {
566
  "accuracy": 1.0,
567
- "count": 55
568
  }
569
  }
570
  },
571
  "add_S1": {
572
  "full_accuracy": 1.0,
573
  "digit_accuracy": 1.0,
574
- "n_examples": 50,
575
  "per_subtask": {
576
  "SA": {
577
  "accuracy": 1.0,
578
- "count": 126
579
  },
580
  "SC": {
581
  "accuracy": 1.0,
582
- "count": 79
583
  },
584
  "SS": {
585
  "accuracy": 1.0,
586
- "count": 21
587
  },
588
  "UC": {
589
  "accuracy": 1.0,
590
- "count": 124
591
  }
592
  }
593
  },
594
  "add_S2": {
595
- "full_accuracy": 0.96,
596
- "digit_accuracy": 0.9942857142857143,
597
- "n_examples": 50,
598
  "per_subtask": {
599
  "SA": {
600
  "accuracy": 1.0,
601
- "count": 75
602
  },
603
  "SC": {
604
- "accuracy": 0.967741935483871,
605
- "count": 62
606
  },
607
  "SS": {
608
  "accuracy": 1.0,
609
- "count": 39
610
  },
611
  "UC": {
612
  "accuracy": 1.0,
613
- "count": 111
614
  },
615
  "US": {
616
  "accuracy": 1.0,
617
- "count": 63
618
  }
619
  }
620
  },
621
  "add_S3": {
622
- "full_accuracy": 0.68,
623
- "digit_accuracy": 0.9542857142857143,
624
- "n_examples": 50,
625
  "per_subtask": {
626
  "SA": {
627
  "accuracy": 1.0,
628
- "count": 60
629
  },
630
  "SC": {
631
- "accuracy": 0.9824561403508771,
632
- "count": 57
633
  },
634
  "SS": {
635
  "accuracy": 1.0,
636
- "count": 19
637
  },
638
  "UC": {
639
- "accuracy": 0.8557692307692307,
640
- "count": 104
641
  },
642
  "US": {
643
  "accuracy": 1.0,
644
- "count": 110
645
  }
646
  }
647
  },
648
  "add_S4": {
649
- "full_accuracy": 0.66,
650
- "digit_accuracy": 0.94,
651
- "n_examples": 50,
652
  "per_subtask": {
653
  "SA": {
654
  "accuracy": 1.0,
655
- "count": 48
656
  },
657
  "SC": {
658
  "accuracy": 1.0,
659
- "count": 52
660
  },
661
  "SS": {
662
  "accuracy": 1.0,
663
- "count": 7
664
  },
665
  "UC": {
666
- "accuracy": 0.898876404494382,
667
- "count": 89
668
  },
669
  "US": {
670
- "accuracy": 0.922077922077922,
671
- "count": 154
672
  }
673
  }
674
  },
675
  "add_S5": {
676
- "full_accuracy": 0.68,
677
- "digit_accuracy": 0.8714285714285714,
678
- "n_examples": 50,
679
  "per_subtask": {
680
  "SA": {
681
  "accuracy": 1.0,
682
- "count": 50
683
  },
684
  "SC": {
685
  "accuracy": 1.0,
686
- "count": 50
687
  },
688
  "UC": {
689
- "accuracy": 0.74,
690
- "count": 50
691
  },
692
  "US": {
693
- "accuracy": 0.84,
694
- "count": 200
695
  }
696
  }
697
  },
698
  "add_S6": {
699
- "full_accuracy": 1.0,
700
- "digit_accuracy": 1.0,
701
- "n_examples": 50,
702
  "per_subtask": {
703
  "SC": {
704
  "accuracy": 1.0,
705
- "count": 50
706
  },
707
  "UC": {
708
- "accuracy": 1.0,
709
- "count": 50
710
  },
711
  "US": {
712
- "accuracy": 1.0,
713
- "count": 250
714
  }
715
  }
716
  },
717
  "add_random": {
718
- "full_accuracy": 0.99,
719
- "digit_accuracy": 0.9985714285714286,
720
  "n_examples": 200,
721
  "per_subtask": {
722
  "SA": {
723
  "accuracy": 1.0,
724
- "count": 431
725
  },
726
  "SC": {
727
  "accuracy": 1.0,
728
- "count": 316
729
  },
730
  "SS": {
731
  "accuracy": 1.0,
732
- "count": 39
733
  },
734
  "UC": {
735
- "accuracy": 0.9964285714285714,
736
- "count": 560
737
  },
738
  "US": {
739
  "accuracy": 1.0,
740
- "count": 54
741
  }
742
  }
743
  },
744
  "add_C1": {
745
- "full_accuracy": 0.98,
746
- "digit_accuracy": 0.9971428571428571,
747
- "n_examples": 50,
748
  "per_subtask": {
749
  "SA": {
750
- "accuracy": 0.996,
751
- "count": 250
752
  },
753
  "SC": {
754
  "accuracy": 1.0,
755
- "count": 50
756
  },
757
  "UC": {
758
  "accuracy": 1.0,
759
- "count": 50
760
  }
761
  }
762
  },
763
  "add_C2": {
764
  "full_accuracy": 1.0,
765
  "digit_accuracy": 1.0,
766
- "n_examples": 50,
767
  "per_subtask": {
768
  "SA": {
769
  "accuracy": 1.0,
770
- "count": 200
771
  },
772
  "SC": {
773
  "accuracy": 1.0,
774
- "count": 50
775
  },
776
  "UC": {
777
  "accuracy": 1.0,
778
- "count": 83
779
  },
780
  "US": {
781
  "accuracy": 1.0,
782
- "count": 17
783
  }
784
  }
785
  },
786
  "add_C3": {
787
- "full_accuracy": 0.8,
788
- "digit_accuracy": 0.9714285714285714,
789
- "n_examples": 50,
790
  "per_subtask": {
791
  "SA": {
792
- "accuracy": 0.9933333333333333,
793
- "count": 150
794
  },
795
  "SC": {
796
  "accuracy": 1.0,
797
- "count": 50
798
  },
799
  "UC": {
800
- "accuracy": 0.91,
801
- "count": 100
802
  },
803
  "US": {
804
  "accuracy": 1.0,
805
- "count": 50
806
  }
807
  }
808
  },
809
  "add_C4": {
810
- "full_accuracy": 0.86,
811
- "digit_accuracy": 0.9771428571428571,
812
- "n_examples": 50,
813
  "per_subtask": {
814
  "SA": {
815
  "accuracy": 1.0,
816
- "count": 100
817
  },
818
  "SC": {
819
  "accuracy": 1.0,
820
- "count": 50
821
  },
822
  "UC": {
823
- "accuracy": 0.9545454545454546,
824
- "count": 132
825
  },
826
  "US": {
827
- "accuracy": 0.9705882352941176,
828
- "count": 68
829
  }
830
  }
831
  },
832
  "add_C5": {
833
- "full_accuracy": 0.76,
834
- "digit_accuracy": 0.9571428571428572,
835
- "n_examples": 50,
836
  "per_subtask": {
837
  "SA": {
838
  "accuracy": 1.0,
839
- "count": 50
840
  },
841
  "SC": {
842
  "accuracy": 1.0,
843
- "count": 50
844
  },
845
  "UC": {
846
- "accuracy": 0.9383561643835616,
847
- "count": 146
848
  },
849
  "US": {
850
- "accuracy": 0.9423076923076923,
851
- "count": 104
852
  }
853
  }
854
  },
855
  "add_C6": {
856
- "full_accuracy": 0.92,
857
- "digit_accuracy": 0.9885714285714285,
858
- "n_examples": 50,
859
  "per_subtask": {
860
  "SC": {
861
  "accuracy": 1.0,
862
- "count": 50
863
  },
864
  "UC": {
865
- "accuracy": 0.9788359788359788,
866
- "count": 189
867
  },
868
  "US": {
869
  "accuracy": 1.0,
870
- "count": 111
871
  }
872
  }
873
  },
874
  "sub_M0": {
875
  "full_accuracy": 1.0,
876
  "digit_accuracy": 1.0,
877
- "n_examples": 50,
878
  "per_subtask": {
879
  "MD": {
880
  "accuracy": 1.0,
881
- "count": 303
882
  },
883
  "ME": {
884
  "accuracy": 1.0,
885
- "count": 47
886
  }
887
  }
888
  },
889
  "sub_M1": {
890
- "full_accuracy": 1.0,
891
- "digit_accuracy": 1.0,
892
- "n_examples": 50,
893
  "per_subtask": {
894
  "MD": {
895
  "accuracy": 1.0,
896
- "count": 141
897
  },
898
  "MB": {
899
- "accuracy": 1.0,
900
- "count": 72
901
  },
902
  "ME": {
903
  "accuracy": 1.0,
904
- "count": 18
905
  },
906
  "UB": {
907
  "accuracy": 1.0,
908
- "count": 119
909
  }
910
  }
911
  },
912
  "sub_M2": {
913
  "full_accuracy": 1.0,
914
  "digit_accuracy": 1.0,
915
- "n_examples": 50,
916
  "per_subtask": {
917
  "MD": {
918
  "accuracy": 1.0,
919
- "count": 112
920
  },
921
  "MB": {
922
  "accuracy": 1.0,
923
- "count": 53
924
  },
925
  "ME": {
926
  "accuracy": 1.0,
927
- "count": 47
928
  },
929
  "UB": {
930
  "accuracy": 1.0,
931
- "count": 85
932
  },
933
  "UD": {
934
  "accuracy": 1.0,
935
- "count": 53
936
  }
937
  }
938
  },
939
  "sub_M3": {
940
- "full_accuracy": 0.32,
941
- "digit_accuracy": 0.9028571428571428,
942
- "n_examples": 50,
943
  "per_subtask": {
944
  "MD": {
945
  "accuracy": 1.0,
946
- "count": 97
947
  },
948
  "MB": {
949
  "accuracy": 1.0,
950
- "count": 51
951
  },
952
  "ME": {
953
  "accuracy": 1.0,
954
- "count": 27
955
  },
956
  "UB": {
957
- "accuracy": 0.5405405405405406,
958
- "count": 74
959
  },
960
  "UD": {
961
  "accuracy": 1.0,
962
- "count": 101
963
  }
964
  }
965
  },
966
  "sub_M4": {
967
- "full_accuracy": 0.0,
968
- "digit_accuracy": 0.8028571428571428,
969
- "n_examples": 50,
970
  "per_subtask": {
971
  "MD": {
972
  "accuracy": 1.0,
973
- "count": 100
974
  },
975
  "MB": {
976
  "accuracy": 1.0,
977
- "count": 50
978
  },
979
  "UB": {
980
- "accuracy": 0.6,
981
- "count": 50
982
  },
983
  "UD": {
984
- "accuracy": 0.6733333333333333,
985
- "count": 150
986
  }
987
  }
988
  },
989
  "sub_M5": {
990
- "full_accuracy": 0.0,
991
- "digit_accuracy": 0.6428571428571429,
992
- "n_examples": 50,
993
  "per_subtask": {
994
  "MD": {
995
  "accuracy": 1.0,
996
- "count": 50
997
  },
998
  "MB": {
999
  "accuracy": 1.0,
1000
- "count": 50
1001
  },
1002
  "UB": {
1003
- "accuracy": 0.46,
1004
- "count": 50
1005
  },
1006
  "UD": {
1007
- "accuracy": 0.51,
1008
- "count": 200
1009
  }
1010
  }
1011
  },
@@ -1016,11 +1016,11 @@
1016
  "per_subtask": {
1017
  "MD": {
1018
  "accuracy": 1.0,
1019
- "count": 570
1020
  },
1021
  "MB": {
1022
  "accuracy": 1.0,
1023
- "count": 277
1024
  },
1025
  "ME": {
1026
  "accuracy": 1.0,
@@ -1028,88 +1028,88 @@
1028
  },
1029
  "UB": {
1030
  "accuracy": 1.0,
1031
- "count": 471
1032
  },
1033
  "UD": {
1034
  "accuracy": 1.0,
1035
- "count": 29
1036
  }
1037
  }
1038
  },
1039
  "sub_B3": {
1040
- "full_accuracy": 0.9,
1041
- "digit_accuracy": 0.9857142857142858,
1042
- "n_examples": 50,
1043
  "per_subtask": {
1044
  "MD": {
1045
  "accuracy": 1.0,
1046
- "count": 150
1047
  },
1048
  "MB": {
1049
  "accuracy": 1.0,
1050
- "count": 50
1051
  },
1052
  "UB": {
1053
- "accuracy": 0.9504950495049505,
1054
- "count": 101
1055
  },
1056
  "UD": {
1057
  "accuracy": 1.0,
1058
- "count": 49
1059
  }
1060
  }
1061
  },
1062
  "sub_B4": {
1063
- "full_accuracy": 0.68,
1064
- "digit_accuracy": 0.9457142857142857,
1065
- "n_examples": 50,
1066
  "per_subtask": {
1067
  "MD": {
1068
  "accuracy": 1.0,
1069
- "count": 100
1070
  },
1071
  "MB": {
1072
  "accuracy": 1.0,
1073
- "count": 50
1074
  },
1075
  "UB": {
1076
- "accuracy": 0.9008264462809917,
1077
- "count": 121
1078
  },
1079
  "UD": {
1080
- "accuracy": 0.9113924050632911,
1081
- "count": 79
1082
  }
1083
  }
1084
  },
1085
  "sub_B5": {
1086
- "full_accuracy": 0.74,
1087
- "digit_accuracy": 0.9514285714285714,
1088
- "n_examples": 50,
1089
  "per_subtask": {
1090
  "MD": {
1091
  "accuracy": 1.0,
1092
- "count": 50
1093
  },
1094
  "MB": {
1095
  "accuracy": 1.0,
1096
- "count": 50
1097
  },
1098
  "UB": {
1099
- "accuracy": 0.9407894736842105,
1100
- "count": 152
1101
  },
1102
  "UD": {
1103
- "accuracy": 0.9183673469387755,
1104
- "count": 98
1105
  }
1106
  }
1107
  }
1108
  },
1109
  "summary": {
1110
- "overall_accuracy": 0.828,
1111
- "digit_accuracy": 0.9622857142857143,
1112
- "total_examples": 1500,
1113
  "n_splits": 24
1114
  }
1115
  },
@@ -1119,167 +1119,167 @@
1119
  "K": 1,
1120
  "mode": "sorl",
1121
  "n_digits": 6,
1122
- "n_per_split": 50
1123
  },
1124
  "splits": {
1125
  "add_S0": {
1126
  "full_accuracy": 1.0,
1127
  "digit_accuracy": 1.0,
1128
- "n_examples": 50,
1129
  "per_subtask": {
1130
  "SA": {
1131
  "accuracy": 1.0,
1132
- "count": 295
1133
  },
1134
  "SS": {
1135
  "accuracy": 1.0,
1136
- "count": 55
1137
  }
1138
  }
1139
  },
1140
  "add_S1": {
1141
  "full_accuracy": 1.0,
1142
  "digit_accuracy": 1.0,
1143
- "n_examples": 50,
1144
  "per_subtask": {
1145
  "SA": {
1146
  "accuracy": 1.0,
1147
- "count": 126
1148
  },
1149
  "SC": {
1150
  "accuracy": 1.0,
1151
- "count": 79
1152
  },
1153
  "SS": {
1154
  "accuracy": 1.0,
1155
- "count": 21
1156
  },
1157
  "UC": {
1158
  "accuracy": 1.0,
1159
- "count": 124
1160
  }
1161
  }
1162
  },
1163
  "add_S2": {
1164
  "full_accuracy": 1.0,
1165
  "digit_accuracy": 1.0,
1166
- "n_examples": 50,
1167
  "per_subtask": {
1168
  "SA": {
1169
  "accuracy": 1.0,
1170
- "count": 75
1171
  },
1172
  "SC": {
1173
  "accuracy": 1.0,
1174
- "count": 62
1175
  },
1176
  "SS": {
1177
  "accuracy": 1.0,
1178
- "count": 39
1179
  },
1180
  "UC": {
1181
  "accuracy": 1.0,
1182
- "count": 111
1183
  },
1184
  "US": {
1185
  "accuracy": 1.0,
1186
- "count": 63
1187
  }
1188
  }
1189
  },
1190
  "add_S3": {
1191
  "full_accuracy": 1.0,
1192
  "digit_accuracy": 1.0,
1193
- "n_examples": 50,
1194
  "per_subtask": {
1195
  "SA": {
1196
  "accuracy": 1.0,
1197
- "count": 60
1198
  },
1199
  "SC": {
1200
  "accuracy": 1.0,
1201
- "count": 57
1202
  },
1203
  "SS": {
1204
  "accuracy": 1.0,
1205
- "count": 19
1206
  },
1207
  "UC": {
1208
  "accuracy": 1.0,
1209
- "count": 104
1210
  },
1211
  "US": {
1212
  "accuracy": 1.0,
1213
- "count": 110
1214
  }
1215
  }
1216
  },
1217
  "add_S4": {
1218
  "full_accuracy": 1.0,
1219
  "digit_accuracy": 1.0,
1220
- "n_examples": 50,
1221
  "per_subtask": {
1222
  "SA": {
1223
  "accuracy": 1.0,
1224
- "count": 48
1225
  },
1226
  "SC": {
1227
  "accuracy": 1.0,
1228
- "count": 52
1229
  },
1230
  "SS": {
1231
  "accuracy": 1.0,
1232
- "count": 7
1233
  },
1234
  "UC": {
1235
  "accuracy": 1.0,
1236
- "count": 89
1237
  },
1238
  "US": {
1239
  "accuracy": 1.0,
1240
- "count": 154
1241
  }
1242
  }
1243
  },
1244
  "add_S5": {
1245
- "full_accuracy": 0.98,
1246
- "digit_accuracy": 0.9971428571428571,
1247
- "n_examples": 50,
1248
  "per_subtask": {
1249
  "SA": {
1250
  "accuracy": 1.0,
1251
- "count": 50
1252
  },
1253
  "SC": {
1254
  "accuracy": 1.0,
1255
- "count": 50
1256
  },
1257
  "UC": {
1258
- "accuracy": 0.98,
1259
- "count": 50
1260
  },
1261
  "US": {
1262
  "accuracy": 1.0,
1263
- "count": 200
1264
  }
1265
  }
1266
  },
1267
  "add_S6": {
1268
- "full_accuracy": 1.0,
1269
- "digit_accuracy": 1.0,
1270
- "n_examples": 50,
1271
  "per_subtask": {
1272
  "SC": {
1273
  "accuracy": 1.0,
1274
- "count": 50
1275
  },
1276
  "UC": {
1277
  "accuracy": 1.0,
1278
- "count": 50
1279
  },
1280
  "US": {
1281
- "accuracy": 1.0,
1282
- "count": 250
1283
  }
1284
  }
1285
  },
@@ -1290,291 +1290,291 @@
1290
  "per_subtask": {
1291
  "SA": {
1292
  "accuracy": 1.0,
1293
- "count": 431
1294
  },
1295
  "SC": {
1296
  "accuracy": 1.0,
1297
- "count": 316
1298
  },
1299
  "SS": {
1300
  "accuracy": 1.0,
1301
- "count": 39
1302
  },
1303
  "UC": {
1304
  "accuracy": 1.0,
1305
- "count": 560
1306
  },
1307
  "US": {
1308
  "accuracy": 1.0,
1309
- "count": 54
1310
  }
1311
  }
1312
  },
1313
  "add_C1": {
1314
  "full_accuracy": 1.0,
1315
  "digit_accuracy": 1.0,
1316
- "n_examples": 50,
1317
  "per_subtask": {
1318
  "SA": {
1319
  "accuracy": 1.0,
1320
- "count": 250
1321
  },
1322
  "SC": {
1323
  "accuracy": 1.0,
1324
- "count": 50
1325
  },
1326
  "UC": {
1327
  "accuracy": 1.0,
1328
- "count": 50
1329
  }
1330
  }
1331
  },
1332
  "add_C2": {
1333
  "full_accuracy": 1.0,
1334
  "digit_accuracy": 1.0,
1335
- "n_examples": 50,
1336
  "per_subtask": {
1337
  "SA": {
1338
  "accuracy": 1.0,
1339
- "count": 200
1340
  },
1341
  "SC": {
1342
  "accuracy": 1.0,
1343
- "count": 50
1344
  },
1345
  "UC": {
1346
  "accuracy": 1.0,
1347
- "count": 83
1348
  },
1349
  "US": {
1350
  "accuracy": 1.0,
1351
- "count": 17
1352
  }
1353
  }
1354
  },
1355
  "add_C3": {
1356
  "full_accuracy": 1.0,
1357
  "digit_accuracy": 1.0,
1358
- "n_examples": 50,
1359
  "per_subtask": {
1360
  "SA": {
1361
  "accuracy": 1.0,
1362
- "count": 150
1363
  },
1364
  "SC": {
1365
  "accuracy": 1.0,
1366
- "count": 50
1367
  },
1368
  "UC": {
1369
  "accuracy": 1.0,
1370
- "count": 100
1371
  },
1372
  "US": {
1373
  "accuracy": 1.0,
1374
- "count": 50
1375
  }
1376
  }
1377
  },
1378
  "add_C4": {
1379
  "full_accuracy": 1.0,
1380
  "digit_accuracy": 1.0,
1381
- "n_examples": 50,
1382
  "per_subtask": {
1383
  "SA": {
1384
  "accuracy": 1.0,
1385
- "count": 100
1386
  },
1387
  "SC": {
1388
  "accuracy": 1.0,
1389
- "count": 50
1390
  },
1391
  "UC": {
1392
  "accuracy": 1.0,
1393
- "count": 132
1394
  },
1395
  "US": {
1396
  "accuracy": 1.0,
1397
- "count": 68
1398
  }
1399
  }
1400
  },
1401
  "add_C5": {
1402
- "full_accuracy": 1.0,
1403
- "digit_accuracy": 1.0,
1404
- "n_examples": 50,
1405
  "per_subtask": {
1406
  "SA": {
1407
  "accuracy": 1.0,
1408
- "count": 50
1409
  },
1410
  "SC": {
1411
  "accuracy": 1.0,
1412
- "count": 50
1413
  },
1414
  "UC": {
1415
- "accuracy": 1.0,
1416
- "count": 146
1417
  },
1418
  "US": {
1419
  "accuracy": 1.0,
1420
- "count": 104
1421
  }
1422
  }
1423
  },
1424
  "add_C6": {
1425
- "full_accuracy": 1.0,
1426
- "digit_accuracy": 1.0,
1427
- "n_examples": 50,
1428
  "per_subtask": {
1429
  "SC": {
1430
  "accuracy": 1.0,
1431
- "count": 50
1432
  },
1433
  "UC": {
1434
- "accuracy": 1.0,
1435
- "count": 189
1436
  },
1437
  "US": {
1438
  "accuracy": 1.0,
1439
- "count": 111
1440
  }
1441
  }
1442
  },
1443
  "sub_M0": {
1444
  "full_accuracy": 1.0,
1445
  "digit_accuracy": 1.0,
1446
- "n_examples": 50,
1447
  "per_subtask": {
1448
  "MD": {
1449
  "accuracy": 1.0,
1450
- "count": 303
1451
  },
1452
  "ME": {
1453
  "accuracy": 1.0,
1454
- "count": 47
1455
  }
1456
  }
1457
  },
1458
  "sub_M1": {
1459
  "full_accuracy": 1.0,
1460
  "digit_accuracy": 1.0,
1461
- "n_examples": 50,
1462
  "per_subtask": {
1463
  "MD": {
1464
  "accuracy": 1.0,
1465
- "count": 141
1466
  },
1467
  "MB": {
1468
  "accuracy": 1.0,
1469
- "count": 72
1470
  },
1471
  "ME": {
1472
  "accuracy": 1.0,
1473
- "count": 18
1474
  },
1475
  "UB": {
1476
  "accuracy": 1.0,
1477
- "count": 119
1478
  }
1479
  }
1480
  },
1481
  "sub_M2": {
1482
  "full_accuracy": 1.0,
1483
  "digit_accuracy": 1.0,
1484
- "n_examples": 50,
1485
  "per_subtask": {
1486
  "MD": {
1487
  "accuracy": 1.0,
1488
- "count": 112
1489
  },
1490
  "MB": {
1491
  "accuracy": 1.0,
1492
- "count": 53
1493
  },
1494
  "ME": {
1495
  "accuracy": 1.0,
1496
- "count": 47
1497
  },
1498
  "UB": {
1499
  "accuracy": 1.0,
1500
- "count": 85
1501
  },
1502
  "UD": {
1503
  "accuracy": 1.0,
1504
- "count": 53
1505
  }
1506
  }
1507
  },
1508
  "sub_M3": {
1509
  "full_accuracy": 1.0,
1510
  "digit_accuracy": 1.0,
1511
- "n_examples": 50,
1512
  "per_subtask": {
1513
  "MD": {
1514
  "accuracy": 1.0,
1515
- "count": 97
1516
  },
1517
  "MB": {
1518
  "accuracy": 1.0,
1519
- "count": 51
1520
  },
1521
  "ME": {
1522
  "accuracy": 1.0,
1523
- "count": 27
1524
  },
1525
  "UB": {
1526
  "accuracy": 1.0,
1527
- "count": 74
1528
  },
1529
  "UD": {
1530
  "accuracy": 1.0,
1531
- "count": 101
1532
  }
1533
  }
1534
  },
1535
  "sub_M4": {
1536
  "full_accuracy": 1.0,
1537
  "digit_accuracy": 1.0,
1538
- "n_examples": 50,
1539
  "per_subtask": {
1540
  "MD": {
1541
  "accuracy": 1.0,
1542
- "count": 100
1543
  },
1544
  "MB": {
1545
  "accuracy": 1.0,
1546
- "count": 50
1547
  },
1548
  "UB": {
1549
  "accuracy": 1.0,
1550
- "count": 50
1551
  },
1552
  "UD": {
1553
  "accuracy": 1.0,
1554
- "count": 150
1555
  }
1556
  }
1557
  },
1558
  "sub_M5": {
1559
- "full_accuracy": 0.06,
1560
- "digit_accuracy": 0.8657142857142858,
1561
- "n_examples": 50,
1562
  "per_subtask": {
1563
  "MD": {
1564
  "accuracy": 1.0,
1565
- "count": 50
1566
  },
1567
  "MB": {
1568
  "accuracy": 1.0,
1569
- "count": 50
1570
  },
1571
  "UB": {
1572
- "accuracy": 0.06,
1573
- "count": 50
1574
  },
1575
  "UD": {
1576
  "accuracy": 1.0,
1577
- "count": 200
1578
  }
1579
  }
1580
  },
@@ -1585,11 +1585,11 @@
1585
  "per_subtask": {
1586
  "MD": {
1587
  "accuracy": 1.0,
1588
- "count": 570
1589
  },
1590
  "MB": {
1591
  "accuracy": 1.0,
1592
- "count": 277
1593
  },
1594
  "ME": {
1595
  "accuracy": 1.0,
@@ -1597,91 +1597,91 @@
1597
  },
1598
  "UB": {
1599
  "accuracy": 1.0,
1600
- "count": 471
1601
  },
1602
  "UD": {
1603
  "accuracy": 1.0,
1604
- "count": 29
1605
  }
1606
  }
1607
  },
1608
  "sub_B3": {
1609
  "full_accuracy": 1.0,
1610
  "digit_accuracy": 1.0,
1611
- "n_examples": 50,
1612
  "per_subtask": {
1613
  "MD": {
1614
  "accuracy": 1.0,
1615
- "count": 150
1616
  },
1617
  "MB": {
1618
  "accuracy": 1.0,
1619
- "count": 50
1620
  },
1621
  "UB": {
1622
  "accuracy": 1.0,
1623
- "count": 101
1624
  },
1625
  "UD": {
1626
  "accuracy": 1.0,
1627
- "count": 49
1628
  }
1629
  }
1630
  },
1631
  "sub_B4": {
1632
  "full_accuracy": 1.0,
1633
  "digit_accuracy": 1.0,
1634
- "n_examples": 50,
1635
  "per_subtask": {
1636
  "MD": {
1637
  "accuracy": 1.0,
1638
- "count": 100
1639
  },
1640
  "MB": {
1641
  "accuracy": 1.0,
1642
- "count": 50
1643
  },
1644
  "UB": {
1645
  "accuracy": 1.0,
1646
- "count": 121
1647
  },
1648
  "UD": {
1649
  "accuracy": 1.0,
1650
- "count": 79
1651
  }
1652
  }
1653
  },
1654
  "sub_B5": {
1655
- "full_accuracy": 0.98,
1656
- "digit_accuracy": 0.9971428571428571,
1657
- "n_examples": 50,
1658
  "per_subtask": {
1659
  "MD": {
1660
  "accuracy": 1.0,
1661
- "count": 50
1662
  },
1663
  "MB": {
1664
  "accuracy": 1.0,
1665
- "count": 50
1666
  },
1667
  "UB": {
1668
- "accuracy": 0.993421052631579,
1669
- "count": 152
1670
  },
1671
  "UD": {
1672
  "accuracy": 1.0,
1673
- "count": 98
1674
  }
1675
  }
1676
  }
1677
  },
1678
  "summary": {
1679
- "overall_accuracy": 0.9673333333333334,
1680
- "digit_accuracy": 0.9953333333333333,
1681
- "total_examples": 1500,
1682
  "n_splits": 24
1683
  }
1684
  },
1685
- "sorl_overall_accuracy": 0.96125,
1686
- "sft_overall_accuracy": 0.7683333333333333
1687
  }
 
63
  3133
64
  ],
65
  "loss": [
66
+ 8.310182571411133,
67
+ 4.133201599121094,
68
+ 2.8132777214050293,
69
+ 2.6971330642700195,
70
+ 2.7512383460998535,
71
+ 2.432722568511963,
72
+ 1.8309378623962402,
73
+ 1.1754705905914307,
74
+ -3.723365545272827,
75
+ -7.398290634155273,
76
+ -7.010554790496826,
77
+ -6.42205286026001,
78
+ -5.359836578369141,
79
+ -4.193981647491455,
80
+ -4.243052005767822,
81
+ -3.7155613899230957,
82
+ -2.9050772190093994,
83
+ -0.9977436065673828,
84
+ -0.83854740858078,
85
+ -0.7859895825386047,
86
+ -0.39460235834121704,
87
+ -0.8910185694694519,
88
+ -0.7928282022476196,
89
+ -0.1503787636756897,
90
+ -0.6589345932006836,
91
+ -0.35099464654922485,
92
+ -0.17189741134643555,
93
+ -0.28238868713378906,
94
+ -0.5372843742370605,
95
+ -0.45871204137802124,
96
+ -0.17290014028549194,
97
+ -0.2313457876443863,
98
+ -0.2840215265750885,
99
+ -0.5072467923164368,
100
+ -0.1789221465587616,
101
+ -0.4149681329727173,
102
+ -0.27094072103500366,
103
+ -0.3303571939468384,
104
+ -0.2898772358894348,
105
+ -0.21254831552505493,
106
+ -0.4309675097465515,
107
+ -0.16034571826457977,
108
+ -0.05542676895856857,
109
+ -0.1668514758348465,
110
+ -0.18458348512649536,
111
+ -0.044108353555202484,
112
+ -0.10001308470964432,
113
+ -0.03989148885011673,
114
+ -0.11058878898620605,
115
+ -0.03912587836384773,
116
+ 0.0665445327758789,
117
+ -0.05913455784320831,
118
+ 0.003248041495680809,
119
+ 0.005677715875208378,
120
+ 0.003825967200100422,
121
+ -0.008632136508822441,
122
+ 0.0008527245372533798,
123
+ 0.010085301473736763,
124
+ -0.02317584678530693,
125
+ 0.007569985464215279
126
  ],
127
  "base_loss": [
128
+ 7.6308159828186035,
129
+ 3.6053097248077393,
130
+ 1.895028829574585,
131
+ 1.900036096572876,
132
+ 1.9309532642364502,
133
+ 1.8586256504058838,
134
+ 1.728783369064331,
135
+ 1.764841914176941,
136
+ 1.8143653869628906,
137
+ 1.7625930309295654,
138
+ 1.6197043657302856,
139
+ 1.358901858329773,
140
+ 1.0819851160049438,
141
+ 0.791595995426178,
142
+ 0.7541593313217163,
143
+ 0.6051990389823914,
144
+ 0.4855411946773529,
145
+ 0.22208479046821594,
146
+ 0.20987896621227264,
147
+ 0.17736339569091797,
148
+ 0.12146198749542236,
149
+ 0.16632437705993652,
150
+ 0.1565568745136261,
151
+ 0.08355117589235306,
152
+ 0.12975403666496277,
153
+ 0.08682022988796234,
154
+ 0.056998688727617264,
155
+ 0.06084975227713585,
156
+ 0.08375317603349686,
157
+ 0.06962383538484573,
158
+ 0.06018713489174843,
159
+ 0.04455818608403206,
160
+ 0.04680037871003151,
161
+ 0.06599079072475433,
162
+ 0.025991132482886314,
163
+ 0.051596079021692276,
164
+ 0.03439215198159218,
165
+ 0.041100483387708664,
166
+ 0.036433007568120956,
167
+ 0.027153104543685913,
168
+ 0.05090184137225151,
169
+ 0.021161774173378944,
170
+ 0.007903954945504665,
171
+ 0.020700249820947647,
172
+ 0.022984957322478294,
173
+ 0.008006555028259754,
174
+ 0.013453503139317036,
175
+ 0.006606706418097019,
176
+ 0.013815460726618767,
177
+ 0.0056317755952477455,
178
+ 0.008193695917725563,
179
+ 0.008682571351528168,
180
+ 0.001918728114105761,
181
+ 0.0012790235923603177,
182
+ 0.001985017443075776,
183
+ 0.0020421200897544622,
184
+ 0.0012287904974073172,
185
+ 0.0005331250140443444,
186
+ 0.004157603718340397,
187
+ 0.0003490203234832734
188
  ],
189
  "info_loss": [
190
+ -0.44722652435302734,
191
+ -0.09009504318237305,
192
+ -0.0214996337890625,
193
+ -0.032089829444885254,
194
+ -0.029031038284301758,
195
+ -0.05408179759979248,
196
+ -0.10042738914489746,
197
+ -0.1698007583618164,
198
+ -0.664747953414917,
199
+ -1.0273367166519165,
200
+ -0.9735860824584961,
201
+ -0.8874133825302124,
202
+ -0.750395655632019,
203
+ -0.6002907156944275,
204
+ -0.5956957340240479,
205
+ -0.5156505703926086,
206
+ -0.412106990814209,
207
+ -0.19066482782363892,
208
+ -0.16895775496959686,
209
+ -0.15515057742595673,
210
+ -0.10569143295288086,
211
+ -0.15466587245464325,
212
+ -0.13811513781547546,
213
+ -0.06318927556276321,
214
+ -0.11460509151220322,
215
+ -0.07420219480991364,
216
+ -0.04607734829187393,
217
+ -0.054609108716249466,
218
+ -0.07958278805017471,
219
+ -0.06794232130050659,
220
+ -0.03601129353046417,
221
+ -0.03566997870802879,
222
+ -0.038879334926605225,
223
+ -0.06411565095186234,
224
+ -0.025443758815526962,
225
+ -0.050301406532526016,
226
+ -0.03383207321166992,
227
+ -0.04046107083559036,
228
+ -0.0359560027718544,
229
+ -0.02669592574238777,
230
+ -0.05045830458402634,
231
+ -0.0207816269248724,
232
+ -0.007726653013378382,
233
+ -0.020415037870407104,
234
+ -0.02282050997018814,
235
+ -0.007889396511018276,
236
+ -0.013376262970268726,
237
+ -0.006515223998576403,
238
+ -0.013753198087215424,
239
+ -0.005556280724704266,
240
+ 0.0042526256293058395,
241
+ -0.008607309311628342,
242
+ -0.001882747863419354,
243
+ -0.0012353716883808374,
244
+ -0.0019492279971018434,
245
+ -0.0020032450556755066,
246
+ -0.0011978205293416977,
247
+ -0.0005038329400122166,
248
+ -0.004128722008317709,
249
+ -0.00031519742333330214
250
  ],
251
  "abs_loss": [
252
+ 3.3277130126953125,
253
+ 2.7578437328338623,
254
+ 2.754722833633423,
255
+ 2.7363626956939697,
256
+ 2.6784064769744873,
257
+ 2.7312171459198,
258
+ 2.702913999557495,
259
+ 2.7030067443847656,
260
+ 2.714343309402466,
261
+ 2.718477487564087,
262
+ 2.6630165576934814,
263
+ 2.6015138626098633,
264
+ 2.487117052078247,
265
+ 2.3467166423797607,
266
+ 2.2234556674957275,
267
+ 2.0171573162078857,
268
+ 1.5574852228164673,
269
+ 1.4047346115112305,
270
+ 1.3112843036651611,
271
+ 1.1558475494384766,
272
+ 1.0216270685195923,
273
+ 0.8954780101776123,
274
+ 0.8539987206459045,
275
+ 0.7943210601806641,
276
+ 0.6508080363273621,
277
+ 0.7308263182640076,
278
+ 0.6387749314308167,
279
+ 0.4604922831058502,
280
+ 0.4742761552333832,
281
+ 0.42960110306739807,
282
+ 0.3602112829685211,
283
+ 0.29546570777893066,
284
+ 0.2083757370710373,
285
+ 0.17783701419830322,
286
+ 0.17184175550937653,
287
+ 0.15632905066013336,
288
+ 0.10660465806722641,
289
+ 0.14435525238513947,
290
+ 0.11503338813781738,
291
+ 0.09272787719964981,
292
+ 0.0732387825846672,
293
+ 0.06837724149227142,
294
+ 0.06349286437034607,
295
+ 0.06759530305862427,
296
+ 0.05148301646113396,
297
+ 0.03341341018676758,
298
+ 0.05618534982204437,
299
+ 0.040469493716955185,
300
+ 0.037190549075603485,
301
+ 0.024392684921622276,
302
+ 0.024227822199463844,
303
+ 0.02828081138432026,
304
+ 0.026742493733763695,
305
+ 0.026506895199418068,
306
+ 0.01890699751675129,
307
+ 0.022441355511546135,
308
+ 0.013721623457968235,
309
+ 0.01978396065533161,
310
+ 0.016148246824741364,
311
+ 0.017677132040262222
312
  ],
313
  "zipf_loss": [
314
+ 4.81886100769043,
315
+ 1.1530582904815674,
316
+ 0.8577730059623718,
317
+ 0.8443589210510254,
318
+ 0.8427549600601196,
319
+ 0.8417932987213135,
320
+ 0.8361369371414185,
321
+ 0.8383356332778931,
322
+ 0.8383142352104187,
323
+ 0.8406355381011963,
324
+ 0.839300274848938,
325
+ 0.833027720451355,
326
+ 0.8134238719940186,
327
+ 0.7826582193374634,
328
+ 0.7374005317687988,
329
+ 0.6340293884277344,
330
+ 0.5747029781341553,
331
+ 0.5463464260101318,
332
+ 0.5100228190422058,
333
+ 0.4725680947303772,
334
+ 0.438687264919281,
335
+ 0.39976805448532104,
336
+ 0.3463665246963501,
337
+ 0.31853070855140686,
338
+ 0.29228144884109497,
339
+ 0.2311244010925293,
340
+ 0.16799986362457275,
341
+ 0.1568034142255783,
342
+ 0.1273626983165741,
343
+ 0.10812723636627197,
344
+ 0.09100455790758133,
345
+ 0.05124925076961517,
346
+ 0.03713386133313179,
347
+ 0.05013521760702133,
348
+ 0.032340142875909805,
349
+ 0.02081698551774025,
350
+ 0.022327393293380737,
351
+ 0.018717491999268532,
352
+ 0.021746426820755005,
353
+ 0.017985042184591293,
354
+ 0.015389842912554741,
355
+ 0.019471045583486557,
356
+ 0.007586517836898565,
357
+ 0.009839128702878952,
358
+ 0.015488353557884693,
359
+ 0.02343771606683731,
360
+ 0.014677501283586025,
361
+ 0.014607097953557968,
362
+ 0.009408676065504551,
363
+ 0.008365889079868793,
364
+ 0.013401798903942108,
365
+ 0.015427881851792336,
366
+ 0.017482543364167213,
367
+ 0.014101719483733177,
368
+ 0.01944253034889698,
369
+ 0.00711405836045742,
370
+ 0.01022997684776783,
371
+ 0.012612110003829002,
372
+ 0.012338947504758835,
373
+ 0.008605225943028927
374
  ],
375
  "denoise_loss": [],
376
  "ortho_loss": [
377
+ 0.3160897493362427,
378
+ 0.08690287917852402,
379
+ 0.08605827391147614,
380
+ 0.06739893555641174,
381
+ 0.07452061027288437,
382
+ 0.07991696149110794,
383
+ 0.08710527420043945,
384
+ 0.11880933493375778,
385
+ 0.15567420423030853,
386
+ 0.1762562394142151,
387
+ 0.17974773049354553,
388
+ 0.18837907910346985,
389
+ 0.20129692554473877,
390
+ 0.2209535837173462,
391
+ 0.23735277354717255,
392
+ 0.24846896529197693,
393
+ 0.2586727738380432,
394
+ 0.2635071575641632,
395
+ 0.2690512239933014,
396
+ 0.2682317793369293,
397
+ 0.2739230692386627,
398
+ 0.28098905086517334,
399
+ 0.28371700644493103,
400
+ 0.28486451506614685,
401
+ 0.2812826335430145,
402
+ 0.2855672538280487,
403
+ 0.28155791759490967,
404
+ 0.28803756833076477,
405
+ 0.286459743976593,
406
+ 0.2784627676010132,
407
+ 0.27852895855903625,
408
+ 0.27721235156059265,
409
+ 0.28099218010902405,
410
+ 0.2766082286834717,
411
+ 0.2736574113368988,
412
+ 0.26976510882377625,
413
+ 0.264602929353714,
414
+ 0.25994396209716797,
415
+ 0.2587146759033203,
416
+ 0.24565429985523224,
417
+ 0.24309058487415314,
418
+ 0.24578610062599182,
419
+ 0.2516838312149048,
420
+ 0.24798940122127533,
421
+ 0.2389592081308365,
422
+ 0.24198728799819946,
423
+ 0.2408341020345688,
424
+ 0.240533247590065,
425
+ 0.23796656727790833,
426
+ 0.2359270453453064,
427
+ 0.23633089661598206,
428
+ 0.23868851363658905,
429
+ 0.23841753602027893,
430
+ 0.240677610039711,
431
+ 0.2402603179216385,
432
+ 0.23987752199172974,
433
+ 0.23913918435573578,
434
+ 0.2392318993806839,
435
+ 0.2396986186504364,
436
+ 0.23982712626457214
437
  ],
438
  "lr": [
439
  3.9200000000000004e-05,
 
521
  3083
522
  ],
523
  "eval_accuracy": [
524
+ 0.02,
525
+ 0.02,
526
  0.01,
527
  0.0,
528
  0.0,
 
540
  0.0,
541
  0.0,
542
  0.0,
 
 
543
  0.0
544
  ]
545
  },
546
+ "final_accuracy": 0.9738461538461538,
547
  "sft_eval": {
548
  "config": {
549
  "ops": "add_sub",
550
  "K": null,
551
  "mode": "sft",
552
  "n_digits": 6,
553
+ "n_per_split": 100
554
  },
555
  "splits": {
556
  "add_S0": {
557
+ "full_accuracy": 1.0,
558
+ "digit_accuracy": 1.0,
559
+ "n_examples": 100,
560
  "per_subtask": {
561
  "SA": {
562
+ "accuracy": 1.0,
563
+ "count": 605
564
  },
565
  "SS": {
566
  "accuracy": 1.0,
567
+ "count": 95
568
  }
569
  }
570
  },
571
  "add_S1": {
572
  "full_accuracy": 1.0,
573
  "digit_accuracy": 1.0,
574
+ "n_examples": 100,
575
  "per_subtask": {
576
  "SA": {
577
  "accuracy": 1.0,
578
+ "count": 204
579
  },
580
  "SC": {
581
  "accuracy": 1.0,
582
+ "count": 169
583
  },
584
  "SS": {
585
  "accuracy": 1.0,
586
+ "count": 31
587
  },
588
  "UC": {
589
  "accuracy": 1.0,
590
+ "count": 296
591
  }
592
  }
593
  },
594
  "add_S2": {
595
+ "full_accuracy": 1.0,
596
+ "digit_accuracy": 1.0,
597
+ "n_examples": 100,
598
  "per_subtask": {
599
  "SA": {
600
  "accuracy": 1.0,
601
+ "count": 163
602
  },
603
  "SC": {
604
+ "accuracy": 1.0,
605
+ "count": 130
606
  },
607
  "SS": {
608
  "accuracy": 1.0,
609
+ "count": 87
610
  },
611
  "UC": {
612
  "accuracy": 1.0,
613
+ "count": 203
614
  },
615
  "US": {
616
  "accuracy": 1.0,
617
+ "count": 117
618
  }
619
  }
620
  },
621
  "add_S3": {
622
+ "full_accuracy": 1.0,
623
+ "digit_accuracy": 1.0,
624
+ "n_examples": 100,
625
  "per_subtask": {
626
  "SA": {
627
  "accuracy": 1.0,
628
+ "count": 121
629
  },
630
  "SC": {
631
+ "accuracy": 1.0,
632
+ "count": 121
633
  },
634
  "SS": {
635
  "accuracy": 1.0,
636
+ "count": 49
637
  },
638
  "UC": {
639
+ "accuracy": 1.0,
640
+ "count": 186
641
  },
642
  "US": {
643
  "accuracy": 1.0,
644
+ "count": 223
645
  }
646
  }
647
  },
648
  "add_S4": {
649
+ "full_accuracy": 0.87,
650
+ "digit_accuracy": 0.9814285714285714,
651
+ "n_examples": 100,
652
  "per_subtask": {
653
  "SA": {
654
  "accuracy": 1.0,
655
+ "count": 104
656
  },
657
  "SC": {
658
  "accuracy": 1.0,
659
+ "count": 106
660
  },
661
  "SS": {
662
  "accuracy": 1.0,
663
+ "count": 23
664
  },
665
  "UC": {
666
+ "accuracy": 0.91875,
667
+ "count": 160
668
  },
669
  "US": {
670
+ "accuracy": 1.0,
671
+ "count": 307
672
  }
673
  }
674
  },
675
  "add_S5": {
676
+ "full_accuracy": 0.76,
677
+ "digit_accuracy": 0.9657142857142857,
678
+ "n_examples": 100,
679
  "per_subtask": {
680
  "SA": {
681
  "accuracy": 1.0,
682
+ "count": 100
683
  },
684
  "SC": {
685
  "accuracy": 1.0,
686
+ "count": 100
687
  },
688
  "UC": {
689
+ "accuracy": 0.83,
690
+ "count": 100
691
  },
692
  "US": {
693
+ "accuracy": 0.9825,
694
+ "count": 400
695
  }
696
  }
697
  },
698
  "add_S6": {
699
+ "full_accuracy": 0.92,
700
+ "digit_accuracy": 0.9828571428571429,
701
+ "n_examples": 100,
702
  "per_subtask": {
703
  "SC": {
704
  "accuracy": 1.0,
705
+ "count": 100
706
  },
707
  "UC": {
708
+ "accuracy": 0.92,
709
+ "count": 100
710
  },
711
  "US": {
712
+ "accuracy": 0.992,
713
+ "count": 500
714
  }
715
  }
716
  },
717
  "add_random": {
718
+ "full_accuracy": 1.0,
719
+ "digit_accuracy": 1.0,
720
  "n_examples": 200,
721
  "per_subtask": {
722
  "SA": {
723
  "accuracy": 1.0,
724
+ "count": 447
725
  },
726
  "SC": {
727
  "accuracy": 1.0,
728
+ "count": 320
729
  },
730
  "SS": {
731
  "accuracy": 1.0,
732
+ "count": 56
733
  },
734
  "UC": {
735
+ "accuracy": 1.0,
736
+ "count": 529
737
  },
738
  "US": {
739
  "accuracy": 1.0,
740
+ "count": 48
741
  }
742
  }
743
  },
744
  "add_C1": {
745
+ "full_accuracy": 1.0,
746
+ "digit_accuracy": 1.0,
747
+ "n_examples": 100,
748
  "per_subtask": {
749
  "SA": {
750
+ "accuracy": 1.0,
751
+ "count": 500
752
  },
753
  "SC": {
754
  "accuracy": 1.0,
755
+ "count": 100
756
  },
757
  "UC": {
758
  "accuracy": 1.0,
759
+ "count": 100
760
  }
761
  }
762
  },
763
  "add_C2": {
764
  "full_accuracy": 1.0,
765
  "digit_accuracy": 1.0,
766
+ "n_examples": 100,
767
  "per_subtask": {
768
  "SA": {
769
  "accuracy": 1.0,
770
+ "count": 400
771
  },
772
  "SC": {
773
  "accuracy": 1.0,
774
+ "count": 100
775
  },
776
  "UC": {
777
  "accuracy": 1.0,
778
+ "count": 156
779
  },
780
  "US": {
781
  "accuracy": 1.0,
782
+ "count": 44
783
  }
784
  }
785
  },
786
  "add_C3": {
787
+ "full_accuracy": 0.98,
788
+ "digit_accuracy": 0.9971428571428571,
789
+ "n_examples": 100,
790
  "per_subtask": {
791
  "SA": {
792
+ "accuracy": 1.0,
793
+ "count": 300
794
  },
795
  "SC": {
796
  "accuracy": 1.0,
797
+ "count": 100
798
  },
799
  "UC": {
800
+ "accuracy": 0.9899497487437185,
801
+ "count": 199
802
  },
803
  "US": {
804
  "accuracy": 1.0,
805
+ "count": 101
806
  }
807
  }
808
  },
809
  "add_C4": {
810
+ "full_accuracy": 0.97,
811
+ "digit_accuracy": 0.9957142857142857,
812
+ "n_examples": 100,
813
  "per_subtask": {
814
  "SA": {
815
  "accuracy": 1.0,
816
+ "count": 200
817
  },
818
  "SC": {
819
  "accuracy": 1.0,
820
+ "count": 100
821
  },
822
  "UC": {
823
+ "accuracy": 0.9886363636363636,
824
+ "count": 264
825
  },
826
  "US": {
827
+ "accuracy": 1.0,
828
+ "count": 136
829
  }
830
  }
831
  },
832
  "add_C5": {
833
+ "full_accuracy": 0.98,
834
+ "digit_accuracy": 0.9971428571428571,
835
+ "n_examples": 100,
836
  "per_subtask": {
837
  "SA": {
838
  "accuracy": 1.0,
839
+ "count": 100
840
  },
841
  "SC": {
842
  "accuracy": 1.0,
843
+ "count": 100
844
  },
845
  "UC": {
846
+ "accuracy": 0.9935483870967742,
847
+ "count": 310
848
  },
849
  "US": {
850
+ "accuracy": 1.0,
851
+ "count": 190
852
  }
853
  }
854
  },
855
  "add_C6": {
856
+ "full_accuracy": 1.0,
857
+ "digit_accuracy": 1.0,
858
+ "n_examples": 100,
859
  "per_subtask": {
860
  "SC": {
861
  "accuracy": 1.0,
862
+ "count": 100
863
  },
864
  "UC": {
865
+ "accuracy": 1.0,
866
+ "count": 370
867
  },
868
  "US": {
869
  "accuracy": 1.0,
870
+ "count": 230
871
  }
872
  }
873
  },
874
  "sub_M0": {
875
  "full_accuracy": 1.0,
876
  "digit_accuracy": 1.0,
877
+ "n_examples": 100,
878
  "per_subtask": {
879
  "MD": {
880
  "accuracy": 1.0,
881
+ "count": 615
882
  },
883
  "ME": {
884
  "accuracy": 1.0,
885
+ "count": 85
886
  }
887
  }
888
  },
889
  "sub_M1": {
890
+ "full_accuracy": 0.99,
891
+ "digit_accuracy": 0.9985714285714286,
892
+ "n_examples": 100,
893
  "per_subtask": {
894
  "MD": {
895
  "accuracy": 1.0,
896
+ "count": 292
897
  },
898
  "MB": {
899
+ "accuracy": 0.9930555555555556,
900
+ "count": 144
901
  },
902
  "ME": {
903
  "accuracy": 1.0,
904
+ "count": 25
905
  },
906
  "UB": {
907
  "accuracy": 1.0,
908
+ "count": 239
909
  }
910
  }
911
  },
912
  "sub_M2": {
913
  "full_accuracy": 1.0,
914
  "digit_accuracy": 1.0,
915
+ "n_examples": 100,
916
  "per_subtask": {
917
  "MD": {
918
  "accuracy": 1.0,
919
+ "count": 211
920
  },
921
  "MB": {
922
  "accuracy": 1.0,
923
+ "count": 115
924
  },
925
  "ME": {
926
  "accuracy": 1.0,
927
+ "count": 85
928
  },
929
  "UB": {
930
  "accuracy": 1.0,
931
+ "count": 181
932
  },
933
  "UD": {
934
  "accuracy": 1.0,
935
+ "count": 108
936
  }
937
  }
938
  },
939
  "sub_M3": {
940
+ "full_accuracy": 1.0,
941
+ "digit_accuracy": 1.0,
942
+ "n_examples": 100,
943
  "per_subtask": {
944
  "MD": {
945
  "accuracy": 1.0,
946
+ "count": 179
947
  },
948
  "MB": {
949
  "accuracy": 1.0,
950
+ "count": 103
951
  },
952
  "ME": {
953
  "accuracy": 1.0,
954
+ "count": 56
955
  },
956
  "UB": {
957
+ "accuracy": 1.0,
958
+ "count": 149
959
  },
960
  "UD": {
961
  "accuracy": 1.0,
962
+ "count": 213
963
  }
964
  }
965
  },
966
  "sub_M4": {
967
+ "full_accuracy": 0.24,
968
+ "digit_accuracy": 0.89,
969
+ "n_examples": 100,
970
  "per_subtask": {
971
  "MD": {
972
  "accuracy": 1.0,
973
+ "count": 200
974
  },
975
  "MB": {
976
  "accuracy": 1.0,
977
+ "count": 100
978
  },
979
  "UB": {
980
+ "accuracy": 0.24,
981
+ "count": 100
982
  },
983
  "UD": {
984
+ "accuracy": 0.9966666666666667,
985
+ "count": 300
986
  }
987
  }
988
  },
989
  "sub_M5": {
990
+ "full_accuracy": 0.56,
991
+ "digit_accuracy": 0.9271428571428572,
992
+ "n_examples": 100,
993
  "per_subtask": {
994
  "MD": {
995
  "accuracy": 1.0,
996
+ "count": 100
997
  },
998
  "MB": {
999
  "accuracy": 1.0,
1000
+ "count": 100
1001
  },
1002
  "UB": {
1003
+ "accuracy": 0.85,
1004
+ "count": 100
1005
  },
1006
  "UD": {
1007
+ "accuracy": 0.91,
1008
+ "count": 400
1009
  }
1010
  }
1011
  },
 
1016
  "per_subtask": {
1017
  "MD": {
1018
  "accuracy": 1.0,
1019
+ "count": 600
1020
  },
1021
  "MB": {
1022
  "accuracy": 1.0,
1023
+ "count": 267
1024
  },
1025
  "ME": {
1026
  "accuracy": 1.0,
 
1028
  },
1029
  "UB": {
1030
  "accuracy": 1.0,
1031
+ "count": 439
1032
  },
1033
  "UD": {
1034
  "accuracy": 1.0,
1035
+ "count": 41
1036
  }
1037
  }
1038
  },
1039
  "sub_B3": {
1040
+ "full_accuracy": 1.0,
1041
+ "digit_accuracy": 1.0,
1042
+ "n_examples": 100,
1043
  "per_subtask": {
1044
  "MD": {
1045
  "accuracy": 1.0,
1046
+ "count": 300
1047
  },
1048
  "MB": {
1049
  "accuracy": 1.0,
1050
+ "count": 100
1051
  },
1052
  "UB": {
1053
+ "accuracy": 1.0,
1054
+ "count": 197
1055
  },
1056
  "UD": {
1057
  "accuracy": 1.0,
1058
+ "count": 103
1059
  }
1060
  }
1061
  },
1062
  "sub_B4": {
1063
+ "full_accuracy": 0.88,
1064
+ "digit_accuracy": 0.9828571428571429,
1065
+ "n_examples": 100,
1066
  "per_subtask": {
1067
  "MD": {
1068
  "accuracy": 1.0,
1069
+ "count": 200
1070
  },
1071
  "MB": {
1072
  "accuracy": 1.0,
1073
+ "count": 100
1074
  },
1075
  "UB": {
1076
+ "accuracy": 0.951417004048583,
1077
+ "count": 247
1078
  },
1079
  "UD": {
1080
+ "accuracy": 1.0,
1081
+ "count": 153
1082
  }
1083
  }
1084
  },
1085
  "sub_B5": {
1086
+ "full_accuracy": 0.97,
1087
+ "digit_accuracy": 0.9957142857142857,
1088
+ "n_examples": 100,
1089
  "per_subtask": {
1090
  "MD": {
1091
  "accuracy": 1.0,
1092
+ "count": 100
1093
  },
1094
  "MB": {
1095
  "accuracy": 1.0,
1096
+ "count": 100
1097
  },
1098
  "UB": {
1099
+ "accuracy": 0.9966442953020134,
1100
+ "count": 298
1101
  },
1102
  "UD": {
1103
+ "accuracy": 0.9900990099009901,
1104
+ "count": 202
1105
  }
1106
  }
1107
  }
1108
  },
1109
  "summary": {
1110
+ "overall_accuracy": 0.9276923076923077,
1111
+ "digit_accuracy": 0.989010989010989,
1112
+ "total_examples": 2600,
1113
  "n_splits": 24
1114
  }
1115
  },
 
1119
  "K": 1,
1120
  "mode": "sorl",
1121
  "n_digits": 6,
1122
+ "n_per_split": 100
1123
  },
1124
  "splits": {
1125
  "add_S0": {
1126
  "full_accuracy": 1.0,
1127
  "digit_accuracy": 1.0,
1128
+ "n_examples": 100,
1129
  "per_subtask": {
1130
  "SA": {
1131
  "accuracy": 1.0,
1132
+ "count": 605
1133
  },
1134
  "SS": {
1135
  "accuracy": 1.0,
1136
+ "count": 95
1137
  }
1138
  }
1139
  },
1140
  "add_S1": {
1141
  "full_accuracy": 1.0,
1142
  "digit_accuracy": 1.0,
1143
+ "n_examples": 100,
1144
  "per_subtask": {
1145
  "SA": {
1146
  "accuracy": 1.0,
1147
+ "count": 204
1148
  },
1149
  "SC": {
1150
  "accuracy": 1.0,
1151
+ "count": 169
1152
  },
1153
  "SS": {
1154
  "accuracy": 1.0,
1155
+ "count": 31
1156
  },
1157
  "UC": {
1158
  "accuracy": 1.0,
1159
+ "count": 296
1160
  }
1161
  }
1162
  },
1163
  "add_S2": {
1164
  "full_accuracy": 1.0,
1165
  "digit_accuracy": 1.0,
1166
+ "n_examples": 100,
1167
  "per_subtask": {
1168
  "SA": {
1169
  "accuracy": 1.0,
1170
+ "count": 163
1171
  },
1172
  "SC": {
1173
  "accuracy": 1.0,
1174
+ "count": 130
1175
  },
1176
  "SS": {
1177
  "accuracy": 1.0,
1178
+ "count": 87
1179
  },
1180
  "UC": {
1181
  "accuracy": 1.0,
1182
+ "count": 203
1183
  },
1184
  "US": {
1185
  "accuracy": 1.0,
1186
+ "count": 117
1187
  }
1188
  }
1189
  },
1190
  "add_S3": {
1191
  "full_accuracy": 1.0,
1192
  "digit_accuracy": 1.0,
1193
+ "n_examples": 100,
1194
  "per_subtask": {
1195
  "SA": {
1196
  "accuracy": 1.0,
1197
+ "count": 121
1198
  },
1199
  "SC": {
1200
  "accuracy": 1.0,
1201
+ "count": 121
1202
  },
1203
  "SS": {
1204
  "accuracy": 1.0,
1205
+ "count": 49
1206
  },
1207
  "UC": {
1208
  "accuracy": 1.0,
1209
+ "count": 186
1210
  },
1211
  "US": {
1212
  "accuracy": 1.0,
1213
+ "count": 223
1214
  }
1215
  }
1216
  },
1217
  "add_S4": {
1218
  "full_accuracy": 1.0,
1219
  "digit_accuracy": 1.0,
1220
+ "n_examples": 100,
1221
  "per_subtask": {
1222
  "SA": {
1223
  "accuracy": 1.0,
1224
+ "count": 104
1225
  },
1226
  "SC": {
1227
  "accuracy": 1.0,
1228
+ "count": 106
1229
  },
1230
  "SS": {
1231
  "accuracy": 1.0,
1232
+ "count": 23
1233
  },
1234
  "UC": {
1235
  "accuracy": 1.0,
1236
+ "count": 160
1237
  },
1238
  "US": {
1239
  "accuracy": 1.0,
1240
+ "count": 307
1241
  }
1242
  }
1243
  },
1244
  "add_S5": {
1245
+ "full_accuracy": 0.7,
1246
+ "digit_accuracy": 0.9571428571428572,
1247
+ "n_examples": 100,
1248
  "per_subtask": {
1249
  "SA": {
1250
  "accuracy": 1.0,
1251
+ "count": 100
1252
  },
1253
  "SC": {
1254
  "accuracy": 1.0,
1255
+ "count": 100
1256
  },
1257
  "UC": {
1258
+ "accuracy": 0.7,
1259
+ "count": 100
1260
  },
1261
  "US": {
1262
  "accuracy": 1.0,
1263
+ "count": 400
1264
  }
1265
  }
1266
  },
1267
  "add_S6": {
1268
+ "full_accuracy": 0.98,
1269
+ "digit_accuracy": 0.9971428571428571,
1270
+ "n_examples": 100,
1271
  "per_subtask": {
1272
  "SC": {
1273
  "accuracy": 1.0,
1274
+ "count": 100
1275
  },
1276
  "UC": {
1277
  "accuracy": 1.0,
1278
+ "count": 100
1279
  },
1280
  "US": {
1281
+ "accuracy": 0.996,
1282
+ "count": 500
1283
  }
1284
  }
1285
  },
 
1290
  "per_subtask": {
1291
  "SA": {
1292
  "accuracy": 1.0,
1293
+ "count": 447
1294
  },
1295
  "SC": {
1296
  "accuracy": 1.0,
1297
+ "count": 320
1298
  },
1299
  "SS": {
1300
  "accuracy": 1.0,
1301
+ "count": 56
1302
  },
1303
  "UC": {
1304
  "accuracy": 1.0,
1305
+ "count": 529
1306
  },
1307
  "US": {
1308
  "accuracy": 1.0,
1309
+ "count": 48
1310
  }
1311
  }
1312
  },
1313
  "add_C1": {
1314
  "full_accuracy": 1.0,
1315
  "digit_accuracy": 1.0,
1316
+ "n_examples": 100,
1317
  "per_subtask": {
1318
  "SA": {
1319
  "accuracy": 1.0,
1320
+ "count": 500
1321
  },
1322
  "SC": {
1323
  "accuracy": 1.0,
1324
+ "count": 100
1325
  },
1326
  "UC": {
1327
  "accuracy": 1.0,
1328
+ "count": 100
1329
  }
1330
  }
1331
  },
1332
  "add_C2": {
1333
  "full_accuracy": 1.0,
1334
  "digit_accuracy": 1.0,
1335
+ "n_examples": 100,
1336
  "per_subtask": {
1337
  "SA": {
1338
  "accuracy": 1.0,
1339
+ "count": 400
1340
  },
1341
  "SC": {
1342
  "accuracy": 1.0,
1343
+ "count": 100
1344
  },
1345
  "UC": {
1346
  "accuracy": 1.0,
1347
+ "count": 156
1348
  },
1349
  "US": {
1350
  "accuracy": 1.0,
1351
+ "count": 44
1352
  }
1353
  }
1354
  },
1355
  "add_C3": {
1356
  "full_accuracy": 1.0,
1357
  "digit_accuracy": 1.0,
1358
+ "n_examples": 100,
1359
  "per_subtask": {
1360
  "SA": {
1361
  "accuracy": 1.0,
1362
+ "count": 300
1363
  },
1364
  "SC": {
1365
  "accuracy": 1.0,
1366
+ "count": 100
1367
  },
1368
  "UC": {
1369
  "accuracy": 1.0,
1370
+ "count": 199
1371
  },
1372
  "US": {
1373
  "accuracy": 1.0,
1374
+ "count": 101
1375
  }
1376
  }
1377
  },
1378
  "add_C4": {
1379
  "full_accuracy": 1.0,
1380
  "digit_accuracy": 1.0,
1381
+ "n_examples": 100,
1382
  "per_subtask": {
1383
  "SA": {
1384
  "accuracy": 1.0,
1385
+ "count": 200
1386
  },
1387
  "SC": {
1388
  "accuracy": 1.0,
1389
+ "count": 100
1390
  },
1391
  "UC": {
1392
  "accuracy": 1.0,
1393
+ "count": 264
1394
  },
1395
  "US": {
1396
  "accuracy": 1.0,
1397
+ "count": 136
1398
  }
1399
  }
1400
  },
1401
  "add_C5": {
1402
+ "full_accuracy": 0.99,
1403
+ "digit_accuracy": 0.9985714285714286,
1404
+ "n_examples": 100,
1405
  "per_subtask": {
1406
  "SA": {
1407
  "accuracy": 1.0,
1408
+ "count": 100
1409
  },
1410
  "SC": {
1411
  "accuracy": 1.0,
1412
+ "count": 100
1413
  },
1414
  "UC": {
1415
+ "accuracy": 0.9967741935483871,
1416
+ "count": 310
1417
  },
1418
  "US": {
1419
  "accuracy": 1.0,
1420
+ "count": 190
1421
  }
1422
  }
1423
  },
1424
  "add_C6": {
1425
+ "full_accuracy": 0.97,
1426
+ "digit_accuracy": 0.9957142857142857,
1427
+ "n_examples": 100,
1428
  "per_subtask": {
1429
  "SC": {
1430
  "accuracy": 1.0,
1431
+ "count": 100
1432
  },
1433
  "UC": {
1434
+ "accuracy": 0.9918918918918919,
1435
+ "count": 370
1436
  },
1437
  "US": {
1438
  "accuracy": 1.0,
1439
+ "count": 230
1440
  }
1441
  }
1442
  },
1443
  "sub_M0": {
1444
  "full_accuracy": 1.0,
1445
  "digit_accuracy": 1.0,
1446
+ "n_examples": 100,
1447
  "per_subtask": {
1448
  "MD": {
1449
  "accuracy": 1.0,
1450
+ "count": 615
1451
  },
1452
  "ME": {
1453
  "accuracy": 1.0,
1454
+ "count": 85
1455
  }
1456
  }
1457
  },
1458
  "sub_M1": {
1459
  "full_accuracy": 1.0,
1460
  "digit_accuracy": 1.0,
1461
+ "n_examples": 100,
1462
  "per_subtask": {
1463
  "MD": {
1464
  "accuracy": 1.0,
1465
+ "count": 292
1466
  },
1467
  "MB": {
1468
  "accuracy": 1.0,
1469
+ "count": 144
1470
  },
1471
  "ME": {
1472
  "accuracy": 1.0,
1473
+ "count": 25
1474
  },
1475
  "UB": {
1476
  "accuracy": 1.0,
1477
+ "count": 239
1478
  }
1479
  }
1480
  },
1481
  "sub_M2": {
1482
  "full_accuracy": 1.0,
1483
  "digit_accuracy": 1.0,
1484
+ "n_examples": 100,
1485
  "per_subtask": {
1486
  "MD": {
1487
  "accuracy": 1.0,
1488
+ "count": 211
1489
  },
1490
  "MB": {
1491
  "accuracy": 1.0,
1492
+ "count": 115
1493
  },
1494
  "ME": {
1495
  "accuracy": 1.0,
1496
+ "count": 85
1497
  },
1498
  "UB": {
1499
  "accuracy": 1.0,
1500
+ "count": 181
1501
  },
1502
  "UD": {
1503
  "accuracy": 1.0,
1504
+ "count": 108
1505
  }
1506
  }
1507
  },
1508
  "sub_M3": {
1509
  "full_accuracy": 1.0,
1510
  "digit_accuracy": 1.0,
1511
+ "n_examples": 100,
1512
  "per_subtask": {
1513
  "MD": {
1514
  "accuracy": 1.0,
1515
+ "count": 179
1516
  },
1517
  "MB": {
1518
  "accuracy": 1.0,
1519
+ "count": 103
1520
  },
1521
  "ME": {
1522
  "accuracy": 1.0,
1523
+ "count": 56
1524
  },
1525
  "UB": {
1526
  "accuracy": 1.0,
1527
+ "count": 149
1528
  },
1529
  "UD": {
1530
  "accuracy": 1.0,
1531
+ "count": 213
1532
  }
1533
  }
1534
  },
1535
  "sub_M4": {
1536
  "full_accuracy": 1.0,
1537
  "digit_accuracy": 1.0,
1538
+ "n_examples": 100,
1539
  "per_subtask": {
1540
  "MD": {
1541
  "accuracy": 1.0,
1542
+ "count": 200
1543
  },
1544
  "MB": {
1545
  "accuracy": 1.0,
1546
+ "count": 100
1547
  },
1548
  "UB": {
1549
  "accuracy": 1.0,
1550
+ "count": 100
1551
  },
1552
  "UD": {
1553
  "accuracy": 1.0,
1554
+ "count": 300
1555
  }
1556
  }
1557
  },
1558
  "sub_M5": {
1559
+ "full_accuracy": 0.68,
1560
+ "digit_accuracy": 0.9542857142857143,
1561
+ "n_examples": 100,
1562
  "per_subtask": {
1563
  "MD": {
1564
  "accuracy": 1.0,
1565
+ "count": 100
1566
  },
1567
  "MB": {
1568
  "accuracy": 1.0,
1569
+ "count": 100
1570
  },
1571
  "UB": {
1572
+ "accuracy": 0.68,
1573
+ "count": 100
1574
  },
1575
  "UD": {
1576
  "accuracy": 1.0,
1577
+ "count": 400
1578
  }
1579
  }
1580
  },
 
1585
  "per_subtask": {
1586
  "MD": {
1587
  "accuracy": 1.0,
1588
+ "count": 600
1589
  },
1590
  "MB": {
1591
  "accuracy": 1.0,
1592
+ "count": 267
1593
  },
1594
  "ME": {
1595
  "accuracy": 1.0,
 
1597
  },
1598
  "UB": {
1599
  "accuracy": 1.0,
1600
+ "count": 439
1601
  },
1602
  "UD": {
1603
  "accuracy": 1.0,
1604
+ "count": 41
1605
  }
1606
  }
1607
  },
1608
  "sub_B3": {
1609
  "full_accuracy": 1.0,
1610
  "digit_accuracy": 1.0,
1611
+ "n_examples": 100,
1612
  "per_subtask": {
1613
  "MD": {
1614
  "accuracy": 1.0,
1615
+ "count": 300
1616
  },
1617
  "MB": {
1618
  "accuracy": 1.0,
1619
+ "count": 100
1620
  },
1621
  "UB": {
1622
  "accuracy": 1.0,
1623
+ "count": 197
1624
  },
1625
  "UD": {
1626
  "accuracy": 1.0,
1627
+ "count": 103
1628
  }
1629
  }
1630
  },
1631
  "sub_B4": {
1632
  "full_accuracy": 1.0,
1633
  "digit_accuracy": 1.0,
1634
+ "n_examples": 100,
1635
  "per_subtask": {
1636
  "MD": {
1637
  "accuracy": 1.0,
1638
+ "count": 200
1639
  },
1640
  "MB": {
1641
  "accuracy": 1.0,
1642
+ "count": 100
1643
  },
1644
  "UB": {
1645
  "accuracy": 1.0,
1646
+ "count": 247
1647
  },
1648
  "UD": {
1649
  "accuracy": 1.0,
1650
+ "count": 153
1651
  }
1652
  }
1653
  },
1654
  "sub_B5": {
1655
+ "full_accuracy": 1.0,
1656
+ "digit_accuracy": 1.0,
1657
+ "n_examples": 100,
1658
  "per_subtask": {
1659
  "MD": {
1660
  "accuracy": 1.0,
1661
+ "count": 100
1662
  },
1663
  "MB": {
1664
  "accuracy": 1.0,
1665
+ "count": 100
1666
  },
1667
  "UB": {
1668
+ "accuracy": 1.0,
1669
+ "count": 298
1670
  },
1671
  "UD": {
1672
  "accuracy": 1.0,
1673
+ "count": 202
1674
  }
1675
  }
1676
  }
1677
  },
1678
  "summary": {
1679
+ "overall_accuracy": 0.9738461538461538,
1680
+ "digit_accuracy": 0.9962637362637362,
1681
+ "total_examples": 2600,
1682
  "n_splits": 24
1683
  }
1684
  },
1685
+ "sorl_overall_accuracy": 0.9738461538461538,
1686
+ "sft_overall_accuracy": 0.9276923076923077
1687
  }
add_sub_sorl_v1_abs30_K1_10K/model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d36997c56cc0957d831d187e34abff36823520d50c5fbaaa3cdca415484c9628
3
  size 650385300
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a4a7a8e2c5ce0e55740188c0559e29b3f3764f48bd06b2cc510392bcbc6f8749
3
  size 650385300
add_sub_sorl_v1_abs30_K1_10K/train_config.json CHANGED
@@ -36,7 +36,7 @@
36
  "eval_every": 156,
37
  "save_every": 999999,
38
  "eval_samples": 100,
39
- "output_dir": "ckpt/sweep/as_sorl_abs30_K1_10K",
40
  "eval_K": 4,
41
  "alpha_traj": 0.0,
42
  "corrupt_method": "shuffle",
@@ -69,16 +69,21 @@
69
  "no_wandb": false,
70
  "n_params": 162519662,
71
  "run_name": "add_sub_sorl_v1_abs30_K1_10K",
72
- "git_commit": "57deaa28d9c21e39ddac5ef448d6e1be992fba91",
73
- "timestamp": "2026-04-13T09:57:42.064112+00:00",
74
  "tokenizer": "Qwen/Qwen3-0.6B",
75
  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
76
  "dataset_config": "add_sub_6digit",
 
77
  "model_repo": "thoughtworks/arithmetic-sorl",
78
  "trainer_version": "v1",
79
- "wandb_run_id": "6mbrz6zi",
80
- "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/6mbrz6zi",
81
- "final_accuracy": 0.96125,
82
- "sft_accuracy": 0.7683333333333333,
 
 
 
 
83
  "eval_method": "ArithmeticEvaluator"
84
  }
 
36
  "eval_every": 156,
37
  "save_every": 999999,
38
  "eval_samples": 100,
39
+ "output_dir": "ckpt/sweep/as_sorl_abs30_K1_10K_2L3H510d",
40
  "eval_K": 4,
41
  "alpha_traj": 0.0,
42
  "corrupt_method": "shuffle",
 
69
  "no_wandb": false,
70
  "n_params": 162519662,
71
  "run_name": "add_sub_sorl_v1_abs30_K1_10K",
72
+ "git_commit": "f835493c19eb98267697007042c9d440cad2afbb",
73
+ "timestamp": "2026-04-16T03:41:36.307160+00:00",
74
  "tokenizer": "Qwen/Qwen3-0.6B",
75
  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
76
  "dataset_config": "add_sub_6digit",
77
+ "train_dataset": "fixed_train/train_10K_seed42.pt",
78
  "model_repo": "thoughtworks/arithmetic-sorl",
79
  "trainer_version": "v1",
80
+ "wandb_run_id": "xhx3a9go",
81
+ "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/xhx3a9go",
82
+ "eval_final_dataset": "eval_sets/eval_add_sub_6d_N100_seed42.json",
83
+ "eval_epoch_dataset": "eval_sets/eval_add_sub_6d_N25_seed42.json",
84
+ "eval_hf_repo": "thoughtworks/arithmetic-sorl-data",
85
+ "config_hash": "715cfa395fac",
86
+ "final_accuracy": 0.9738461538461538,
87
+ "sft_accuracy": 0.9276923076923077,
88
  "eval_method": "ArithmeticEvaluator"
89
  }