walke007 commited on
Commit
0282214
·
verified ·
1 Parent(s): a3baf17

Upload Part 2 checkpoints and recorded results

Browse files
.gitattributes CHANGED
@@ -34,3 +34,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  comparison_test_bleu.png filter=lfs diff=lfs merge=lfs -text
 
 
 
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  comparison_test_bleu.png filter=lfs diff=lfs merge=lfs -text
37
+ comparison_dataset_dynamics.png filter=lfs diff=lfs merge=lfs -text
38
+ comparison_test_bleu_vs_dataset.png filter=lfs diff=lfs merge=lfs -text
README.md CHANGED
@@ -23,11 +23,11 @@ sys.path.insert(0, str(folder / "source"))
23
  from part2.model import DecoderOnlyTransformer, TransformerConfig
24
  from part2.data import load_tokenizer
25
 
26
- checkpoint = torch.load(folder / 'adamw' / "final.pt", map_location="cpu", weights_only=True)
27
  model = DecoderOnlyTransformer(TransformerConfig(**checkpoint["config"]), False, 0, 0, False)
28
  model.load_state_dict(checkpoint["model_state_dict"])
29
  model.eval()
30
- tokenizer = load_tokenizer(str(folder / 'adamw' / "tokenizer"))
31
  ```
32
 
33
  The final checkpoints include optimizer/scaler state for resuming training.
 
23
  from part2.model import DecoderOnlyTransformer, TransformerConfig
24
  from part2.data import load_tokenizer
25
 
26
+ checkpoint = torch.load(folder / 'muon' / "final.pt", map_location="cpu", weights_only=True)
27
  model = DecoderOnlyTransformer(TransformerConfig(**checkpoint["config"]), False, 0, 0, False)
28
  model.load_state_dict(checkpoint["model_state_dict"])
29
  model.eval()
30
+ tokenizer = load_tokenizer(str(folder / 'muon' / "tokenizer"))
31
  ```
32
 
33
  The final checkpoints include optimizer/scaler state for resuming training.
comparison_dataset_dynamics.png ADDED

Git LFS Details

  • SHA256: 30e53cb0c4b0d8ace3d5e65c2a6cb600b2d792c10011fc93bb7d7b6104ff57e2
  • Pointer size: 131 Bytes
  • Size of remote file: 210 kB
comparison_dataset_metrics.csv ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ optimizer,step,successful_updates,skipped_updates,train_tokens,training_budget_tokens,selected_dataset_fraction,full_dataset_token_fraction,tokens_per_parameter,train_loss,train_perplexity,cumulative_train_loss,validation_loss,validation_perplexity,validation_tokens,test_loss,test_perplexity,test_tokens,test_bleu,bleu_signature,bleu_samples,generated_tokens,learning_rates,last_gradient_norm,training_seconds,evaluation_seconds,total_evaluation_seconds,train_tokens_per_second,prediction_file,experiment_dataset_tokens,experiment_dataset_passes,original_train_split_tokens
2
+ adamw,0,0,0,0,42733510,0.0,0.0,0.0,,,,6.152310108351061,469.80142637427707,720136,6.153817186992487,470.5099878645091,708110,0.002109752683516993,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],,0.0,12.185807590999957,12.185807590999957,,predictions_00.jsonl,42733510,0.0,170790691
3
+ adamw,261,261,0,4273351,42733510,0.1,0.025020983140117397,1.273721524692013,2.640498514812532,14.02019113885227,2.640498514812532,2.442611973858063,11.503047196263836,720136,2.44764719613434,11.561113661849923,708110,0.05532590836351658,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],0.45159417390823364,40.88506411700109,12.159935426000004,24.34574301699996,104521.07859659751,predictions_01.jsonl,42733510,0.1,170790691
4
+ adamw,522,522,0,8546702,42733510,0.2,0.050041966280234794,2.547443049384026,2.2929823701285956,9.904432360498767,2.466740442470564,2.1418738580716004,8.515379300241122,720136,2.1500780604575915,8.585528561309054,708110,0.10125930673494067,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.0839279890060425,82.78312815800115,12.447991385000023,36.793734401999984,103242.07589362453,predictions_02.jsonl,42733510,0.2,170790691
5
+ adamw,783,783,0,12820053,42733510,0.3,0.07506294942035219,3.821164574076039,2.025029414033044,7.576333791891917,2.319503432991391,1.9197091547638223,6.818974914481506,720136,1.9306420518837115,6.8939350849472545,708110,0.1722466981765809,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.8015629053115845,124.17741831200016,12.350141832000077,49.14387623400006,103239.80941356957,predictions_03.jsonl,42733510,0.3,170790691
6
+ adamw,1044,1044,0,17093404,42733510,0.4,0.10008393256046959,5.094886098768052,1.8591893168135667,6.418531266327286,2.2044249039469346,1.76821006672264,5.860354324675273,720136,1.7770830621081117,5.912584598370209,708110,0.24831101149416698,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.264988899230957,165.69403289600137,12.310649127000033,61.454525361000094,103162.45975332591,predictions_04.jsonl,42733510,0.4,170790691
7
+ adamw,1305,1305,0,21366755,42733510,0.5,0.125104915700587,6.368607623460065,1.731667797280856,5.650069224916237,2.109873482613719,1.66693166833244,5.295893285074366,720136,1.6760529018266936,5.344419336577045,708110,0.26162517431091903,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],2.0656886100769043,207.10014180200125,12.336321245000022,73.79084660600012,103171.12684755067,predictions_05.jsonl,42733510,0.5,170790691
8
+ adamw,1566,1566,0,25640106,42733510,0.6,0.15012589884070437,7.642329148152078,1.6438578430091555,5.175095758349662,2.0322042093462915,1.6001972305404812,4.954009409999732,720136,1.6092371785126076,4.998996431121064,708110,0.410308644723639,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.3277252912521362,248.49769949300202,12.298542177000058,86.08938878300017,103180.45620668636,predictions_06.jsonl,42733510,0.6,170790691
9
+ adamw,1827,1827,0,29913457,42733510,0.7,0.1751468819808218,8.916050672844092,1.5925767979894636,4.916401192870087,1.9694002934381734,1.5608394358409383,4.762817647378607,720136,1.566571245496374,4.790195599205834,708110,0.3047428372679637,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.0786640644073486,289.87049320100147,12.298826415000008,98.38821519800018,103195.93646690166,predictions_07.jsonl,42733510,0.7,170790691
10
+ adamw,2088,2088,0,34186808,42733510,0.8,0.20016786512093918,10.189772197536104,1.5402283544145818,4.665655572397232,1.9157538010602244,1.523367808302876,4.587649531642508,720136,1.5293414451188727,4.615136500630765,708110,0.5040661650051995,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.9299545288085938,331.26377932600093,12.284402999000008,110.67261819700019,103201.16515472198,predictions_08.jsonl,42733510,0.8,170790691
11
+ adamw,2349,2349,0,38460159,42733510,0.9,0.22518884826105656,11.463493722228117,1.5047946790418072,4.503228928082886,1.8700916763915114,1.485235500980891,4.416005263902061,720136,1.491143989921234,4.442174416106604,708110,0.4872224185479862,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.259429931640625,372.6202616480012,12.283753058000002,122.95637125500019,103215.42588666771,predictions_09.jsonl,42733510,0.9,170790691
12
+ adamw,2610,2610,0,42733510,42733510,1.0,0.250209831401174,12.73721524692013,1.4782456321713036,4.385245594709652,1.8309070719694907,1.4478944097668704,4.254147586534556,720136,1.452432387551678,4.273496683456494,708110,0.47516893154716505,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.2657051086425781,413.9625683530009,12.29498626499992,135.2513575200001,103230.37218079965,predictions_10.jsonl,42733510,1.0,170790691
13
+ mars,0,0,0,0,42733510,0.0,0.0,0.0,,,,6.152309945621383,469.80134992364816,720136,6.1538172007836,470.5099943533652,708110,0.002109752683516993,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],,0.0,12.137210432000018,12.137210432000018,,predictions_00.jsonl,42733510,0.0,170790691
14
+ mars,261,261,0,4273351,42733510,0.1,0.025020983140117397,1.273721524692013,2.6350216016477592,13.943613664940504,2.6350216016477592,2.43761234857027,11.445679797860048,720136,2.4428096350341932,11.505321126828223,708110,0.06660427037878557,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],0.8482398390769958,42.77929676000008,12.14897425700002,24.286184689000038,99892.96981608428,predictions_01.jsonl,42733510,0.1,170790691
15
+ mars,522,522,0,8546702,42733510,0.2,0.050041966280234794,2.547443049384026,2.2968188837789993,9.942503834617087,2.4659202427133793,2.1463869595899245,8.553896923083718,720136,2.1558107833925875,8.634888365662338,708110,0.12884781642453785,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.0802031755447388,86.43895843200028,12.409839562999991,36.69602425200003,98875.57826976261,predictions_02.jsonl,42733510,0.2,170790691
16
+ mars,783,783,0,12820053,42733510,0.3,0.07506294942035219,3.821164574076039,2.0407106237667065,7.6960762708417425,2.3241837030644885,1.957633121990152,7.08254369397268,720136,1.9675556565187613,7.153170296305804,708110,0.15496398107655524,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.1142001152038574,129.72636143600107,12.415474795000023,49.11149904700005,98823.80772950775,predictions_03.jsonl,42733510,0.3,170790691
17
+ mars,1044,1044,0,17093404,42733510,0.4,0.10008393256046959,5.094886098768052,1.8945741906420932,6.6497162902814395,2.2167813249588897,1.81029685169711,6.112261598176767,720136,1.820918442198908,6.177529549019345,708110,0.17599086736363748,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.237196922302246,173.21508395600074,12.36834270899999,61.47984175600004,98683.1147127001,predictions_04.jsonl,42733510,0.4,170790691
18
+ mars,1305,1305,0,21366755,42733510,0.5,0.125104915700587,6.368607623460065,1.7801481340870975,5.930734897523064,2.129454686784531,1.714784321121288,5.555477184049246,720136,1.724313380479816,5.608668685895299,708110,0.24534585874439038,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.592305302619934,216.62708447400064,12.436493875999986,73.91633563200003,98633.81142704903,predictions_05.jsonl,42733510,0.5,170790691
19
+ mars,1566,1566,0,25640106,42733510,0.6,0.15012589884070437,7.642329148152078,1.6931412306945999,5.43653131019262,2.0567357774362094,1.657125382610021,5.244214047188049,720136,1.6676527921542468,5.299713657195096,708110,0.2440907782990909,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.0193804502487183,260.1115367720008,12.393600711999966,86.309936344,98573.50549766148,predictions_06.jsonl,42733510,0.6,170790691
20
+ mars,1827,1827,0,29913457,42733510,0.7,0.1751468819808218,8.916050672844092,1.6378689231330386,5.144195147502627,1.9968976553928992,1.601110594257543,4.958536289477835,720136,1.609088186913271,4.99825167813006,708110,0.27954084634942006,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.314588189125061,303.5638545300005,12.412496107999914,98.72243245199991,98540.90516248772,predictions_07.jsonl,42733510,0.7,170790691
21
+ mars,2088,2088,0,34186808,42733510,0.8,0.20016786512093918,10.189772197536104,1.5845113037937653,4.876907469877936,1.9453493614430073,1.5681297796961269,4.7976671036559315,720136,1.5752427994392644,4.831914661867808,708110,0.291937689998352,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.9565479755401611,347.0294731490004,12.407213007999985,111.12964545999989,98512.69314327538,predictions_08.jsonl,42733510,0.8,170790691
22
+ mars,2349,2349,0,38460159,42733510,0.9,0.22518884826105656,11.463493722228117,1.5487290109711367,4.705485759552475,1.9012804336127995,1.5326619945746358,4.630486761171459,720136,1.5411480120993295,4.669948352040108,708110,0.46672146049280167,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],2.234539747238159,390.5177544279994,12.427969979000068,123.55761543899996,98485.04597782885,predictions_09.jsonl,42733510,0.9,170790691
23
+ mars,2610,2610,0,42733510,42733510,1.0,0.250209831401174,12.73721524692013,1.5211646478296674,4.577553329390168,1.8632688550344862,1.4913745594799108,4.443198764388519,720136,1.49689860280028,4.4678111040721165,708110,0.4529043189821306,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.104906678199768,433.93933552499936,12.412074367999935,135.9696898069999,98478.07401073487,predictions_10.jsonl,42733510,1.0,170790691
24
+ lion,0,0,0,0,42733510,0.0,0.0,0.0,,,,6.152310151745643,469.8014467611137,720136,6.153817203541823,470.5099956511367,708110,0.002109752683516993,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],,0.0,12.305204914,12.305204914,,predictions_00.jsonl,42733510,0.0,170790691
25
+ lion,261,261,0,4273351,42733510,0.1,0.025020983140117397,1.273721524692013,2.8078204490104692,16.57375547868066,2.8078204490104692,2.493524042246881,12.103855548831,720136,2.497530412989772,12.152445350965934,708110,0.023491054883226305,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],0.5989791750907898,40.75469877799992,12.05658539000001,24.36179030400001,104855.41859302927,predictions_01.jsonl,42733510,0.1,170790691
26
+ lion,522,522,0,8546702,42733510,0.2,0.050041966280234794,2.547443049384026,2.438017814129468,11.450321567795019,2.6229191315699687,2.335367805091677,10.33325985609919,720136,2.344312593283088,10.426103290238618,708110,0.06359788640230295,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],0.7433375120162964,81.26354952799852,12.179590484000073,36.54138078800008,105172.63951232308,predictions_02.jsonl,42733510,0.2,170790691
27
+ lion,783,783,0,12820053,42733510,0.3,0.07506294942035219,3.821164574076039,2.192337917007256,8.95612733973809,2.4793920600490646,2.0457414521788246,7.734891464431085,720136,2.0569859330923603,7.822357135321954,708110,0.1473893318542731,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],1.2614325284957886,121.98602876099653,12.17860739400021,48.71998818200029,105094.43688111148,predictions_03.jsonl,42733510,0.3,170790691
28
+ lion,1044,1044,0,17093404,42733510,0.4,0.10008393256046959,5.094886098768052,1.9468686484814643,7.0067127125346875,2.3462612071571645,1.822833350879209,6.189370287323649,720136,1.8345315415441281,6.262199877713038,708110,0.25862362069881323,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],1.393475890159607,162.67717684499303,12.244096438000042,60.964084620000335,105075.61251992622,predictions_04.jsonl,42733510,0.4,170790691
29
+ lion,1305,1305,0,21366755,42733510,0.5,0.125104915700587,6.368607623460065,1.758935989300471,5.806256187842715,2.2287961635858258,1.6703211946467404,5.313874311087103,720136,1.6780224823826806,5.354955973957277,708110,0.23976036675370394,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],1.1012609004974365,203.45306142699064,12.25387805499986,73.2179626750002,105020.56272899822,predictions_05.jsonl,42733510,0.5,170790691
30
+ lion,1566,1566,0,25640106,42733510,0.6,0.15012589884070437,7.642329148152078,1.6225743659270604,5.066115578803247,2.127759197309365,1.5729091530757733,4.820651828634709,720136,1.5809962055959261,4.859794755275936,708110,0.4276805014235463,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],1.2251908779144287,244.21673679899163,12.240612259000045,85.45857493400024,104989.14339807798,predictions_06.jsonl,42733510,0.6,170790691
31
+ lion,1827,1827,0,29913457,42733510,0.7,0.1751468819808218,8.916050672844092,1.5423915518299942,4.6757592306364,2.044135247955169,1.501913337593524,4.490272263173101,720136,1.5057976073632628,4.507747609490873,708110,0.5148404590813516,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],1.1267120838165283,285.19143254099345,12.46664583200004,97.92522076600028,104889.04499506744,predictions_07.jsonl,42733510,0.7,170790691
32
+ lion,2088,2088,0,34186808,42733510,0.8,0.20016786512093918,10.189772197536104,1.4766037938191805,4.378051637582159,1.9731938161881706,1.456802736848491,4.292214227993766,720136,1.4601326796088354,4.306530879273558,708110,0.6183258743967537,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],1.731221318244934,326.1571126909937,12.370582222999928,110.29580298900021,104816.9936198482,predictions_08.jsonl,42733510,0.8,170790691
33
+ lion,2349,2349,0,38460159,42733510,0.9,0.22518884826105656,11.463493722228117,1.4372478736383814,4.2090958983168925,1.9136442670159717,1.4123747599845724,4.105693873750749,720136,1.4161928332164582,4.121399677503366,708110,0.47246291242006627,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],0.9133048057556152,367.225464117995,12.380042230000072,122.67584521900028,104731.73229523697,predictions_09.jsonl,42733510,0.9,170790691
34
+ lion,2610,2610,0,42733510,42733510,1.0,0.250209831401174,12.73721524692013,1.4083791706187379,4.089321936519697,1.8631177573762483,1.3890832556281036,4.011171148369319,720136,1.391464327824464,4.020733416172768,708110,0.5714227299959309,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],0.9298375248908997,408.1954670909936,12.43709894199992,135.1129441610002,104688.84013960397,predictions_10.jsonl,42733510,1.0,170790691
35
+ muon,0,0,0,0,42733510,0.0,0.0,0.0,,,,6.152294551393765,469.7941177503992,720136,6.153802047108677,470.50286445188505,708110,0.002089086296127387,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0, 0.0, 0.0]",,0.0,82.7089025840396,82.7089025840396,,predictions_00.jsonl,42733510,0.0,170790691
36
+ muon,261,261,0,4273351,42733510,0.1,0.025020983140117397,1.273721524692013,4.0481406036315155,57.290831582475136,4.0481406036315155,2.5621703693382223,12.963923305966714,720136,2.5636621853640325,12.98327752739135,708110,0.03083398708662475,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0009938625865312252, 9.938625865312251e-05, 9.938625865312251e-05]",0.3203253746032715,281.1368540887488,78.27012733393349,160.9790299179731,15200.25189814138,predictions_01.jsonl,42733510,0.1,170790691
37
+ muon,522,522,0,8546702,42733510,0.2,0.050041966280234794,2.547443049384026,2.5045045028839676,12.237493820398619,3.2763225532577414,2.463011343300085,11.740111869820579,720136,2.467185018978778,11.789213685445926,708110,0.05582133505440844,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0009457631880429201, 9.4576318804292e-05, 9.4576318804292e-05]",0.774559497833252,568.6849648010684,79.9007168749813,240.8797467929544,15028.886868830303,predictions_02.jsonl,42733510,0.2,170790691
38
+ muon,783,783,0,12820053,42733510,0.3,0.07506294942035219,3.821164574076039,2.430226130058437,11.36145095510898,2.9942904121913068,2.3827435894053344,10.834587783949875,720136,2.39104282934943,10.92488079106834,708110,0.05654392092884844,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0008547767072315835, 8.547767072315835e-05, 8.547767072315835e-05]",0.8050723671913147,840.1531296785688,72.94383920903783,313.8235860019922,15259.186149678188,predictions_03.jsonl,42733510,0.3,170790691
39
+ muon,1044,1044,0,17093404,42733510,0.4,0.10008393256046959,5.094886098768052,2.3242174275854817,10.218680098756618,2.8267721660398504,2.223032372531022,9.235293297912508,720136,2.2355781408321977,9.351886997309776,708110,0.100897461835075,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0007307629410938362, 7.307629410938362e-05, 7.307629410938362e-05]",0.6698967218399048,1115.7324636415578,77.46016829099972,391.28375429299194,15320.342964844893,predictions_04.jsonl,42733510,0.4,170790691
40
+ muon,1305,1305,0,21366755,42733510,0.5,0.125104915700587,6.368607623460065,2.1430165465797297,8.525115287851063,2.6900210421478263,2.0409670501908668,7.6980500012074335,720136,2.0537061002671284,7.7967431393926105,708110,0.12345122653106588,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0005871607054625496, 5.8716070546254966e-05, 5.8716070546254966e-05]",0.6055232286453247,1406.0170264714397,78.36569400003646,469.6494482930284,15196.654519627198,predictions_05.jsonl,42733510,0.5,170790691
41
+ muon,1566,1566,0,25640106,42733510,0.6,0.15012589884070437,7.642329148152078,1.9805903106765663,7.24701971584848,2.5717825869026165,1.9150039671325971,6.786965721738856,720136,1.9279956241901859,6.875714904050608,708110,0.12877604487102923,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00043953153078664056, 4.3953153078664054e-05, 4.3953153078664054e-05]",0.8913306593894958,1680.1966308491537,73.43593741592485,543.0853857089533,15260.18177232135,predictions_06.jsonl,42733510,0.6,170790691
42
+ muon,1827,1827,0,29913457,42733510,0.7,0.1751468819808218,8.916050672844092,1.884803411397829,6.585059767065248,2.473642704687647,1.8385603145838079,6.287479748988225,720136,1.8502426677341355,6.361363032867387,708110,0.1802229209222577,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00030387332884490803, 3.0387332884490805e-05, 3.0387332884490805e-05]",0.75834721326828,1950.8896681739716,73.07733799994458,616.1627237088978,15333.23872077242,predictions_07.jsonl,42733510,0.7,170790691
43
+ muon,2088,2088,0,34186808,42733510,0.8,0.20016786512093918,10.189772197536104,1.8216860253402747,6.182273136872596,2.3921481197692254,1.7927702604742368,6.006067813606906,720136,1.8042369539932885,6.075333920134396,708110,0.2571039476796964,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00019488677077162295, 1.9488677077162295e-05, 1.9488677077162295e-05]",1.9932366609573364,2219.8103655477753,77.00118562497664,693.1639093338745,15400.778611808957,predictions_08.jsonl,42733510,0.8,170790691
44
+ muon,2349,2349,0,38460159,42733510,0.9,0.22518884826105656,11.463493722228117,1.7873370917559692,5.9735243210402365,2.324946894434419,1.7655405758426186,5.844731024658187,720136,1.7764862606588754,5.909057012052186,708110,0.22627215440136425,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00012438224123471442, 1.2438224123471442e-05, 1.2438224123471442e-05]",0.6744703054428101,2485.9783932112623,73.41921604203526,766.5831253759097,15470.833980306279,predictions_09.jsonl,42733510,0.9,170790691
45
+ muon,2610,2610,0,42733510,42733510,1.0,0.250209831401174,12.73721524692013,1.7692924620928165,5.86670097923902,2.2693814512002586,1.747523807568987,5.740370800119521,720136,1.7582855103506165,5.802480568528633,708110,0.24637446793816298,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0001, 1e-05, 1e-05]",0.7609756588935852,2754.439210804994,73.39668541704305,839.9798107929528,15514.41390769012,predictions_10.jsonl,42733510,1.0,170790691
comparison_test_bleu.png CHANGED

Git LFS Details

  • SHA256: 1d66e1f69e31f2517c21371810bbb28e386c83f0a2e50bae6510f925cd25853c
  • Pointer size: 131 Bytes
  • Size of remote file: 107 kB

Git LFS Details

  • SHA256: 2150d9544a1f2c19accba35fd5ef2b7b8ebfd3015cd1c8af695ea1615bcf426a
  • Pointer size: 131 Bytes
  • Size of remote file: 113 kB
comparison_test_bleu_vs_dataset.png ADDED

Git LFS Details

  • SHA256: 5bc14c07558e37a667d5ab9de4b53607646d5fd5c59eaf376fa03cd29f58f6a5
  • Pointer size: 131 Bytes
  • Size of remote file: 127 kB
comparison_validation_loss.png CHANGED
comparison_validation_loss_vs_dataset.png ADDED
muon/bleu_vs_tokens.png CHANGED
muon/final.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4817ff33ca563eaf7539a0055d959625f2a5bce924428ed6d90d30342f9449a4
3
- size 28036795
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b543c13675576ec83ff68c8e175d230b721cf88e3338ed4c2c52b8f3657f2919
3
+ size 28036667
muon/latest.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d3b49b4214988c7d8b2a75314528f785b5ae4917bef47ddac3265a8ee7377a7b
3
- size 28041551
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a5f8e5f2225e0d49620500ac824cf1254e2853103e645e725d851ad731d166e
3
+ size 28039439
muon/loss_vs_tokens.png CHANGED
muon/metrics.csv CHANGED
@@ -1,12 +1,12 @@
1
  optimizer,step,successful_updates,skipped_updates,train_tokens,training_budget_tokens,selected_dataset_fraction,full_dataset_token_fraction,tokens_per_parameter,train_loss,train_perplexity,cumulative_train_loss,validation_loss,validation_perplexity,validation_tokens,test_loss,test_perplexity,test_tokens,test_bleu,bleu_signature,bleu_samples,generated_tokens,learning_rates,last_gradient_norm,training_seconds,evaluation_seconds,total_evaluation_seconds,train_tokens_per_second,prediction_file
2
- muon,0,0,0,0,42733510,0.0,0.0,0.0,,,,6.15231006495648,469.80140598744134,720136,6.1538171125204775,470.50995282468597,708110,0.002109752683516993,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.02, 0.0003]",,0.0,12.77970295199998,12.77970295199998,,predictions_00.jsonl
3
- muon,261,260,1,4273351,42733510,0.1,0.025020983140117397,1.273721524692013,2.6981312647732096,14.85195141114328,2.6981312647732096,2.731121214647303,15.350088118679313,720136,2.732742017758982,15.374987762525775,708110,0.018045797928904022,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.02, 0.0003]",8.129785537719727,46.58533229100021,12.303350157000068,25.083053109000048,91731.6844131552,predictions_01.jsonl
4
- muon,522,517,5,8546702,42733510,0.2,0.050041966280234794,2.547443049384026,2.8843010257334596,17.8910578350012,2.7912161452533346,2.996136048867853,20.00807713684173,720136,2.9921943321173616,19.929366194078916,708110,0.011246155383913024,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.02, 0.0003]",31.54551887512207,92.22961090400327,12.420992829999932,37.50404593899998,92667.65755843633,predictions_02.jsonl
5
- muon,783,778,5,12820053,42733510,0.3,0.07506294942035219,3.821164574076039,3.0492889790322595,21.100336306090146,2.877240423179643,3.054537094382172,21.211364394218784,720136,3.0524864797444957,21.167912626548613,708110,0.0,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.02, 0.0003]",0.8065668344497681,137.98718706500176,12.2808918180001,49.78493775700008,92907.56100391295,predictions_03.jsonl
6
- muon,1044,1039,5,17093404,42733510,0.4,0.10008393256046959,5.094886098768052,3.031590114572922,20.73016974053102,2.9158278460279625,3.0327870523328233,20.754997319031602,720136,3.029445111535902,20.68575111954907,708110,0.0,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.02, 0.0003]",1.1100726127624512,183.77406036300226,12.332217346999869,62.11715510399995,93013.14867961244,predictions_04.jsonl
7
- muon,1305,1299,6,21366755,42733510,0.5,0.125104915700587,6.368607623460065,3.083644418651538,21.837843794647593,2.9493911605526777,3.1040202388854325,22.287371974196184,720136,3.1009577403890285,22.21922133990873,708110,0.0,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.02, 0.0003]",0.3168768286705017,229.55735211700062,12.252023473000008,74.36917857699996,93078.0687394835,predictions_05.jsonl
8
- muon,1566,1560,6,25640106,42733510,0.6,0.15012589884070437,7.642329148152078,3.104715551778856,22.302874060065903,2.975278559090374,3.104101647119433,22.289186423643677,720136,3.1019721856972255,22.24177296150817,708110,0.0,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.02, 0.0003]",0.44369596242904663,301.63689511500047,23.203688336999676,97.57286691399963,85003.21550593004,predictions_06.jsonl
9
- muon,1827,1821,6,29913457,42733510,0.7,0.1751468819808218,8.916050672844092,3.1049437043658776,22.307963098997348,2.9938021512725888,3.104289111709455,22.293365248517116,720136,3.1018113013316255,22.23819489581057,708110,0.0,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.02, 0.0003]",0.5960580706596375,376.472215739004,23.19767409399992,120.77054100799955,79457.27665793545,predictions_07.jsonl
10
- muon,2088,2082,6,34186808,42733510,0.8,0.20016786512093918,10.189772197536104,3.10324956243961,22.27020223858161,3.0074830776684665,3.102636233815557,22.25654747394229,720136,3.100530291738307,22.209725793300777,708110,0.0,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.02, 0.0003]",0.7817509174346924,450.8752336010009,23.10945858199966,143.87999958999922,75823.21106209482,predictions_08.jsonl
11
- muon,2349,2343,6,38460159,42733510,0.9,0.22518884826105656,11.463493722228117,3.0995623015217837,22.188237397977314,3.017714102541057,3.0984756969950813,22.164140652968165,720136,3.096594844573054,22.12249235476808,708110,0.0,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.02, 0.0003]",0.2551116347312927,525.3090313580044,22.832562151000275,166.7125617409995,73214.34946696917,predictions_09.jsonl
12
- muon,2610,2604,6,42733510,42733510,1.0,0.250209831401174,12.73721524692013,3.0962514508087104,22.11489693302659,3.0255678373678228,3.0828583615629546,21.82068474761224,720136,3.080461163508759,21.76843889144591,708110,0.0,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.02, 0.0003]",0.28920120000839233,599.7354844430072,23.06644315299991,189.7790048939994,71253.92962146958,predictions_10.jsonl
 
1
  optimizer,step,successful_updates,skipped_updates,train_tokens,training_budget_tokens,selected_dataset_fraction,full_dataset_token_fraction,tokens_per_parameter,train_loss,train_perplexity,cumulative_train_loss,validation_loss,validation_perplexity,validation_tokens,test_loss,test_perplexity,test_tokens,test_bleu,bleu_signature,bleu_samples,generated_tokens,learning_rates,last_gradient_norm,training_seconds,evaluation_seconds,total_evaluation_seconds,train_tokens_per_second,prediction_file
2
+ muon,0,0,0,0,42733510,0.0,0.0,0.0,,,,6.152294551393765,469.7941177503992,720136,6.153802047108677,470.50286445188505,708110,0.002089086296127387,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0, 0.0, 0.0]",,0.0,82.7089025840396,82.7089025840396,,predictions_00.jsonl
3
+ muon,261,261,0,4273351,42733510,0.1,0.025020983140117397,1.273721524692013,4.0481406036315155,57.290831582475136,4.0481406036315155,2.5621703693382223,12.963923305966714,720136,2.5636621853640325,12.98327752739135,708110,0.03083398708662475,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0009938625865312252, 9.938625865312251e-05, 9.938625865312251e-05]",0.3203253746032715,281.1368540887488,78.27012733393349,160.9790299179731,15200.25189814138,predictions_01.jsonl
4
+ muon,522,522,0,8546702,42733510,0.2,0.050041966280234794,2.547443049384026,2.5045045028839676,12.237493820398619,3.2763225532577414,2.463011343300085,11.740111869820579,720136,2.467185018978778,11.789213685445926,708110,0.05582133505440844,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0009457631880429201, 9.4576318804292e-05, 9.4576318804292e-05]",0.774559497833252,568.6849648010684,79.9007168749813,240.8797467929544,15028.886868830303,predictions_02.jsonl
5
+ muon,783,783,0,12820053,42733510,0.3,0.07506294942035219,3.821164574076039,2.430226130058437,11.36145095510898,2.9942904121913068,2.3827435894053344,10.834587783949875,720136,2.39104282934943,10.92488079106834,708110,0.05654392092884844,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0008547767072315835, 8.547767072315835e-05, 8.547767072315835e-05]",0.8050723671913147,840.1531296785688,72.94383920903783,313.8235860019922,15259.186149678188,predictions_03.jsonl
6
+ muon,1044,1044,0,17093404,42733510,0.4,0.10008393256046959,5.094886098768052,2.3242174275854817,10.218680098756618,2.8267721660398504,2.223032372531022,9.235293297912508,720136,2.2355781408321977,9.351886997309776,708110,0.100897461835075,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0007307629410938362, 7.307629410938362e-05, 7.307629410938362e-05]",0.6698967218399048,1115.7324636415578,77.46016829099972,391.28375429299194,15320.342964844893,predictions_04.jsonl
7
+ muon,1305,1305,0,21366755,42733510,0.5,0.125104915700587,6.368607623460065,2.1430165465797297,8.525115287851063,2.6900210421478263,2.0409670501908668,7.6980500012074335,720136,2.0537061002671284,7.7967431393926105,708110,0.12345122653106588,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0005871607054625496, 5.8716070546254966e-05, 5.8716070546254966e-05]",0.6055232286453247,1406.0170264714397,78.36569400003646,469.6494482930284,15196.654519627198,predictions_05.jsonl
8
+ muon,1566,1566,0,25640106,42733510,0.6,0.15012589884070437,7.642329148152078,1.9805903106765663,7.24701971584848,2.5717825869026165,1.9150039671325971,6.786965721738856,720136,1.9279956241901859,6.875714904050608,708110,0.12877604487102923,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00043953153078664056, 4.3953153078664054e-05, 4.3953153078664054e-05]",0.8913306593894958,1680.1966308491537,73.43593741592485,543.0853857089533,15260.18177232135,predictions_06.jsonl
9
+ muon,1827,1827,0,29913457,42733510,0.7,0.1751468819808218,8.916050672844092,1.884803411397829,6.585059767065248,2.473642704687647,1.8385603145838079,6.287479748988225,720136,1.8502426677341355,6.361363032867387,708110,0.1802229209222577,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00030387332884490803, 3.0387332884490805e-05, 3.0387332884490805e-05]",0.75834721326828,1950.8896681739716,73.07733799994458,616.1627237088978,15333.23872077242,predictions_07.jsonl
10
+ muon,2088,2088,0,34186808,42733510,0.8,0.20016786512093918,10.189772197536104,1.8216860253402747,6.182273136872596,2.3921481197692254,1.7927702604742368,6.006067813606906,720136,1.8042369539932885,6.075333920134396,708110,0.2571039476796964,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00019488677077162295, 1.9488677077162295e-05, 1.9488677077162295e-05]",1.9932366609573364,2219.8103655477753,77.00118562497664,693.1639093338745,15400.778611808957,predictions_08.jsonl
11
+ muon,2349,2349,0,38460159,42733510,0.9,0.22518884826105656,11.463493722228117,1.7873370917559692,5.9735243210402365,2.324946894434419,1.7655405758426186,5.844731024658187,720136,1.7764862606588754,5.909057012052186,708110,0.22627215440136425,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00012438224123471442, 1.2438224123471442e-05, 1.2438224123471442e-05]",0.6744703054428101,2485.9783932112623,73.41921604203526,766.5831253759097,15470.833980306279,predictions_09.jsonl
12
+ muon,2610,2610,0,42733510,42733510,1.0,0.250209831401174,12.73721524692013,1.7692924620928165,5.86670097923902,2.2693814512002586,1.747523807568987,5.740370800119521,720136,1.7582855103506165,5.802480568528633,708110,0.24637446793816298,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0001, 1e-05, 1e-05]",0.7609756588935852,2754.439210804994,73.39668541704305,839.9798107929528,15514.41390769012,predictions_10.jsonl
muon/metrics.json CHANGED
@@ -12,365 +12,376 @@
12
  "train_loss": null,
13
  "train_perplexity": null,
14
  "cumulative_train_loss": null,
15
- "validation_loss": 6.15231006495648,
16
- "validation_perplexity": 469.80140598744134,
17
  "validation_tokens": 720136,
18
- "test_loss": 6.1538171125204775,
19
- "test_perplexity": 470.50995282468597,
20
  "test_tokens": 708110,
21
- "test_bleu": 0.002109752683516993,
22
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
23
  "bleu_samples": 250,
24
  "generated_tokens": 16000,
25
  "learning_rates": [
26
- 0.02,
27
- 0.0003
 
28
  ],
29
  "last_gradient_norm": null,
30
  "training_seconds": 0.0,
31
- "evaluation_seconds": 12.77970295199998,
32
- "total_evaluation_seconds": 12.77970295199998,
33
  "train_tokens_per_second": null,
34
  "prediction_file": "predictions_00.jsonl"
35
  },
36
  {
37
  "optimizer": "muon",
38
  "step": 261,
39
- "successful_updates": 260,
40
- "skipped_updates": 1,
41
  "train_tokens": 4273351,
42
  "training_budget_tokens": 42733510,
43
  "selected_dataset_fraction": 0.1,
44
  "full_dataset_token_fraction": 0.025020983140117397,
45
  "tokens_per_parameter": 1.273721524692013,
46
- "train_loss": 2.6981312647732096,
47
- "train_perplexity": 14.85195141114328,
48
- "cumulative_train_loss": 2.6981312647732096,
49
- "validation_loss": 2.731121214647303,
50
- "validation_perplexity": 15.350088118679313,
51
  "validation_tokens": 720136,
52
- "test_loss": 2.732742017758982,
53
- "test_perplexity": 15.374987762525775,
54
  "test_tokens": 708110,
55
- "test_bleu": 0.018045797928904022,
56
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
57
  "bleu_samples": 250,
58
  "generated_tokens": 16000,
59
  "learning_rates": [
60
- 0.02,
61
- 0.0003
 
62
  ],
63
- "last_gradient_norm": 8.129785537719727,
64
- "training_seconds": 46.58533229100021,
65
- "evaluation_seconds": 12.303350157000068,
66
- "total_evaluation_seconds": 25.083053109000048,
67
- "train_tokens_per_second": 91731.6844131552,
68
  "prediction_file": "predictions_01.jsonl"
69
  },
70
  {
71
  "optimizer": "muon",
72
  "step": 522,
73
- "successful_updates": 517,
74
- "skipped_updates": 5,
75
  "train_tokens": 8546702,
76
  "training_budget_tokens": 42733510,
77
  "selected_dataset_fraction": 0.2,
78
  "full_dataset_token_fraction": 0.050041966280234794,
79
  "tokens_per_parameter": 2.547443049384026,
80
- "train_loss": 2.8843010257334596,
81
- "train_perplexity": 17.8910578350012,
82
- "cumulative_train_loss": 2.7912161452533346,
83
- "validation_loss": 2.996136048867853,
84
- "validation_perplexity": 20.00807713684173,
85
  "validation_tokens": 720136,
86
- "test_loss": 2.9921943321173616,
87
- "test_perplexity": 19.929366194078916,
88
  "test_tokens": 708110,
89
- "test_bleu": 0.011246155383913024,
90
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
91
  "bleu_samples": 250,
92
  "generated_tokens": 16000,
93
  "learning_rates": [
94
- 0.02,
95
- 0.0003
 
96
  ],
97
- "last_gradient_norm": 31.54551887512207,
98
- "training_seconds": 92.22961090400327,
99
- "evaluation_seconds": 12.420992829999932,
100
- "total_evaluation_seconds": 37.50404593899998,
101
- "train_tokens_per_second": 92667.65755843633,
102
  "prediction_file": "predictions_02.jsonl"
103
  },
104
  {
105
  "optimizer": "muon",
106
  "step": 783,
107
- "successful_updates": 778,
108
- "skipped_updates": 5,
109
  "train_tokens": 12820053,
110
  "training_budget_tokens": 42733510,
111
  "selected_dataset_fraction": 0.3,
112
  "full_dataset_token_fraction": 0.07506294942035219,
113
  "tokens_per_parameter": 3.821164574076039,
114
- "train_loss": 3.0492889790322595,
115
- "train_perplexity": 21.100336306090146,
116
- "cumulative_train_loss": 2.877240423179643,
117
- "validation_loss": 3.054537094382172,
118
- "validation_perplexity": 21.211364394218784,
119
  "validation_tokens": 720136,
120
- "test_loss": 3.0524864797444957,
121
- "test_perplexity": 21.167912626548613,
122
  "test_tokens": 708110,
123
- "test_bleu": 0.0,
124
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
125
  "bleu_samples": 250,
126
  "generated_tokens": 16000,
127
  "learning_rates": [
128
- 0.02,
129
- 0.0003
 
130
  ],
131
- "last_gradient_norm": 0.8065668344497681,
132
- "training_seconds": 137.98718706500176,
133
- "evaluation_seconds": 12.2808918180001,
134
- "total_evaluation_seconds": 49.78493775700008,
135
- "train_tokens_per_second": 92907.56100391295,
136
  "prediction_file": "predictions_03.jsonl"
137
  },
138
  {
139
  "optimizer": "muon",
140
  "step": 1044,
141
- "successful_updates": 1039,
142
- "skipped_updates": 5,
143
  "train_tokens": 17093404,
144
  "training_budget_tokens": 42733510,
145
  "selected_dataset_fraction": 0.4,
146
  "full_dataset_token_fraction": 0.10008393256046959,
147
  "tokens_per_parameter": 5.094886098768052,
148
- "train_loss": 3.031590114572922,
149
- "train_perplexity": 20.73016974053102,
150
- "cumulative_train_loss": 2.9158278460279625,
151
- "validation_loss": 3.0327870523328233,
152
- "validation_perplexity": 20.754997319031602,
153
  "validation_tokens": 720136,
154
- "test_loss": 3.029445111535902,
155
- "test_perplexity": 20.68575111954907,
156
  "test_tokens": 708110,
157
- "test_bleu": 0.0,
158
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
159
  "bleu_samples": 250,
160
  "generated_tokens": 16000,
161
  "learning_rates": [
162
- 0.02,
163
- 0.0003
 
164
  ],
165
- "last_gradient_norm": 1.1100726127624512,
166
- "training_seconds": 183.77406036300226,
167
- "evaluation_seconds": 12.332217346999869,
168
- "total_evaluation_seconds": 62.11715510399995,
169
- "train_tokens_per_second": 93013.14867961244,
170
  "prediction_file": "predictions_04.jsonl"
171
  },
172
  {
173
  "optimizer": "muon",
174
  "step": 1305,
175
- "successful_updates": 1299,
176
- "skipped_updates": 6,
177
  "train_tokens": 21366755,
178
  "training_budget_tokens": 42733510,
179
  "selected_dataset_fraction": 0.5,
180
  "full_dataset_token_fraction": 0.125104915700587,
181
  "tokens_per_parameter": 6.368607623460065,
182
- "train_loss": 3.083644418651538,
183
- "train_perplexity": 21.837843794647593,
184
- "cumulative_train_loss": 2.9493911605526777,
185
- "validation_loss": 3.1040202388854325,
186
- "validation_perplexity": 22.287371974196184,
187
  "validation_tokens": 720136,
188
- "test_loss": 3.1009577403890285,
189
- "test_perplexity": 22.21922133990873,
190
  "test_tokens": 708110,
191
- "test_bleu": 0.0,
192
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
193
  "bleu_samples": 250,
194
  "generated_tokens": 16000,
195
  "learning_rates": [
196
- 0.02,
197
- 0.0003
 
198
  ],
199
- "last_gradient_norm": 0.3168768286705017,
200
- "training_seconds": 229.55735211700062,
201
- "evaluation_seconds": 12.252023473000008,
202
- "total_evaluation_seconds": 74.36917857699996,
203
- "train_tokens_per_second": 93078.0687394835,
204
  "prediction_file": "predictions_05.jsonl"
205
  },
206
  {
207
  "optimizer": "muon",
208
  "step": 1566,
209
- "successful_updates": 1560,
210
- "skipped_updates": 6,
211
  "train_tokens": 25640106,
212
  "training_budget_tokens": 42733510,
213
  "selected_dataset_fraction": 0.6,
214
  "full_dataset_token_fraction": 0.15012589884070437,
215
  "tokens_per_parameter": 7.642329148152078,
216
- "train_loss": 3.104715551778856,
217
- "train_perplexity": 22.302874060065903,
218
- "cumulative_train_loss": 2.975278559090374,
219
- "validation_loss": 3.104101647119433,
220
- "validation_perplexity": 22.289186423643677,
221
  "validation_tokens": 720136,
222
- "test_loss": 3.1019721856972255,
223
- "test_perplexity": 22.24177296150817,
224
  "test_tokens": 708110,
225
- "test_bleu": 0.0,
226
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
227
  "bleu_samples": 250,
228
  "generated_tokens": 16000,
229
  "learning_rates": [
230
- 0.02,
231
- 0.0003
 
232
  ],
233
- "last_gradient_norm": 0.44369596242904663,
234
- "training_seconds": 301.63689511500047,
235
- "evaluation_seconds": 23.203688336999676,
236
- "total_evaluation_seconds": 97.57286691399963,
237
- "train_tokens_per_second": 85003.21550593004,
238
  "prediction_file": "predictions_06.jsonl"
239
  },
240
  {
241
  "optimizer": "muon",
242
  "step": 1827,
243
- "successful_updates": 1821,
244
- "skipped_updates": 6,
245
  "train_tokens": 29913457,
246
  "training_budget_tokens": 42733510,
247
  "selected_dataset_fraction": 0.7,
248
  "full_dataset_token_fraction": 0.1751468819808218,
249
  "tokens_per_parameter": 8.916050672844092,
250
- "train_loss": 3.1049437043658776,
251
- "train_perplexity": 22.307963098997348,
252
- "cumulative_train_loss": 2.9938021512725888,
253
- "validation_loss": 3.104289111709455,
254
- "validation_perplexity": 22.293365248517116,
255
  "validation_tokens": 720136,
256
- "test_loss": 3.1018113013316255,
257
- "test_perplexity": 22.23819489581057,
258
  "test_tokens": 708110,
259
- "test_bleu": 0.0,
260
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
261
  "bleu_samples": 250,
262
  "generated_tokens": 16000,
263
  "learning_rates": [
264
- 0.02,
265
- 0.0003
 
266
  ],
267
- "last_gradient_norm": 0.5960580706596375,
268
- "training_seconds": 376.472215739004,
269
- "evaluation_seconds": 23.19767409399992,
270
- "total_evaluation_seconds": 120.77054100799955,
271
- "train_tokens_per_second": 79457.27665793545,
272
  "prediction_file": "predictions_07.jsonl"
273
  },
274
  {
275
  "optimizer": "muon",
276
  "step": 2088,
277
- "successful_updates": 2082,
278
- "skipped_updates": 6,
279
  "train_tokens": 34186808,
280
  "training_budget_tokens": 42733510,
281
  "selected_dataset_fraction": 0.8,
282
  "full_dataset_token_fraction": 0.20016786512093918,
283
  "tokens_per_parameter": 10.189772197536104,
284
- "train_loss": 3.10324956243961,
285
- "train_perplexity": 22.27020223858161,
286
- "cumulative_train_loss": 3.0074830776684665,
287
- "validation_loss": 3.102636233815557,
288
- "validation_perplexity": 22.25654747394229,
289
  "validation_tokens": 720136,
290
- "test_loss": 3.100530291738307,
291
- "test_perplexity": 22.209725793300777,
292
  "test_tokens": 708110,
293
- "test_bleu": 0.0,
294
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
295
  "bleu_samples": 250,
296
  "generated_tokens": 16000,
297
  "learning_rates": [
298
- 0.02,
299
- 0.0003
 
300
  ],
301
- "last_gradient_norm": 0.7817509174346924,
302
- "training_seconds": 450.8752336010009,
303
- "evaluation_seconds": 23.10945858199966,
304
- "total_evaluation_seconds": 143.87999958999922,
305
- "train_tokens_per_second": 75823.21106209482,
306
  "prediction_file": "predictions_08.jsonl"
307
  },
308
  {
309
  "optimizer": "muon",
310
  "step": 2349,
311
- "successful_updates": 2343,
312
- "skipped_updates": 6,
313
  "train_tokens": 38460159,
314
  "training_budget_tokens": 42733510,
315
  "selected_dataset_fraction": 0.9,
316
  "full_dataset_token_fraction": 0.22518884826105656,
317
  "tokens_per_parameter": 11.463493722228117,
318
- "train_loss": 3.0995623015217837,
319
- "train_perplexity": 22.188237397977314,
320
- "cumulative_train_loss": 3.017714102541057,
321
- "validation_loss": 3.0984756969950813,
322
- "validation_perplexity": 22.164140652968165,
323
  "validation_tokens": 720136,
324
- "test_loss": 3.096594844573054,
325
- "test_perplexity": 22.12249235476808,
326
  "test_tokens": 708110,
327
- "test_bleu": 0.0,
328
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
329
  "bleu_samples": 250,
330
  "generated_tokens": 16000,
331
  "learning_rates": [
332
- 0.02,
333
- 0.0003
 
334
  ],
335
- "last_gradient_norm": 0.2551116347312927,
336
- "training_seconds": 525.3090313580044,
337
- "evaluation_seconds": 22.832562151000275,
338
- "total_evaluation_seconds": 166.7125617409995,
339
- "train_tokens_per_second": 73214.34946696917,
340
  "prediction_file": "predictions_09.jsonl"
341
  },
342
  {
343
  "optimizer": "muon",
344
  "step": 2610,
345
- "successful_updates": 2604,
346
- "skipped_updates": 6,
347
  "train_tokens": 42733510,
348
  "training_budget_tokens": 42733510,
349
  "selected_dataset_fraction": 1.0,
350
  "full_dataset_token_fraction": 0.250209831401174,
351
  "tokens_per_parameter": 12.73721524692013,
352
- "train_loss": 3.0962514508087104,
353
- "train_perplexity": 22.11489693302659,
354
- "cumulative_train_loss": 3.0255678373678228,
355
- "validation_loss": 3.0828583615629546,
356
- "validation_perplexity": 21.82068474761224,
357
  "validation_tokens": 720136,
358
- "test_loss": 3.080461163508759,
359
- "test_perplexity": 21.76843889144591,
360
  "test_tokens": 708110,
361
- "test_bleu": 0.0,
362
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
363
  "bleu_samples": 250,
364
  "generated_tokens": 16000,
365
  "learning_rates": [
366
- 0.02,
367
- 0.0003
 
368
  ],
369
- "last_gradient_norm": 0.28920120000839233,
370
- "training_seconds": 599.7354844430072,
371
- "evaluation_seconds": 23.06644315299991,
372
- "total_evaluation_seconds": 189.7790048939994,
373
- "train_tokens_per_second": 71253.92962146958,
374
  "prediction_file": "predictions_10.jsonl"
375
  }
376
  ]
 
12
  "train_loss": null,
13
  "train_perplexity": null,
14
  "cumulative_train_loss": null,
15
+ "validation_loss": 6.152294551393765,
16
+ "validation_perplexity": 469.7941177503992,
17
  "validation_tokens": 720136,
18
+ "test_loss": 6.153802047108677,
19
+ "test_perplexity": 470.50286445188505,
20
  "test_tokens": 708110,
21
+ "test_bleu": 0.002089086296127387,
22
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
23
  "bleu_samples": 250,
24
  "generated_tokens": 16000,
25
  "learning_rates": [
26
+ 0.0,
27
+ 0.0,
28
+ 0.0
29
  ],
30
  "last_gradient_norm": null,
31
  "training_seconds": 0.0,
32
+ "evaluation_seconds": 82.7089025840396,
33
+ "total_evaluation_seconds": 82.7089025840396,
34
  "train_tokens_per_second": null,
35
  "prediction_file": "predictions_00.jsonl"
36
  },
37
  {
38
  "optimizer": "muon",
39
  "step": 261,
40
+ "successful_updates": 261,
41
+ "skipped_updates": 0,
42
  "train_tokens": 4273351,
43
  "training_budget_tokens": 42733510,
44
  "selected_dataset_fraction": 0.1,
45
  "full_dataset_token_fraction": 0.025020983140117397,
46
  "tokens_per_parameter": 1.273721524692013,
47
+ "train_loss": 4.0481406036315155,
48
+ "train_perplexity": 57.290831582475136,
49
+ "cumulative_train_loss": 4.0481406036315155,
50
+ "validation_loss": 2.5621703693382223,
51
+ "validation_perplexity": 12.963923305966714,
52
  "validation_tokens": 720136,
53
+ "test_loss": 2.5636621853640325,
54
+ "test_perplexity": 12.98327752739135,
55
  "test_tokens": 708110,
56
+ "test_bleu": 0.03083398708662475,
57
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
58
  "bleu_samples": 250,
59
  "generated_tokens": 16000,
60
  "learning_rates": [
61
+ 0.0009938625865312252,
62
+ 9.938625865312251e-05,
63
+ 9.938625865312251e-05
64
  ],
65
+ "last_gradient_norm": 0.3203253746032715,
66
+ "training_seconds": 281.1368540887488,
67
+ "evaluation_seconds": 78.27012733393349,
68
+ "total_evaluation_seconds": 160.9790299179731,
69
+ "train_tokens_per_second": 15200.25189814138,
70
  "prediction_file": "predictions_01.jsonl"
71
  },
72
  {
73
  "optimizer": "muon",
74
  "step": 522,
75
+ "successful_updates": 522,
76
+ "skipped_updates": 0,
77
  "train_tokens": 8546702,
78
  "training_budget_tokens": 42733510,
79
  "selected_dataset_fraction": 0.2,
80
  "full_dataset_token_fraction": 0.050041966280234794,
81
  "tokens_per_parameter": 2.547443049384026,
82
+ "train_loss": 2.5045045028839676,
83
+ "train_perplexity": 12.237493820398619,
84
+ "cumulative_train_loss": 3.2763225532577414,
85
+ "validation_loss": 2.463011343300085,
86
+ "validation_perplexity": 11.740111869820579,
87
  "validation_tokens": 720136,
88
+ "test_loss": 2.467185018978778,
89
+ "test_perplexity": 11.789213685445926,
90
  "test_tokens": 708110,
91
+ "test_bleu": 0.05582133505440844,
92
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
93
  "bleu_samples": 250,
94
  "generated_tokens": 16000,
95
  "learning_rates": [
96
+ 0.0009457631880429201,
97
+ 9.4576318804292e-05,
98
+ 9.4576318804292e-05
99
  ],
100
+ "last_gradient_norm": 0.774559497833252,
101
+ "training_seconds": 568.6849648010684,
102
+ "evaluation_seconds": 79.9007168749813,
103
+ "total_evaluation_seconds": 240.8797467929544,
104
+ "train_tokens_per_second": 15028.886868830303,
105
  "prediction_file": "predictions_02.jsonl"
106
  },
107
  {
108
  "optimizer": "muon",
109
  "step": 783,
110
+ "successful_updates": 783,
111
+ "skipped_updates": 0,
112
  "train_tokens": 12820053,
113
  "training_budget_tokens": 42733510,
114
  "selected_dataset_fraction": 0.3,
115
  "full_dataset_token_fraction": 0.07506294942035219,
116
  "tokens_per_parameter": 3.821164574076039,
117
+ "train_loss": 2.430226130058437,
118
+ "train_perplexity": 11.36145095510898,
119
+ "cumulative_train_loss": 2.9942904121913068,
120
+ "validation_loss": 2.3827435894053344,
121
+ "validation_perplexity": 10.834587783949875,
122
  "validation_tokens": 720136,
123
+ "test_loss": 2.39104282934943,
124
+ "test_perplexity": 10.92488079106834,
125
  "test_tokens": 708110,
126
+ "test_bleu": 0.05654392092884844,
127
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
128
  "bleu_samples": 250,
129
  "generated_tokens": 16000,
130
  "learning_rates": [
131
+ 0.0008547767072315835,
132
+ 8.547767072315835e-05,
133
+ 8.547767072315835e-05
134
  ],
135
+ "last_gradient_norm": 0.8050723671913147,
136
+ "training_seconds": 840.1531296785688,
137
+ "evaluation_seconds": 72.94383920903783,
138
+ "total_evaluation_seconds": 313.8235860019922,
139
+ "train_tokens_per_second": 15259.186149678188,
140
  "prediction_file": "predictions_03.jsonl"
141
  },
142
  {
143
  "optimizer": "muon",
144
  "step": 1044,
145
+ "successful_updates": 1044,
146
+ "skipped_updates": 0,
147
  "train_tokens": 17093404,
148
  "training_budget_tokens": 42733510,
149
  "selected_dataset_fraction": 0.4,
150
  "full_dataset_token_fraction": 0.10008393256046959,
151
  "tokens_per_parameter": 5.094886098768052,
152
+ "train_loss": 2.3242174275854817,
153
+ "train_perplexity": 10.218680098756618,
154
+ "cumulative_train_loss": 2.8267721660398504,
155
+ "validation_loss": 2.223032372531022,
156
+ "validation_perplexity": 9.235293297912508,
157
  "validation_tokens": 720136,
158
+ "test_loss": 2.2355781408321977,
159
+ "test_perplexity": 9.351886997309776,
160
  "test_tokens": 708110,
161
+ "test_bleu": 0.100897461835075,
162
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
163
  "bleu_samples": 250,
164
  "generated_tokens": 16000,
165
  "learning_rates": [
166
+ 0.0007307629410938362,
167
+ 7.307629410938362e-05,
168
+ 7.307629410938362e-05
169
  ],
170
+ "last_gradient_norm": 0.6698967218399048,
171
+ "training_seconds": 1115.7324636415578,
172
+ "evaluation_seconds": 77.46016829099972,
173
+ "total_evaluation_seconds": 391.28375429299194,
174
+ "train_tokens_per_second": 15320.342964844893,
175
  "prediction_file": "predictions_04.jsonl"
176
  },
177
  {
178
  "optimizer": "muon",
179
  "step": 1305,
180
+ "successful_updates": 1305,
181
+ "skipped_updates": 0,
182
  "train_tokens": 21366755,
183
  "training_budget_tokens": 42733510,
184
  "selected_dataset_fraction": 0.5,
185
  "full_dataset_token_fraction": 0.125104915700587,
186
  "tokens_per_parameter": 6.368607623460065,
187
+ "train_loss": 2.1430165465797297,
188
+ "train_perplexity": 8.525115287851063,
189
+ "cumulative_train_loss": 2.6900210421478263,
190
+ "validation_loss": 2.0409670501908668,
191
+ "validation_perplexity": 7.6980500012074335,
192
  "validation_tokens": 720136,
193
+ "test_loss": 2.0537061002671284,
194
+ "test_perplexity": 7.7967431393926105,
195
  "test_tokens": 708110,
196
+ "test_bleu": 0.12345122653106588,
197
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
198
  "bleu_samples": 250,
199
  "generated_tokens": 16000,
200
  "learning_rates": [
201
+ 0.0005871607054625496,
202
+ 5.8716070546254966e-05,
203
+ 5.8716070546254966e-05
204
  ],
205
+ "last_gradient_norm": 0.6055232286453247,
206
+ "training_seconds": 1406.0170264714397,
207
+ "evaluation_seconds": 78.36569400003646,
208
+ "total_evaluation_seconds": 469.6494482930284,
209
+ "train_tokens_per_second": 15196.654519627198,
210
  "prediction_file": "predictions_05.jsonl"
211
  },
212
  {
213
  "optimizer": "muon",
214
  "step": 1566,
215
+ "successful_updates": 1566,
216
+ "skipped_updates": 0,
217
  "train_tokens": 25640106,
218
  "training_budget_tokens": 42733510,
219
  "selected_dataset_fraction": 0.6,
220
  "full_dataset_token_fraction": 0.15012589884070437,
221
  "tokens_per_parameter": 7.642329148152078,
222
+ "train_loss": 1.9805903106765663,
223
+ "train_perplexity": 7.24701971584848,
224
+ "cumulative_train_loss": 2.5717825869026165,
225
+ "validation_loss": 1.9150039671325971,
226
+ "validation_perplexity": 6.786965721738856,
227
  "validation_tokens": 720136,
228
+ "test_loss": 1.9279956241901859,
229
+ "test_perplexity": 6.875714904050608,
230
  "test_tokens": 708110,
231
+ "test_bleu": 0.12877604487102923,
232
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
233
  "bleu_samples": 250,
234
  "generated_tokens": 16000,
235
  "learning_rates": [
236
+ 0.00043953153078664056,
237
+ 4.3953153078664054e-05,
238
+ 4.3953153078664054e-05
239
  ],
240
+ "last_gradient_norm": 0.8913306593894958,
241
+ "training_seconds": 1680.1966308491537,
242
+ "evaluation_seconds": 73.43593741592485,
243
+ "total_evaluation_seconds": 543.0853857089533,
244
+ "train_tokens_per_second": 15260.18177232135,
245
  "prediction_file": "predictions_06.jsonl"
246
  },
247
  {
248
  "optimizer": "muon",
249
  "step": 1827,
250
+ "successful_updates": 1827,
251
+ "skipped_updates": 0,
252
  "train_tokens": 29913457,
253
  "training_budget_tokens": 42733510,
254
  "selected_dataset_fraction": 0.7,
255
  "full_dataset_token_fraction": 0.1751468819808218,
256
  "tokens_per_parameter": 8.916050672844092,
257
+ "train_loss": 1.884803411397829,
258
+ "train_perplexity": 6.585059767065248,
259
+ "cumulative_train_loss": 2.473642704687647,
260
+ "validation_loss": 1.8385603145838079,
261
+ "validation_perplexity": 6.287479748988225,
262
  "validation_tokens": 720136,
263
+ "test_loss": 1.8502426677341355,
264
+ "test_perplexity": 6.361363032867387,
265
  "test_tokens": 708110,
266
+ "test_bleu": 0.1802229209222577,
267
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
268
  "bleu_samples": 250,
269
  "generated_tokens": 16000,
270
  "learning_rates": [
271
+ 0.00030387332884490803,
272
+ 3.0387332884490805e-05,
273
+ 3.0387332884490805e-05
274
  ],
275
+ "last_gradient_norm": 0.75834721326828,
276
+ "training_seconds": 1950.8896681739716,
277
+ "evaluation_seconds": 73.07733799994458,
278
+ "total_evaluation_seconds": 616.1627237088978,
279
+ "train_tokens_per_second": 15333.23872077242,
280
  "prediction_file": "predictions_07.jsonl"
281
  },
282
  {
283
  "optimizer": "muon",
284
  "step": 2088,
285
+ "successful_updates": 2088,
286
+ "skipped_updates": 0,
287
  "train_tokens": 34186808,
288
  "training_budget_tokens": 42733510,
289
  "selected_dataset_fraction": 0.8,
290
  "full_dataset_token_fraction": 0.20016786512093918,
291
  "tokens_per_parameter": 10.189772197536104,
292
+ "train_loss": 1.8216860253402747,
293
+ "train_perplexity": 6.182273136872596,
294
+ "cumulative_train_loss": 2.3921481197692254,
295
+ "validation_loss": 1.7927702604742368,
296
+ "validation_perplexity": 6.006067813606906,
297
  "validation_tokens": 720136,
298
+ "test_loss": 1.8042369539932885,
299
+ "test_perplexity": 6.075333920134396,
300
  "test_tokens": 708110,
301
+ "test_bleu": 0.2571039476796964,
302
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
303
  "bleu_samples": 250,
304
  "generated_tokens": 16000,
305
  "learning_rates": [
306
+ 0.00019488677077162295,
307
+ 1.9488677077162295e-05,
308
+ 1.9488677077162295e-05
309
  ],
310
+ "last_gradient_norm": 1.9932366609573364,
311
+ "training_seconds": 2219.8103655477753,
312
+ "evaluation_seconds": 77.00118562497664,
313
+ "total_evaluation_seconds": 693.1639093338745,
314
+ "train_tokens_per_second": 15400.778611808957,
315
  "prediction_file": "predictions_08.jsonl"
316
  },
317
  {
318
  "optimizer": "muon",
319
  "step": 2349,
320
+ "successful_updates": 2349,
321
+ "skipped_updates": 0,
322
  "train_tokens": 38460159,
323
  "training_budget_tokens": 42733510,
324
  "selected_dataset_fraction": 0.9,
325
  "full_dataset_token_fraction": 0.22518884826105656,
326
  "tokens_per_parameter": 11.463493722228117,
327
+ "train_loss": 1.7873370917559692,
328
+ "train_perplexity": 5.9735243210402365,
329
+ "cumulative_train_loss": 2.324946894434419,
330
+ "validation_loss": 1.7655405758426186,
331
+ "validation_perplexity": 5.844731024658187,
332
  "validation_tokens": 720136,
333
+ "test_loss": 1.7764862606588754,
334
+ "test_perplexity": 5.909057012052186,
335
  "test_tokens": 708110,
336
+ "test_bleu": 0.22627215440136425,
337
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
338
  "bleu_samples": 250,
339
  "generated_tokens": 16000,
340
  "learning_rates": [
341
+ 0.00012438224123471442,
342
+ 1.2438224123471442e-05,
343
+ 1.2438224123471442e-05
344
  ],
345
+ "last_gradient_norm": 0.6744703054428101,
346
+ "training_seconds": 2485.9783932112623,
347
+ "evaluation_seconds": 73.41921604203526,
348
+ "total_evaluation_seconds": 766.5831253759097,
349
+ "train_tokens_per_second": 15470.833980306279,
350
  "prediction_file": "predictions_09.jsonl"
351
  },
352
  {
353
  "optimizer": "muon",
354
  "step": 2610,
355
+ "successful_updates": 2610,
356
+ "skipped_updates": 0,
357
  "train_tokens": 42733510,
358
  "training_budget_tokens": 42733510,
359
  "selected_dataset_fraction": 1.0,
360
  "full_dataset_token_fraction": 0.250209831401174,
361
  "tokens_per_parameter": 12.73721524692013,
362
+ "train_loss": 1.7692924620928165,
363
+ "train_perplexity": 5.86670097923902,
364
+ "cumulative_train_loss": 2.2693814512002586,
365
+ "validation_loss": 1.747523807568987,
366
+ "validation_perplexity": 5.740370800119521,
367
  "validation_tokens": 720136,
368
+ "test_loss": 1.7582855103506165,
369
+ "test_perplexity": 5.802480568528633,
370
  "test_tokens": 708110,
371
+ "test_bleu": 0.24637446793816298,
372
  "bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
373
  "bleu_samples": 250,
374
  "generated_tokens": 16000,
375
  "learning_rates": [
376
+ 0.0001,
377
+ 1e-05,
378
+ 1e-05
379
  ],
380
+ "last_gradient_norm": 0.7609756588935852,
381
+ "training_seconds": 2754.439210804994,
382
+ "evaluation_seconds": 73.39668541704305,
383
+ "total_evaluation_seconds": 839.9798107929528,
384
+ "train_tokens_per_second": 15514.41390769012,
385
  "prediction_file": "predictions_10.jsonl"
386
  }
387
  ]
muon/muon.log CHANGED
The diff for this file is too large to render. See raw diff
 
muon/predictions_00.jsonl CHANGED
@@ -1,13 +1,13 @@
1
  {"doc_id": "spok_1146@gpt-4o-mini-2024-07-18", "prompt": "Hello Pete, and thank you for having me on this journey today. I'm thrilled to share my story with you and the audience. I bega", "reference": "n my career in recruitment much like many, inspired by a vision ", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
2
  {"doc_id": "blog_0819@gpt-4o-mini-2024-07-18", "prompt": "Why do they do this? It's as if the higher-ups are blind to the essence of what gatherings like this mean to people. The festiv", "reference": "al wasn't a den of iniquity; it was an oasis of community and cr", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
3
- {"doc_id": "acad_1084@chunk_2", "prompt": "The functional relevance of these resting-state networks (RSNs) can be inferred from their spatial congruence with networks of ", "reference": "regions co-activated in association with specific cognitive and ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_94><extra_id_77><extra_id_110>\u0003<extra_id_16><extra_id_58>㥦\u0017\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
4
  {"doc_id": "spok_0062@Meta-Llama-3-70B", "prompt": "We talked about networking and what that means and what that looks like and what her experience has been with networking and th", "reference": "en also what her experience has been with finding jobs online an", "prediction": "Թ<extra_id_51>\rx4<extra_id_25>\u001c&\u0012<extra_id_68>\u0017S<extra_id_27>)<extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>i2<extra_id_56><extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
5
  {"doc_id": "fic_1082@Meta-Llama-3-70B", "prompt": "The latter species attains greater height than the former, while the coastal variety is distinguished by its larger diameter, a", "reference": "lthough some specimens of both types exceed three hundred feet i", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
6
  {"doc_id": "fic_0946@chunk_2", "prompt": "Linda paused and waited, wondering whether she might be of any help. \"Doctor!\" gasped the woman, frantically. \"My baby swallowe", "reference": "d a pin! And I'm sure it's in her lungs now. She breathes so que", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
7
  {"doc_id": "acad_1085@chunk_1", "prompt": "Attention Deficit Hyperactivity Disorder (ADHD) is one of the most prevalent developmental disorders, affecting between 5 and 9", "reference": "% of school-aged children (Dittmann et al., 2009; Willcutt, 2012", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_27>)s<extra_id_33><extra_id_33><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
8
  {"doc_id": "acad_1085@Meta-Llama-3-8B", "prompt": "Since the NA system plays an important role in regulating arousal levels, it has been proposed that ATX may improve attention b", "reference": "y increasing arousal (Del Campo et al., 2011; Posner and Peterse", "prediction": "<extra_id_64>\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_107>P\u0017\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
9
  {"doc_id": "acad_0646@chunk_1", "prompt": "A plethora of studies in the economics/operational research literature is engaged with the measurement of producers' inefficien", "reference": "cy, thus relaxing the behavioral assumption that they are perfec", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
10
- {"doc_id": "news_0186@chunk_1", "prompt": "The suspected gunman . arrested for shooting a newspaper photographer in Paris had lived in Britain for several years before re", "reference": "turning to France in July, the Paris prosecutor says. Abdelhakim", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
11
  {"doc_id": "tvm_0398@chunk_2", "prompt": "Hey Holden, now my compass has gone crazy. It's spinning like a son of a bitch! Mine too, we must have hit a magnetic field or ", "reference": "something !! Mine too, we must have hit a magnetic field or some", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
12
  {"doc_id": "spok_1591@gpt-4o-2024-08-06", "prompt": "Every instinct screamed at me to move, to alert someone, to do anything but sit frozen in terror at the back of the bus. Yet, I", "reference": " was trapped in a spellbinding gaze with the creature-a gaze tha", "prediction": "<extra_id_16><extra_id_58>:Z\u001c\u0012\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
13
  {"doc_id": "fic_0027@chunk_2", "prompt": "Charmed with the discovery he followed his host's lead, and, with a languid air, replied to his rival in monosyllables. The cap", "reference": "tain watched with quiet satisfaction, and at each rebuff his opi", "prediction": "D\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
@@ -15,7 +15,7 @@
15
  {"doc_id": "tvm_0567@chunk_1", "prompt": "In order to show you how a big symphony orchestra is put together, Benjamin Britten has written a big piece of music, which is ", "reference": "made up of smaller pieces that show you all the separate parts o", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
16
  {"doc_id": "blog_0329@chunk_1", "prompt": "Tonight, my summer roommate Alex and I bonded over asian cuisine, broken hearts and a buried friend. I always knew we had poten", "reference": "tial. We have an assignment in Computer-Assisted Reporting class", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
17
  {"doc_id": "fic_1288@chunk_1", "prompt": "If you want views about the world's news, read what Cottswold Ommony calls the views papers; there is plenty in them that thoro", "reference": "ughly zealous people believe. But remember the wise old ambassad", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
18
- {"doc_id": "acad_1041@Meta-Llama-3-8B", "prompt": "On the contrary, WMS sequencing provides much larger amount of data allowing for deeper insights into microbiome structure and ", "reference": "function. As the complete set of DNA molecules present in the sa", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>"}
19
  {"doc_id": "news_0407@Meta-Llama-3-8B-Instruct", "prompt": "As the wedding days approached, Chelsea's anxiety about her hair grew. She had always been proud of her strawberry blonde locks", "reference": ", and the thought of losing them was daunting. But with the scal", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
20
  {"doc_id": "blog_0959@gpt-4o-mini-2024-07-18", "prompt": "4) We must prioritize education and vocational training specifically designed for individuals like myself, those who are physic", "reference": "ally handicapped, ensuring that we are not left behind in the pr", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
21
  {"doc_id": "acad_0444@gpt-4o-mini-2024-07-18", "prompt": "This variability in developmental timing is vital for the successful establishment of clonal lines, facilitating their adaptati", "reference": "on and survival in fluctuating environments. The plasticity inhe", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
@@ -31,7 +31,7 @@
31
  {"doc_id": "spok_0508@chunk_1", "prompt": "Hello, everyone. Today is yodh 9 12 15. Yodh also known as August 9th 2019. The reason I honor the old calendar is because I be", "reference": "lieve this the calendar that more honors Yahweh the Gregorian ca", "prediction": "b<extra_id_64><extra_id_115>\u001c\u0012\u0001\b9ú<extra_id_113><extra_id_107>P<extra_id_110><extra_id_88>Q;w\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
32
  {"doc_id": "acad_0567@gpt-4o-2024-08-06", "prompt": "The establishment of such procedures can be fundamentally linked to a deeper understanding of the decision-making context and t", "reference": "he intrinsic characteristics that govern it. A methodical approa", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_63>)XI<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
33
  {"doc_id": "tvm_0881@Meta-Llama-3-70B", "prompt": "In the Pentagon. You've got the wrong man. You're my only hope. The admiral laughs. You're a great friend, James Greer. What's ", "reference": "this all about? Did you bring the book? Yes. Good. We've been wo", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
34
- {"doc_id": "spok_1694@Meta-Llama-3-70B-Instruct", "prompt": "Yeah, sorry about that, folks. My bad. So, as I was saying, with the internet, fame has become more accessible. I mean, you can", "reference": " be famous for just about anything these days. You can be a soci", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>\u001f^<extra_id_101><extra_id_93>(<extra_id_115>\u001c&<extra_id_64>"}
35
  {"doc_id": "blog_1389@Meta-Llama-3-8B-Instruct", "prompt": "Here is the next 500 words in the same style and tone as the original text: I got off the train at Central Station, and walked ", "reference": "to the food court to grab some lunch. I was thinking about that ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
36
  {"doc_id": "news_0673@gpt-4o-mini-2024-07-18", "prompt": "Many students find themselves navigating the sometimes choppy waters of internship life, where the balance between goals and as", "reference": "signments can feel precarious at times. However, embracing the s", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
37
  {"doc_id": "spok_1589@chunk_1", "prompt": "All right, welcome to the show if we get started here. I just want to talk a little bit about anchor dot. F m -- and if you hav", "reference": "en't heard about anchor dot f m -- it's literally the easiest wa", "prediction": "C<extra_id_115>\u001c\rx\rx<extra_id_113><extra_id_4><extra_id_88>\u0019\u0000<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>i2<extra_id_56><extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
@@ -40,7 +40,7 @@
40
  {"doc_id": "acad_0258@chunk_2", "prompt": "For example, the production of packaging intended to come in contact with food and drink (known as food contact materials, FCMs", "reference": ") needs to comply with the EU food contact legislation (Regulati", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
41
  {"doc_id": "acad_0702@Meta-Llama-3-8B", "prompt": "While some scholars have highlighted the importance of communication (e.g., Bessant and Rush, 1995; Zahra and Covin, 1995), oth", "reference": "ers have questioned its relevance (e.g., Miller and Friesen, 198", "prediction": "Թ<extra_id_51>\rx\u001c\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_63>)XI<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
42
  {"doc_id": "tvm_0398@Meta-Llama-3-8B-Instruct", "prompt": "Here is the continuation of the text: Holden's voice was laced with panic as he frantically scanned the horizon. \"What the hell", "reference": " is going on, Skipper? We're lost!\" Stepek's gruff tone only add", "prediction": "<extra_id_16><extra_id_58>:Zﹷ<extra_id_51>\rx4<extra_id_25>\u001c\b<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017i2<extra_id_56>(<extra_id_115>\u001c&<extra_id_64>"}
43
- {"doc_id": "blog_0081@chunk_2", "prompt": "The views on the road weren't too picturesque, either. We should've taken the Gatwick Express , which costs pretty much the sam", "reference": "e; the saddest part is that the hotel is just a 2-minute walk fr", "prediction": "\u0012\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
44
  {"doc_id": "acad_0820@Meta-Llama-3-8B", "prompt": "Through this new representation, Muslim women have gained greater access to a wider range of resources, and the potential to es", "reference": "tablish their own businesses (Bakr, 2016; El-Ghobashy, 2011; She", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
45
  {"doc_id": "blog_1364@gpt-4o-2024-08-06", "prompt": "It reverberated through the air, words echoing off the walls as if carried by the wind itself: the force, the fervor, the raw p", "reference": "ower of his conviction. Sahir stood there, his presence filling ", "prediction": "\u001c\u0012\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
46
  {"doc_id": "spok_1587@Meta-Llama-3-8B", "prompt": "Okay, well, I think we should just take a step back. You don't have to worry about me. I'm going to be fine. I'm not worried ab", "reference": "out you. I'm worried about the podcast. If you don't mind. I'm g", "prediction": "<extra_id_64><extra_id_115>\u001c\u0001zlú<extra_id_113><extra_id_115>)<extra_id_122>s<extra_id_122>s<extra_id_122>s<extra_id_4><extra_id_110>i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
@@ -50,11 +50,11 @@
50
  {"doc_id": "fic_0923@Meta-Llama-3-8B-Instruct", "prompt": "As the young man continued to walk down Arcadia Street, he couldn't shake off the feeling of being out of place. The grimy faca", "reference": "des of the houses seemed to loom over him, their peeling paint a", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
51
  {"doc_id": "spok_0047@chunk_2", "prompt": "My parents are Mortal Kombat because I was like I came was like this shit and then afterwards I know my parents bake afterward ", "reference": "the newscaster like, you know, if you let your kids play Mortal ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
52
  {"doc_id": "acad_0077@Meta-Llama-3-70B-Instruct", "prompt": "The impact of human activities on the Earth's surface and subsurface is evident in the widespread occurrence of landslides, sub", "reference": "sidence, and sinkholes, often triggered or exacerbated by anthro", "prediction": "<extra_id_64>\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
53
- {"doc_id": "news_0727@chunk_2", "prompt": "I am already in Donetsk. A - Is it what I am thinking about? the one called 'M'? B - yes . A - 'PM'? B - yes, yes, yes. BUK. A ", "reference": "- oh, BM, yes, yes, yes. B - BUK . A - I understand . B - BUK, B", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zlú<extra_id_113><extra_id_115>)<extra_id_122>s<extra_id_122>\u001fZ\u0019\u0000^<extra_id_101>0i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
54
- {"doc_id": "spok_0496@Meta-Llama-3-70B-Instruct", "prompt": "...but it's likely that he had an inkling, given the rising tensions between Italy and Ethiopia. In October 1935, Italy invaded", "reference": " Ethiopia, and Amedeo's unit was deployed to the front lines. Th", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_94><extra_id_77><extra_id_110><extra_id_88>Q<extra_id_54>\u0017<extra_id_88>\u0017i2<extra_id_56>(<extra_id_115>(<extra_id_115><extra_id_64>"}
55
  {"doc_id": "blog_0710@Meta-Llama-3-70B", "prompt": "So, I figured that if I ate just a few bites, then threw the rest away, Jack wouldn't know any better. I started off slowly wit", "reference": "h some beans (which weren't bad) and then moved on to the pork. ", "prediction": "\u0019\u0000\u001aﹷ<extra_id_51>\rx4<extra_id_25>\u001c\b<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
56
- {"doc_id": "fic_0704@Meta-Llama-3-8B-Instruct", "prompt": "As the steamboat, or rather, the raft, drew near to the shore, Freddie Bobbsey called out to his sister, \"All aboard, Flossie! ", "reference": "Get on, get on!\" Flossie, with her blue eyes shining with excite", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P<extra_id_114>\u0017i2<extra_id_56>(<extra_id_115>\u001c&9(<extra_id_115>"}
57
- {"doc_id": "blog_0945@chunk_2", "prompt": "I thought they were cute but really thought nothing of it. They then kept looking our way, mind you Brad stopped headbanging an", "reference": "d was sitting on the bench with Phil and no one was in front of ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
58
  {"doc_id": "fic_0923@chunk_2", "prompt": "Still appearing utterly oblivious of the questions showered upon him by the now frantic man who had constituted himself as guid", "reference": "e, the young man had got midway up the street, and was still sea", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
59
  {"doc_id": "spok_1705@chunk_2", "prompt": "So if you can picture eight feet tall like the basket that you used to be able to dunk on when you're in like 8th grade because", "reference": " you couldn't get 10 that's almost how long his arms are which i", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
60
  {"doc_id": "acad_0521@chunk_1", "prompt": "A wide body of literature in archaeology and anthropology discusses the concept of craft specialisation. This topic is importan", "reference": "t because it contributes to understanding the development of soc", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
@@ -104,7 +104,7 @@
104
  {"doc_id": "acad_0483@Meta-Llama-3-8B", "prompt": "A survey of 5000 UK residents found that perceptions of visual amenity were more negative among those who had previously oppose", "reference": "d onshore wind farms, even if they lived further away from exist", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
105
  {"doc_id": "spok_1550@chunk_1", "prompt": "Wrong, so wrong by c-pillar 42 read by litter Aryan. summary There is a kiss and it is so wrong. It turns out that even after k", "reference": "nowing each other for about six thousand years. It is possible t", "prediction": "U<extra_id_29><extra_id_120><extra_id_115>\u001c\u0012\u0001\b9ú<extra_id_113><extra_id_16><extra_id_58>:<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0i(<extra_id_119>C<extra_id_107>P*\rx<extra_id_63>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
106
  {"doc_id": "news_0483@Meta-Llama-3-8B", "prompt": "'We are working hard to minimise any disruption to our customers and will contact them directly if their flight has been affect", "reference": "ed.' BA added that it was continuing to monitor the situation cl", "prediction": "\u0019\u0000\u001aU\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
107
- {"doc_id": "blog_0065@gpt-4o-mini-2024-07-18", "prompt": "Not for You - A powerful track from Vitalogy, it echoes the frustrations of feeling trapped within the confines of expectation.", "reference": " Eddie's emotive delivery resonated deeply, especially during th", "prediction": "<extra_id_119><extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
108
  {"doc_id": "spok_0558@chunk_1", "prompt": "Hi everyone, and welcome to this incessant home tomorrow. I am like internally screaming right now. This is a podcast about thi", "reference": "ngs that just make isn't comfortable and that we don't want to t", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
109
  {"doc_id": "spok_1215@chunk_1", "prompt": "Welcome to the good Christian talks podcast. I'm Levi and I'm Chris. Thank you so much for joining us this week on this podcast", "reference": ". We select one taco week to help each one of us get the Bible i", "prediction": "\u0019\u0000\u001aﹷ<extra_id_51>\rx4<extra_id_25>\u001c\b<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>i2<extra_id_56>(<extra_id_115>)I<extra_id_16><extra_id_58><extra_id_25><extra_id_64>"}
110
  {"doc_id": "acad_0525@gpt-4o-mini-2024-07-18", "prompt": "This observation suggests a multifaceted role for p53 in modulating not only the cellular response to DNA damage but also the m", "reference": "etabolism of environmental carcinogens, thereby influencing susc", "prediction": "\u0012\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
@@ -115,15 +115,15 @@
115
  {"doc_id": "blog_0358@gpt-4o-2024-08-06", "prompt": "67-year-old S Puhaindran was a young teacher in Raffles Institution when Mr Goh was a student there. \"I remember him well,\" Mr.", "reference": " Puhaindran recounted with an affectionate smile. \"Goh was the k", "prediction": "<extra_id_119><extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_63>)XI<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
116
  {"doc_id": "spok_0718@chunk_2", "prompt": "Welcome to our second of two episodes on John T Scopes last week We examined the rise of the anti-evolution movement in the ear", "reference": "ly 1920s that led to the establishment of the butler act which b", "prediction": "<extra_id_25><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
117
  {"doc_id": "spok_1582@chunk_1", "prompt": "Hello everyone and welcome to the Notre grow podcast. I'm your host. Dr. Ivan Khan, and we'll be breaking down topics around ed", "reference": "ucation growth and culture with the intention to help your growt", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
118
- {"doc_id": "blog_1429@Meta-Llama-3-70B-Instruct", "prompt": "As I sat in the staff meeting, listening to our administrators outline the plan for the year, I couldn't help but feel a sense ", "reference": "of excitement and trepidation. The idea of \"writing it down and ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001z<extra_id_111>\u001c\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
119
  {"doc_id": "tvm_0327@chunk_2", "prompt": "Dead serious. See... the world is made up of rules. Kids don't like rules. They rebel for a while, but ultimately they start to", "reference": " fall in line and become ordinary Joe's like you and me. But som", "prediction": "C<extra_id_115>\u001c\u0012\u0001\b9ú<extra_id_113><extra_id_107>P<extra_id_110><extra_id_88>Q;w\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107>P*\rx<extra_id_63>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
120
  {"doc_id": "tvm_0857@gpt-4o-mini-2024-07-18", "prompt": "With silent determination, MALONE casts an assured glance at the OARSMEN, who hesitate for a moment, caught between the ardor o", "reference": "f loyalty and the dread of what lies ahead. One by one, they asc", "prediction": "C\u001c&<extra_id_121>\u0001\b9ú<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
121
  {"doc_id": "blog_0293@chunk_1", "prompt": "Michael posted more about abortion and guns on his website and, as a woman, I violentely reacted to some of his comments on abo", "reference": "rtion. My reaction was too long to post on his blog, so I'm doin", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
122
- {"doc_id": "news_0530@Meta-Llama-3-8B-Instruct", "prompt": "As the controversy surrounding the botched trial of John Downey continued to simmer, the UK government's refusal to launch a ju", "reference": "dicial inquiry into the matter only added fuel to the fire. Nort", "prediction": "x<extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>"}
123
  {"doc_id": "spok_0772@Meta-Llama-3-8B-Instruct", "prompt": "Here's the next 500 words in the same style and tone: David: Yeah, okay, let's move on to the next question. At wonderland unde", "reference": "rscore underscore, what's the weirdest thing that's happened to ", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
124
  {"doc_id": "news_0390@chunk_2", "prompt": "For example, Benson said that $1 million in $100 bills weighs 22 pounds, while $1 million in 500 euro notes weighs 3.5 pounds. ", "reference": "\"It's a huge difference,\" he said. Though Europe is highly attra", "prediction": "<extra_id_115><extra_id_115>\u001c\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
125
  {"doc_id": "spok_0704@gpt-4o-2024-08-06", "prompt": "You see, creativity intertwines itself with the rituals that one builds around it. For me, it's not so much about adhering to a", "reference": " strict schedule, as it is about weaving those moments of intros", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
126
- {"doc_id": "news_0483@Meta-Llama-3-8B-Instruct", "prompt": "As the storm continues to rage on the East Coast, the travel chaos shows no signs of abating. Thousands of Britons are facing a", "reference": "nother day of uncertainty, with airlines scrambling to rebook fl", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_4><extra_id_88>Q\b\u0017S<extra_id_27>)<extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
127
  {"doc_id": "fic_1229@chunk_2", "prompt": "Phillips, his valet, had heard the shot, and knowing that the prince's cousin, Prince Miguel, with Don Solado, prime minister o", "reference": "f Joyalita, had made attempts on his life before, in New York, h", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0\u0014\u0017i2<extra_id_56>(<extra_id_115><extra_id_64>\u0017\u000b"}
128
  {"doc_id": "fic_0555@gpt-4o-2024-08-06", "prompt": "The memories of carefree childhood days woven amidst the tranquil charm of the estate danced in his mind, anchoring him to this", "reference": " beloved place with an unyielding fondness. The familiar sound o", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
129
  {"doc_id": "blog_1347@Meta-Llama-3-70B-Instruct", "prompt": "As I gazed deeper into the mirror, my eyes seemed to bore into my own soul. The reflection staring back at me was a stranger, y", "reference": "et eerily familiar. The lines on my face told a story of pain an", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\b9ú<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
@@ -135,7 +135,7 @@
135
  {"doc_id": "news_0135@gpt-4o-mini-2024-07-18", "prompt": "However, the events of last October reshaped those timelines and priorities, pushing Virgin Galactic into a more insular approa", "reference": "ch to the development of their space ambitions. The firm recogni", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
136
  {"doc_id": "acad_0489@Meta-Llama-3-8B", "prompt": "Individual hackers may hack into websites or online services to deface content, change information, disrupt service, or simply ", "reference": "out of curiosity.16 Lastly, activist groups may target critical ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
137
  {"doc_id": "news_0246@Meta-Llama-3-70B", "prompt": "That might seem an unlikely trio of activities for a man preparing for a world title fight but there were two serious reasons f", "reference": "or Manny's trip here. One was to promote his forthcoming book, a", "prediction": "<extra_id_16><extra_id_58>:Z\u001c\u0012\u0001\b\u0017\u0017*<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
138
- {"doc_id": "acad_0497@gpt-4o-mini-2024-07-18", "prompt": "Moreover, the limitations inherent within the traditional diagnostic framework underscore a pressing need for an integrative, t", "reference": "ransdiagnostic approach to emotional disorders that recognizes a", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_64>"}
139
  {"doc_id": "spok_1425@chunk_1", "prompt": "Even when you say listen, we might not be getting billboard this time around because you're a brand new artist. They don't nece", "reference": "ssarily hear that. So first off I just want to say welcome to th", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
140
  {"doc_id": "blog_0578@Meta-Llama-3-8B-Instruct", "prompt": "As I reflect on my childhood friendships, I'm reminded of the many memories I've accumulated over the years. One of my earliest", "reference": " and most cherished friendships was with Rachel. We met in kinde", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
141
  {"doc_id": "spok_1607@Meta-Llama-3-70B", "prompt": "We have and if you haven't had that experience yet, then I would definitely suggest you start paying attention to how you feel ", "reference": "and learning to be in tune with yourself and your intuition. So ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
@@ -161,7 +161,7 @@
161
  {"doc_id": "tvm_0962@Meta-Llama-3-70B-Instruct", "prompt": "Walter's eyes dart around the elevator, searching for an escape from the uncomfortable silence. Ted's gaze lingers, as if darin", "reference": "g Walter to ask again. Finally, he responds, his tone measured, ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_94><extra_id_77><extra_id_110>\u0003<extra_id_16><extra_id_58>㥦\u0017\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
162
  {"doc_id": "acad_0646@gpt-4o-2024-08-06", "prompt": "These studies employed Geographic Information Systems (GIS) to accurately delineate spatial relationships and pinpoint geograph", "reference": "ical proximities between producers, allowing for a nuanced analy", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
163
  {"doc_id": "acad_0772@chunk_2", "prompt": "However, the pressure signal of the catheter is affected by the viscosity of the colonic fluids [28]. Arkwright et al. [28] sho", "reference": "wed that the predetermined amplitude of the applied pressure on ", "prediction": "C\u001c&<extra_id_121>\u0001\b\u0017\u0017*<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
164
- {"doc_id": "news_0656@gpt-4o-mini-2024-07-18", "prompt": "In facing this precarious situation, the possibility of requiring a financial rescue looms large on Italy's horizon, though man", "reference": "y experts argue it lacks the structural and political framework ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
165
  {"doc_id": "spok_0704@Meta-Llama-3-8B", "prompt": "That's really cool. I think that's great. I mean, I'm not judging anyone or anything, but it's interesting because I remember r", "reference": "eading an article by Tony Robbins and he said that he stopped dr", "prediction": "<extra_id_25><extra_id_115>\u001cﹷ\u0017\u0017\u0017\u0017*<extra_id_16><extra_id_58>:\u0017S<extra_id_27>)<extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
166
  {"doc_id": "news_1257@gpt-4o-2024-08-06", "prompt": "The goal is to prevent Iran from refining uranium beyond a certain threshold, drastically reducing its capacity to progress tow", "reference": "ard a nuclear weapon in the near term. In exchange, Iran will re", "prediction": "\u0017\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
167
  {"doc_id": "spok_1443@Meta-Llama-3-8B", "prompt": "They said they couldn't stand the smell of your dead cat anymore. So they took off. You've been living alone since 1999 and no ", "reference": "one has heard from them again. I guess they just went back home ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zlú<extra_id_113><extra_id_115>)<extra_id_122>s<extra_id_122>s<extra_id_122>s<extra_id_33><extra_id_33><extra_id_33><extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
@@ -179,7 +179,7 @@
179
  {"doc_id": "blog_0126@chunk_2", "prompt": "What took you so long? God help you become victorious. I want to say hello to Bush, to shake his hand. We came out of the grave", "reference": ".'\" The Guadian, 3-22-03 \"As hundreds of coalition troops swept ", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P<extra_id_114>\u0017i2<extra_id_56>(<extra_id_115>\u001c&9(<extra_id_115>"}
180
  {"doc_id": "fic_0051@chunk_2", "prompt": "Two seamen, each pushing an oar before him, swam out to the nearest boat, and having got on board of her, and freed her from he", "reference": "r entanglements, they rowed out to the capsized boat, and towed ", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
181
  {"doc_id": "tvm_0383@chunk_1", "prompt": "How do you do. My name is Penelope Ryan. This is a simple-minded play about men who enjoy killing -- and those who don't. I am ", "reference": "Harold Ryan, her husband. I have killed perhaps two hundred men ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
182
- {"doc_id": "acad_0983@Meta-Llama-3-8B", "prompt": "Understanding how much water is available for plants in a given ecosystem is critical for managing natural resources and assess", "reference": "ing the impact of land-use/management practices on water availab", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
183
  {"doc_id": "fic_0034@Meta-Llama-3-70B", "prompt": "The ears were short and delicately made; the eyes were well-placed; and the distance to the angle of the jaw was such as good b", "reference": "reeders desire. Considering him closer she became aware of the l", "prediction": "<extra_id_64>\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
184
  {"doc_id": "blog_0069@chunk_1", "prompt": "..Is the one, which tells you immediately, what the outcome will be: Good food, bad husband, horror able dinner, acing a test, ", "reference": "bad sex, insanely demented neighbours... the last thing is an is", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
185
  {"doc_id": "blog_0747@gpt-4o-2024-08-06", "prompt": "I finally reached Plaza Singapura, and there was Darius, waiting patiently with a grin. The moment I saw him, I felt a little f", "reference": "lutter in my stomach. It's funny how a net friend can feel so fa", "prediction": "<extra_id_16><extra_id_58>:Z\u001c\u0012\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
@@ -199,12 +199,12 @@
199
  {"doc_id": "acad_0807@gpt-4o-2024-08-06", "prompt": "As a result, the development of more adaptive operational strategies that can detect and respond to ice formation is paramount.", "reference": " Integrating advanced sensor technology can offer a real-time as", "prediction": "<extra_id_119><extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
200
  {"doc_id": "fic_0975@Meta-Llama-3-70B", "prompt": "\"Well, what does the chief say, sir?\" he asked. \"He doesn't seem very cheerful over it.\" \"He isn't,\" replied Ramshaw grimly. \"S", "reference": "ome scoundrel has tampered with the high-pressure slide valves. ", "prediction": "<extra_id_32>÷\u0017\u0017\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
201
  {"doc_id": "spok_1483@Meta-Llama-3-8B-Instruct", "prompt": "But we're not going to be bored, because we're going to dive into the story of the 1974 Zaire team and explore the complexities", "reference": " of colonialism and its lasting impact on the African continent.", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
202
- {"doc_id": "fic_0841@chunk_1", "prompt": "One evening Mr. Courtnay Fox, the London correspondent of the Edinburgh Chronicle, was as usual in his own room in the office i", "reference": "n Fleet-street, when a card was brought to him. \"Show the gentle", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_16><extra_id_58>:<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
203
  {"doc_id": "news_0227@chunk_2", "prompt": "We've been writers in our own right for several years before we started to write together, and from the beginning we thought it", "reference": " was a great idea to stay secret behind a pen name. Then two thi", "prediction": "\u0019\u0000\u001aU\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_94><extra_id_77><extra_id_110>\u0003<extra_id_16><extra_id_58>Һ<extra_id_113>\u0017i2<extra_id_56>(<extra_id_115>\u001c&9(<extra_id_115>"}
204
  {"doc_id": "tvm_0327@Meta-Llama-3-70B", "prompt": "It's the humane thing to do. We don't want them to suffer. You know how hard it is to find a decent lawyer these days? This is ", "reference": "the place? Yeah, I think so. Come on, Lizzy. Let's check it out.", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zlú<extra_id_113><extra_id_115>)<extra_id_122>s<extra_id_122>\u001fZ\u0019\u0000^<extra_id_101>0i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
205
  {"doc_id": "blog_0026@chunk_1", "prompt": "On Monday morning at work, I slipped on a wet floor. Mondays are always heavy load days. Mondays after a holiday... ew. Twelve ", "reference": "dogs to do. And they wouldn't let me wash some of them a day ear", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
206
  {"doc_id": "news_0187@gpt-4o-2024-08-06", "prompt": "Jimmy was grateful for Claire's presence and quickly found that her vivacious spirit brightened the household. Even Gemma, who ", "reference": "had her reservations at first about her father's burgeoning rela", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
207
- {"doc_id": "fic_1122@Meta-Llama-3-8B-Instruct", "prompt": "As the days passed, the anticipation grew, and the village was abuzz with excitement. The two suitors, myself and my rival, wer", "reference": "e summoned to the farmer's estate, where we were to present ours", "prediction": "<extra_id_25><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
208
  {"doc_id": "news_0530@gpt-4o-mini-2024-07-18", "prompt": "In a shocking escalation of political tension, Mr. Robinson's outrage reverberated throughout Northern Ireland's tumultuous lan", "reference": "dscape, reigniting old wounds and fears among the populace. The ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
209
  {"doc_id": "news_0251@chunk_1", "prompt": "Experts were unable to open the 9th century Carolingian pot (pictured), which was found on church land in Dumfries and Galloway", "reference": " in September . The mystery surrounding the contents of a Viking", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
210
  {"doc_id": "spok_1103@chunk_2", "prompt": "It was a questionable mix at that point in history. One member of the crowd tried to storm the stage, but he was quickly subdue", "reference": "d by security after he managed to break one of the presentation ", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
@@ -222,7 +222,7 @@
222
  {"doc_id": "tvm_0420@gpt-4o-2024-08-06", "prompt": "We simply critique the cult that surrounds them. Sirens wail in the distance, and the thrum of anticipation grows. Young is sti", "reference": "ll spinning his tale, unaware or uncaring of the chaos, maintain", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
223
  {"doc_id": "news_1166@gpt-4o-2024-08-06", "prompt": "But rather than dismissing this sentiment as merely the nostalgic ramblings of a bygone generation, it might serve us well to d", "reference": "elve deeper into the root of such concerns. The discourse surrou", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_107>P<extra_id_110><extra_id_88>Q;w\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107>P*\rx<extra_id_63>\u001f^<extra_id_101><extra_id_93>(<extra_id_115>\u001c&<extra_id_64>"}
224
  {"doc_id": "spok_1150@chunk_2", "prompt": "And they have a new proprietary blend which is their metabolic ferment, which I can't explain to you the science behind it, but", "reference": " I will tell you that it's supposed to help boost your natural c", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_16><extra_id_58>:<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
225
- {"doc_id": "blog_0026@gpt-4o-2024-08-06", "prompt": "I slipped, my feet skidding out from under me like an awkward rendition of a cartoon character slipping on a banana peel. As I ", "reference": "tumbled, I could almost hear the universe snickering. My backsid", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
226
  {"doc_id": "blog_0819@Meta-Llama-3-8B-Instruct", "prompt": "You see, it's all about power, man. The War on Drugs is just a thinly veiled excuse for the government to exert its control ove", "reference": "r our lives. They're like, \"Oh, we're doing this to protect you ", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
227
  {"doc_id": "news_0251@chunk_2", "prompt": "A scan of the Carolingian pot was conducted by Dr John Reid, consultant radiographer at the BGH, who is also a keen amateur arc", "reference": "haeologist. He was approached by Richard Welander, head of colle", "prediction": "i\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017i2<extra_id_56>(<extra_id_115>\u001c&<extra_id_64>"}
228
  {"doc_id": "fic_0017@gpt-4o-2024-08-06", "prompt": "He was a man of rigorous discipline, a stalwart presence in the realm of naval journalism, and one who carried the weight of hi", "reference": "s responsibilities with a steely resolve. His eyes, accustomed t", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_4><extra_id_88>\u0019\u0000<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
@@ -241,7 +241,7 @@
241
  {"doc_id": "fic_0964@chunk_2", "prompt": "The preacher, half amused, half embarrassed, stood still, as a woman came out of a tiny hut near by, and peered about her with ", "reference": "angry, short-sighted eyes. Mrs. Brazybone was a vast woman, with", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
242
  {"doc_id": "blog_0946@Meta-Llama-3-8B-Instruct", "prompt": "I'm still trying to process everything that's going on. I feel like I'm stuck in some kind of never-ending nightmare. My dad's ", "reference": "financial situation is getting worse and worse, and I'm starting", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
243
  {"doc_id": "blog_1429@Meta-Llama-3-8B", "prompt": "It was a great experience! And we were very happy with the outcome. However, when we came back from our break and met again yes", "reference": "terday, I felt like there was something missing. We had a lot of", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_16><extra_id_58><extra_id_25><extra_id_16><extra_id_58>&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
244
- {"doc_id": "blog_1429@chunk_2", "prompt": "Out of those 3 days, our staff formed into 'action teams' to work together the rest of the year on specific areas we felt neede", "reference": "d redefining, exploration, clarity . I chose the numeracy group.", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P<extra_id_114>\u0017i2<extra_id_56>(<extra_id_115>\u001c&9(<extra_id_115>"}
245
  {"doc_id": "fic_1250@Meta-Llama-3-8B-Instruct", "prompt": "As he descended into the canyon, the silence was broken only by the soft rustling of leaves and the gentle crunch of gravel ben", "reference": "eath his hooves. Spread Horn's ears twitched ever so slightly, a", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
246
  {"doc_id": "fic_0546@Meta-Llama-3-8B", "prompt": "It was evident that she was used to asking questions, and receiving answers, and that she did not think it worth while to show ", "reference": "emotion about either process. She laid down the paper, and took ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
247
  {"doc_id": "spok_1404@chunk_1", "prompt": "Is take it away ryos extra. Can I call you Riles? Yeah, so cute as well my friends. Call me. How you that? All right. Well my e", "reference": "lementary school friends. No one here does except for trials. Oh", "prediction": "b<extra_id_64>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_16><extra_id_16><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
 
1
  {"doc_id": "spok_1146@gpt-4o-mini-2024-07-18", "prompt": "Hello Pete, and thank you for having me on this journey today. I'm thrilled to share my story with you and the audience. I bega", "reference": "n my career in recruitment much like many, inspired by a vision ", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
2
  {"doc_id": "blog_0819@gpt-4o-mini-2024-07-18", "prompt": "Why do they do this? It's as if the higher-ups are blind to the essence of what gatherings like this mean to people. The festiv", "reference": "al wasn't a den of iniquity; it was an oasis of community and cr", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
3
+ {"doc_id": "acad_1084@chunk_2", "prompt": "The functional relevance of these resting-state networks (RSNs) can be inferred from their spatial congruence with networks of ", "reference": "regions co-activated in association with specific cognitive and ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
4
  {"doc_id": "spok_0062@Meta-Llama-3-70B", "prompt": "We talked about networking and what that means and what that looks like and what her experience has been with networking and th", "reference": "en also what her experience has been with finding jobs online an", "prediction": "Թ<extra_id_51>\rx4<extra_id_25>\u001c&\u0012<extra_id_68>\u0017S<extra_id_27>)<extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>i2<extra_id_56><extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
5
  {"doc_id": "fic_1082@Meta-Llama-3-70B", "prompt": "The latter species attains greater height than the former, while the coastal variety is distinguished by its larger diameter, a", "reference": "lthough some specimens of both types exceed three hundred feet i", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
6
  {"doc_id": "fic_0946@chunk_2", "prompt": "Linda paused and waited, wondering whether she might be of any help. \"Doctor!\" gasped the woman, frantically. \"My baby swallowe", "reference": "d a pin! And I'm sure it's in her lungs now. She breathes so que", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
7
  {"doc_id": "acad_1085@chunk_1", "prompt": "Attention Deficit Hyperactivity Disorder (ADHD) is one of the most prevalent developmental disorders, affecting between 5 and 9", "reference": "% of school-aged children (Dittmann et al., 2009; Willcutt, 2012", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_27>)s<extra_id_33><extra_id_33><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
8
  {"doc_id": "acad_1085@Meta-Llama-3-8B", "prompt": "Since the NA system plays an important role in regulating arousal levels, it has been proposed that ATX may improve attention b", "reference": "y increasing arousal (Del Campo et al., 2011; Posner and Peterse", "prediction": "<extra_id_64>\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_107>P\u0017\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
9
  {"doc_id": "acad_0646@chunk_1", "prompt": "A plethora of studies in the economics/operational research literature is engaged with the measurement of producers' inefficien", "reference": "cy, thus relaxing the behavioral assumption that they are perfec", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
10
+ {"doc_id": "news_0186@chunk_1", "prompt": "The suspected gunman . arrested for shooting a newspaper photographer in Paris had lived in Britain for several years before re", "reference": "turning to France in July, the Paris prosecutor says. Abdelhakim", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`<extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
11
  {"doc_id": "tvm_0398@chunk_2", "prompt": "Hey Holden, now my compass has gone crazy. It's spinning like a son of a bitch! Mine too, we must have hit a magnetic field or ", "reference": "something !! Mine too, we must have hit a magnetic field or some", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
12
  {"doc_id": "spok_1591@gpt-4o-2024-08-06", "prompt": "Every instinct screamed at me to move, to alert someone, to do anything but sit frozen in terror at the back of the bus. Yet, I", "reference": " was trapped in a spellbinding gaze with the creature-a gaze tha", "prediction": "<extra_id_16><extra_id_58>:Z\u001c\u0012\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
13
  {"doc_id": "fic_0027@chunk_2", "prompt": "Charmed with the discovery he followed his host's lead, and, with a languid air, replied to his rival in monosyllables. The cap", "reference": "tain watched with quiet satisfaction, and at each rebuff his opi", "prediction": "D\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
 
15
  {"doc_id": "tvm_0567@chunk_1", "prompt": "In order to show you how a big symphony orchestra is put together, Benjamin Britten has written a big piece of music, which is ", "reference": "made up of smaller pieces that show you all the separate parts o", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
16
  {"doc_id": "blog_0329@chunk_1", "prompt": "Tonight, my summer roommate Alex and I bonded over asian cuisine, broken hearts and a buried friend. I always knew we had poten", "reference": "tial. We have an assignment in Computer-Assisted Reporting class", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
17
  {"doc_id": "fic_1288@chunk_1", "prompt": "If you want views about the world's news, read what Cottswold Ommony calls the views papers; there is plenty in them that thoro", "reference": "ughly zealous people believe. But remember the wise old ambassad", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
18
+ {"doc_id": "acad_1041@Meta-Llama-3-8B", "prompt": "On the contrary, WMS sequencing provides much larger amount of data allowing for deeper insights into microbiome structure and ", "reference": "function. As the complete set of DNA molecules present in the sa", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
19
  {"doc_id": "news_0407@Meta-Llama-3-8B-Instruct", "prompt": "As the wedding days approached, Chelsea's anxiety about her hair grew. She had always been proud of her strawberry blonde locks", "reference": ", and the thought of losing them was daunting. But with the scal", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
20
  {"doc_id": "blog_0959@gpt-4o-mini-2024-07-18", "prompt": "4) We must prioritize education and vocational training specifically designed for individuals like myself, those who are physic", "reference": "ally handicapped, ensuring that we are not left behind in the pr", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
21
  {"doc_id": "acad_0444@gpt-4o-mini-2024-07-18", "prompt": "This variability in developmental timing is vital for the successful establishment of clonal lines, facilitating their adaptati", "reference": "on and survival in fluctuating environments. The plasticity inhe", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
 
31
  {"doc_id": "spok_0508@chunk_1", "prompt": "Hello, everyone. Today is yodh 9 12 15. Yodh also known as August 9th 2019. The reason I honor the old calendar is because I be", "reference": "lieve this the calendar that more honors Yahweh the Gregorian ca", "prediction": "b<extra_id_64><extra_id_115>\u001c\u0012\u0001\b9ú<extra_id_113><extra_id_107>P<extra_id_110><extra_id_88>Q;w\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
32
  {"doc_id": "acad_0567@gpt-4o-2024-08-06", "prompt": "The establishment of such procedures can be fundamentally linked to a deeper understanding of the decision-making context and t", "reference": "he intrinsic characteristics that govern it. A methodical approa", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_63>)XI<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
33
  {"doc_id": "tvm_0881@Meta-Llama-3-70B", "prompt": "In the Pentagon. You've got the wrong man. You're my only hope. The admiral laughs. You're a great friend, James Greer. What's ", "reference": "this all about? Did you bring the book? Yes. Good. We've been wo", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
34
+ {"doc_id": "spok_1694@Meta-Llama-3-70B-Instruct", "prompt": "Yeah, sorry about that, folks. My bad. So, as I was saying, with the internet, fame has become more accessible. I mean, you can", "reference": " be famous for just about anything these days. You can be a soci", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>i2<extra_id_56>(<extra_id_115>)I<extra_id_114>(<extra_id_115>"}
35
  {"doc_id": "blog_1389@Meta-Llama-3-8B-Instruct", "prompt": "Here is the next 500 words in the same style and tone as the original text: I got off the train at Central Station, and walked ", "reference": "to the food court to grab some lunch. I was thinking about that ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
36
  {"doc_id": "news_0673@gpt-4o-mini-2024-07-18", "prompt": "Many students find themselves navigating the sometimes choppy waters of internship life, where the balance between goals and as", "reference": "signments can feel precarious at times. However, embracing the s", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
37
  {"doc_id": "spok_1589@chunk_1", "prompt": "All right, welcome to the show if we get started here. I just want to talk a little bit about anchor dot. F m -- and if you hav", "reference": "en't heard about anchor dot f m -- it's literally the easiest wa", "prediction": "C<extra_id_115>\u001c\rx\rx<extra_id_113><extra_id_4><extra_id_88>\u0019\u0000<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>i2<extra_id_56><extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
 
40
  {"doc_id": "acad_0258@chunk_2", "prompt": "For example, the production of packaging intended to come in contact with food and drink (known as food contact materials, FCMs", "reference": ") needs to comply with the EU food contact legislation (Regulati", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
41
  {"doc_id": "acad_0702@Meta-Llama-3-8B", "prompt": "While some scholars have highlighted the importance of communication (e.g., Bessant and Rush, 1995; Zahra and Covin, 1995), oth", "reference": "ers have questioned its relevance (e.g., Miller and Friesen, 198", "prediction": "Թ<extra_id_51>\rx\u001c\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_63>)XI<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
42
  {"doc_id": "tvm_0398@Meta-Llama-3-8B-Instruct", "prompt": "Here is the continuation of the text: Holden's voice was laced with panic as he frantically scanned the horizon. \"What the hell", "reference": " is going on, Skipper? We're lost!\" Stepek's gruff tone only add", "prediction": "<extra_id_16><extra_id_58>:Zﹷ<extra_id_51>\rx4<extra_id_25>\u001c\b<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017i2<extra_id_56>(<extra_id_115>\u001c&<extra_id_64>"}
43
+ {"doc_id": "blog_0081@chunk_2", "prompt": "The views on the road weren't too picturesque, either. We should've taken the Gatwick Express , which costs pretty much the sam", "reference": "e; the saddest part is that the hotel is just a 2-minute walk fr", "prediction": "\u0012\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
44
  {"doc_id": "acad_0820@Meta-Llama-3-8B", "prompt": "Through this new representation, Muslim women have gained greater access to a wider range of resources, and the potential to es", "reference": "tablish their own businesses (Bakr, 2016; El-Ghobashy, 2011; She", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
45
  {"doc_id": "blog_1364@gpt-4o-2024-08-06", "prompt": "It reverberated through the air, words echoing off the walls as if carried by the wind itself: the force, the fervor, the raw p", "reference": "ower of his conviction. Sahir stood there, his presence filling ", "prediction": "\u001c\u0012\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
46
  {"doc_id": "spok_1587@Meta-Llama-3-8B", "prompt": "Okay, well, I think we should just take a step back. You don't have to worry about me. I'm going to be fine. I'm not worried ab", "reference": "out you. I'm worried about the podcast. If you don't mind. I'm g", "prediction": "<extra_id_64><extra_id_115>\u001c\u0001zlú<extra_id_113><extra_id_115>)<extra_id_122>s<extra_id_122>s<extra_id_122>s<extra_id_4><extra_id_110>i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
 
50
  {"doc_id": "fic_0923@Meta-Llama-3-8B-Instruct", "prompt": "As the young man continued to walk down Arcadia Street, he couldn't shake off the feeling of being out of place. The grimy faca", "reference": "des of the houses seemed to loom over him, their peeling paint a", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
51
  {"doc_id": "spok_0047@chunk_2", "prompt": "My parents are Mortal Kombat because I was like I came was like this shit and then afterwards I know my parents bake afterward ", "reference": "the newscaster like, you know, if you let your kids play Mortal ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
52
  {"doc_id": "acad_0077@Meta-Llama-3-70B-Instruct", "prompt": "The impact of human activities on the Earth's surface and subsurface is evident in the widespread occurrence of landslides, sub", "reference": "sidence, and sinkholes, often triggered or exacerbated by anthro", "prediction": "<extra_id_64>\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
53
+ {"doc_id": "news_0727@chunk_2", "prompt": "I am already in Donetsk. A - Is it what I am thinking about? the one called 'M'? B - yes . A - 'PM'? B - yes, yes, yes. BUK. A ", "reference": "- oh, BM, yes, yes, yes. B - BUK . A - I understand . B - BUK, B", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0i(<extra_id_119>C<extra_id_107>P*\rx<extra_id_63>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
54
+ {"doc_id": "spok_0496@Meta-Llama-3-70B-Instruct", "prompt": "...but it's likely that he had an inkling, given the rising tensions between Italy and Ethiopia. In October 1935, Italy invaded", "reference": " Ethiopia, and Amedeo's unit was deployed to the front lines. Th", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_94><extra_id_77><extra_id_110><extra_id_88>Q<extra_id_54>\u0017<extra_id_88><extra_id_2>\u0017\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
55
  {"doc_id": "blog_0710@Meta-Llama-3-70B", "prompt": "So, I figured that if I ate just a few bites, then threw the rest away, Jack wouldn't know any better. I started off slowly wit", "reference": "h some beans (which weren't bad) and then moved on to the pork. ", "prediction": "\u0019\u0000\u001aﹷ<extra_id_51>\rx4<extra_id_25>\u001c\b<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
56
+ {"doc_id": "fic_0704@Meta-Llama-3-8B-Instruct", "prompt": "As the steamboat, or rather, the raft, drew near to the shore, Freddie Bobbsey called out to his sister, \"All aboard, Flossie! ", "reference": "Get on, get on!\" Flossie, with her blue eyes shining with excite", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P<extra_id_114>\u0017i2<extra_id_56>(<extra_id_115>(<extra_id_115><extra_id_64>"}
57
+ {"doc_id": "blog_0945@chunk_2", "prompt": "I thought they were cute but really thought nothing of it. They then kept looking our way, mind you Brad stopped headbanging an", "reference": "d was sitting on the bench with Phil and no one was in front of ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
58
  {"doc_id": "fic_0923@chunk_2", "prompt": "Still appearing utterly oblivious of the questions showered upon him by the now frantic man who had constituted himself as guid", "reference": "e, the young man had got midway up the street, and was still sea", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
59
  {"doc_id": "spok_1705@chunk_2", "prompt": "So if you can picture eight feet tall like the basket that you used to be able to dunk on when you're in like 8th grade because", "reference": " you couldn't get 10 that's almost how long his arms are which i", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
60
  {"doc_id": "acad_0521@chunk_1", "prompt": "A wide body of literature in archaeology and anthropology discusses the concept of craft specialisation. This topic is importan", "reference": "t because it contributes to understanding the development of soc", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
 
104
  {"doc_id": "acad_0483@Meta-Llama-3-8B", "prompt": "A survey of 5000 UK residents found that perceptions of visual amenity were more negative among those who had previously oppose", "reference": "d onshore wind farms, even if they lived further away from exist", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
105
  {"doc_id": "spok_1550@chunk_1", "prompt": "Wrong, so wrong by c-pillar 42 read by litter Aryan. summary There is a kiss and it is so wrong. It turns out that even after k", "reference": "nowing each other for about six thousand years. It is possible t", "prediction": "U<extra_id_29><extra_id_120><extra_id_115>\u001c\u0012\u0001\b9ú<extra_id_113><extra_id_16><extra_id_58>:<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0i(<extra_id_119>C<extra_id_107>P*\rx<extra_id_63>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
106
  {"doc_id": "news_0483@Meta-Llama-3-8B", "prompt": "'We are working hard to minimise any disruption to our customers and will contact them directly if their flight has been affect", "reference": "ed.' BA added that it was continuing to monitor the situation cl", "prediction": "\u0019\u0000\u001aU\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
107
+ {"doc_id": "blog_0065@gpt-4o-mini-2024-07-18", "prompt": "Not for You - A powerful track from Vitalogy, it echoes the frustrations of feeling trapped within the confines of expectation.", "reference": " Eddie's emotive delivery resonated deeply, especially during th", "prediction": "<extra_id_119><extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
108
  {"doc_id": "spok_0558@chunk_1", "prompt": "Hi everyone, and welcome to this incessant home tomorrow. I am like internally screaming right now. This is a podcast about thi", "reference": "ngs that just make isn't comfortable and that we don't want to t", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
109
  {"doc_id": "spok_1215@chunk_1", "prompt": "Welcome to the good Christian talks podcast. I'm Levi and I'm Chris. Thank you so much for joining us this week on this podcast", "reference": ". We select one taco week to help each one of us get the Bible i", "prediction": "\u0019\u0000\u001aﹷ<extra_id_51>\rx4<extra_id_25>\u001c\b<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>i2<extra_id_56>(<extra_id_115>)I<extra_id_16><extra_id_58><extra_id_25><extra_id_64>"}
110
  {"doc_id": "acad_0525@gpt-4o-mini-2024-07-18", "prompt": "This observation suggests a multifaceted role for p53 in modulating not only the cellular response to DNA damage but also the m", "reference": "etabolism of environmental carcinogens, thereby influencing susc", "prediction": "\u0012\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
 
115
  {"doc_id": "blog_0358@gpt-4o-2024-08-06", "prompt": "67-year-old S Puhaindran was a young teacher in Raffles Institution when Mr Goh was a student there. \"I remember him well,\" Mr.", "reference": " Puhaindran recounted with an affectionate smile. \"Goh was the k", "prediction": "<extra_id_119><extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_63>)XI<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
116
  {"doc_id": "spok_0718@chunk_2", "prompt": "Welcome to our second of two episodes on John T Scopes last week We examined the rise of the anti-evolution movement in the ear", "reference": "ly 1920s that led to the establishment of the butler act which b", "prediction": "<extra_id_25><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
117
  {"doc_id": "spok_1582@chunk_1", "prompt": "Hello everyone and welcome to the Notre grow podcast. I'm your host. Dr. Ivan Khan, and we'll be breaking down topics around ed", "reference": "ucation growth and culture with the intention to help your growt", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
118
+ {"doc_id": "blog_1429@Meta-Llama-3-70B-Instruct", "prompt": "As I sat in the staff meeting, listening to our administrators outline the plan for the year, I couldn't help but feel a sense ", "reference": "of excitement and trepidation. The idea of \"writing it down and ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001z<extra_id_111>\u001c\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
119
  {"doc_id": "tvm_0327@chunk_2", "prompt": "Dead serious. See... the world is made up of rules. Kids don't like rules. They rebel for a while, but ultimately they start to", "reference": " fall in line and become ordinary Joe's like you and me. But som", "prediction": "C<extra_id_115>\u001c\u0012\u0001\b9ú<extra_id_113><extra_id_107>P<extra_id_110><extra_id_88>Q;w\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107>P*\rx<extra_id_63>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
120
  {"doc_id": "tvm_0857@gpt-4o-mini-2024-07-18", "prompt": "With silent determination, MALONE casts an assured glance at the OARSMEN, who hesitate for a moment, caught between the ardor o", "reference": "f loyalty and the dread of what lies ahead. One by one, they asc", "prediction": "C\u001c&<extra_id_121>\u0001\b9ú<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
121
  {"doc_id": "blog_0293@chunk_1", "prompt": "Michael posted more about abortion and guns on his website and, as a woman, I violentely reacted to some of his comments on abo", "reference": "rtion. My reaction was too long to post on his blog, so I'm doin", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
122
+ {"doc_id": "news_0530@Meta-Llama-3-8B-Instruct", "prompt": "As the controversy surrounding the botched trial of John Downey continued to simmer, the UK government's refusal to launch a ju", "reference": "dicial inquiry into the matter only added fuel to the fire. Nort", "prediction": "x<extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
123
  {"doc_id": "spok_0772@Meta-Llama-3-8B-Instruct", "prompt": "Here's the next 500 words in the same style and tone: David: Yeah, okay, let's move on to the next question. At wonderland unde", "reference": "rscore underscore, what's the weirdest thing that's happened to ", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
124
  {"doc_id": "news_0390@chunk_2", "prompt": "For example, Benson said that $1 million in $100 bills weighs 22 pounds, while $1 million in 500 euro notes weighs 3.5 pounds. ", "reference": "\"It's a huge difference,\" he said. Though Europe is highly attra", "prediction": "<extra_id_115><extra_id_115>\u001c\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
125
  {"doc_id": "spok_0704@gpt-4o-2024-08-06", "prompt": "You see, creativity intertwines itself with the rituals that one builds around it. For me, it's not so much about adhering to a", "reference": " strict schedule, as it is about weaving those moments of intros", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
126
+ {"doc_id": "news_0483@Meta-Llama-3-8B-Instruct", "prompt": "As the storm continues to rage on the East Coast, the travel chaos shows no signs of abating. Thousands of Britons are facing a", "reference": "nother day of uncertainty, with airlines scrambling to rebook fl", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
127
  {"doc_id": "fic_1229@chunk_2", "prompt": "Phillips, his valet, had heard the shot, and knowing that the prince's cousin, Prince Miguel, with Don Solado, prime minister o", "reference": "f Joyalita, had made attempts on his life before, in New York, h", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0\u0014\u0017i2<extra_id_56>(<extra_id_115><extra_id_64>\u0017\u000b"}
128
  {"doc_id": "fic_0555@gpt-4o-2024-08-06", "prompt": "The memories of carefree childhood days woven amidst the tranquil charm of the estate danced in his mind, anchoring him to this", "reference": " beloved place with an unyielding fondness. The familiar sound o", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
129
  {"doc_id": "blog_1347@Meta-Llama-3-70B-Instruct", "prompt": "As I gazed deeper into the mirror, my eyes seemed to bore into my own soul. The reflection staring back at me was a stranger, y", "reference": "et eerily familiar. The lines on my face told a story of pain an", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\b9ú<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
 
135
  {"doc_id": "news_0135@gpt-4o-mini-2024-07-18", "prompt": "However, the events of last October reshaped those timelines and priorities, pushing Virgin Galactic into a more insular approa", "reference": "ch to the development of their space ambitions. The firm recogni", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
136
  {"doc_id": "acad_0489@Meta-Llama-3-8B", "prompt": "Individual hackers may hack into websites or online services to deface content, change information, disrupt service, or simply ", "reference": "out of curiosity.16 Lastly, activist groups may target critical ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
137
  {"doc_id": "news_0246@Meta-Llama-3-70B", "prompt": "That might seem an unlikely trio of activities for a man preparing for a world title fight but there were two serious reasons f", "reference": "or Manny's trip here. One was to promote his forthcoming book, a", "prediction": "<extra_id_16><extra_id_58>:Z\u001c\u0012\u0001\b\u0017\u0017*<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
138
+ {"doc_id": "acad_0497@gpt-4o-mini-2024-07-18", "prompt": "Moreover, the limitations inherent within the traditional diagnostic framework underscore a pressing need for an integrative, t", "reference": "ransdiagnostic approach to emotional disorders that recognizes a", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
139
  {"doc_id": "spok_1425@chunk_1", "prompt": "Even when you say listen, we might not be getting billboard this time around because you're a brand new artist. They don't nece", "reference": "ssarily hear that. So first off I just want to say welcome to th", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
140
  {"doc_id": "blog_0578@Meta-Llama-3-8B-Instruct", "prompt": "As I reflect on my childhood friendships, I'm reminded of the many memories I've accumulated over the years. One of my earliest", "reference": " and most cherished friendships was with Rachel. We met in kinde", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
141
  {"doc_id": "spok_1607@Meta-Llama-3-70B", "prompt": "We have and if you haven't had that experience yet, then I would definitely suggest you start paying attention to how you feel ", "reference": "and learning to be in tune with yourself and your intuition. So ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
 
161
  {"doc_id": "tvm_0962@Meta-Llama-3-70B-Instruct", "prompt": "Walter's eyes dart around the elevator, searching for an escape from the uncomfortable silence. Ted's gaze lingers, as if darin", "reference": "g Walter to ask again. Finally, he responds, his tone measured, ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_94><extra_id_77><extra_id_110>\u0003<extra_id_16><extra_id_58>㥦\u0017\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
162
  {"doc_id": "acad_0646@gpt-4o-2024-08-06", "prompt": "These studies employed Geographic Information Systems (GIS) to accurately delineate spatial relationships and pinpoint geograph", "reference": "ical proximities between producers, allowing for a nuanced analy", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
163
  {"doc_id": "acad_0772@chunk_2", "prompt": "However, the pressure signal of the catheter is affected by the viscosity of the colonic fluids [28]. Arkwright et al. [28] sho", "reference": "wed that the predetermined amplitude of the applied pressure on ", "prediction": "C\u001c&<extra_id_121>\u0001\b\u0017\u0017*<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
164
+ {"doc_id": "news_0656@gpt-4o-mini-2024-07-18", "prompt": "In facing this precarious situation, the possibility of requiring a financial rescue looms large on Italy's horizon, though man", "reference": "y experts argue it lacks the structural and political framework ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
165
  {"doc_id": "spok_0704@Meta-Llama-3-8B", "prompt": "That's really cool. I think that's great. I mean, I'm not judging anyone or anything, but it's interesting because I remember r", "reference": "eading an article by Tony Robbins and he said that he stopped dr", "prediction": "<extra_id_25><extra_id_115>\u001cﹷ\u0017\u0017\u0017\u0017*<extra_id_16><extra_id_58>:\u0017S<extra_id_27>)<extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
166
  {"doc_id": "news_1257@gpt-4o-2024-08-06", "prompt": "The goal is to prevent Iran from refining uranium beyond a certain threshold, drastically reducing its capacity to progress tow", "reference": "ard a nuclear weapon in the near term. In exchange, Iran will re", "prediction": "\u0017\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
167
  {"doc_id": "spok_1443@Meta-Llama-3-8B", "prompt": "They said they couldn't stand the smell of your dead cat anymore. So they took off. You've been living alone since 1999 and no ", "reference": "one has heard from them again. I guess they just went back home ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zlú<extra_id_113><extra_id_115>)<extra_id_122>s<extra_id_122>s<extra_id_122>s<extra_id_33><extra_id_33><extra_id_33><extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
 
179
  {"doc_id": "blog_0126@chunk_2", "prompt": "What took you so long? God help you become victorious. I want to say hello to Bush, to shake his hand. We came out of the grave", "reference": ".'\" The Guadian, 3-22-03 \"As hundreds of coalition troops swept ", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P<extra_id_114>\u0017i2<extra_id_56>(<extra_id_115>\u001c&9(<extra_id_115>"}
180
  {"doc_id": "fic_0051@chunk_2", "prompt": "Two seamen, each pushing an oar before him, swam out to the nearest boat, and having got on board of her, and freed her from he", "reference": "r entanglements, they rowed out to the capsized boat, and towed ", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
181
  {"doc_id": "tvm_0383@chunk_1", "prompt": "How do you do. My name is Penelope Ryan. This is a simple-minded play about men who enjoy killing -- and those who don't. I am ", "reference": "Harold Ryan, her husband. I have killed perhaps two hundred men ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
182
+ {"doc_id": "acad_0983@Meta-Llama-3-8B", "prompt": "Understanding how much water is available for plants in a given ecosystem is critical for managing natural resources and assess", "reference": "ing the impact of land-use/management practices on water availab", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_63>)XI<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
183
  {"doc_id": "fic_0034@Meta-Llama-3-70B", "prompt": "The ears were short and delicately made; the eyes were well-placed; and the distance to the angle of the jaw was such as good b", "reference": "reeders desire. Considering him closer she became aware of the l", "prediction": "<extra_id_64>\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
184
  {"doc_id": "blog_0069@chunk_1", "prompt": "..Is the one, which tells you immediately, what the outcome will be: Good food, bad husband, horror able dinner, acing a test, ", "reference": "bad sex, insanely demented neighbours... the last thing is an is", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
185
  {"doc_id": "blog_0747@gpt-4o-2024-08-06", "prompt": "I finally reached Plaza Singapura, and there was Darius, waiting patiently with a grin. The moment I saw him, I felt a little f", "reference": "lutter in my stomach. It's funny how a net friend can feel so fa", "prediction": "<extra_id_16><extra_id_58>:Z\u001c\u0012\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
 
199
  {"doc_id": "acad_0807@gpt-4o-2024-08-06", "prompt": "As a result, the development of more adaptive operational strategies that can detect and respond to ice formation is paramount.", "reference": " Integrating advanced sensor technology can offer a real-time as", "prediction": "<extra_id_119><extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
200
  {"doc_id": "fic_0975@Meta-Llama-3-70B", "prompt": "\"Well, what does the chief say, sir?\" he asked. \"He doesn't seem very cheerful over it.\" \"He isn't,\" replied Ramshaw grimly. \"S", "reference": "ome scoundrel has tampered with the high-pressure slide valves. ", "prediction": "<extra_id_32>÷\u0017\u0017\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
201
  {"doc_id": "spok_1483@Meta-Llama-3-8B-Instruct", "prompt": "But we're not going to be bored, because we're going to dive into the story of the 1974 Zaire team and explore the complexities", "reference": " of colonialism and its lasting impact on the African continent.", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
202
+ {"doc_id": "fic_0841@chunk_1", "prompt": "One evening Mr. Courtnay Fox, the London correspondent of the Edinburgh Chronicle, was as usual in his own room in the office i", "reference": "n Fleet-street, when a card was brought to him. \"Show the gentle", "prediction": "\u001c\u0012\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
203
  {"doc_id": "news_0227@chunk_2", "prompt": "We've been writers in our own right for several years before we started to write together, and from the beginning we thought it", "reference": " was a great idea to stay secret behind a pen name. Then two thi", "prediction": "\u0019\u0000\u001aU\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_94><extra_id_77><extra_id_110>\u0003<extra_id_16><extra_id_58>Һ<extra_id_113>\u0017i2<extra_id_56>(<extra_id_115>\u001c&9(<extra_id_115>"}
204
  {"doc_id": "tvm_0327@Meta-Llama-3-70B", "prompt": "It's the humane thing to do. We don't want them to suffer. You know how hard it is to find a decent lawyer these days? This is ", "reference": "the place? Yeah, I think so. Come on, Lizzy. Let's check it out.", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zlú<extra_id_113><extra_id_115>)<extra_id_122>s<extra_id_122>\u001fZ\u0019\u0000^<extra_id_101>0i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
205
  {"doc_id": "blog_0026@chunk_1", "prompt": "On Monday morning at work, I slipped on a wet floor. Mondays are always heavy load days. Mondays after a holiday... ew. Twelve ", "reference": "dogs to do. And they wouldn't let me wash some of them a day ear", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
206
  {"doc_id": "news_0187@gpt-4o-2024-08-06", "prompt": "Jimmy was grateful for Claire's presence and quickly found that her vivacious spirit brightened the household. Even Gemma, who ", "reference": "had her reservations at first about her father's burgeoning rela", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
207
+ {"doc_id": "fic_1122@Meta-Llama-3-8B-Instruct", "prompt": "As the days passed, the anticipation grew, and the village was abuzz with excitement. The two suitors, myself and my rival, wer", "reference": "e summoned to the farmer's estate, where we were to present ours", "prediction": "<extra_id_25><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
208
  {"doc_id": "news_0530@gpt-4o-mini-2024-07-18", "prompt": "In a shocking escalation of political tension, Mr. Robinson's outrage reverberated throughout Northern Ireland's tumultuous lan", "reference": "dscape, reigniting old wounds and fears among the populace. The ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
209
  {"doc_id": "news_0251@chunk_1", "prompt": "Experts were unable to open the 9th century Carolingian pot (pictured), which was found on church land in Dumfries and Galloway", "reference": " in September . The mystery surrounding the contents of a Viking", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
210
  {"doc_id": "spok_1103@chunk_2", "prompt": "It was a questionable mix at that point in history. One member of the crowd tried to storm the stage, but he was quickly subdue", "reference": "d by security after he managed to break one of the presentation ", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
 
222
  {"doc_id": "tvm_0420@gpt-4o-2024-08-06", "prompt": "We simply critique the cult that surrounds them. Sirens wail in the distance, and the thrum of anticipation grows. Young is sti", "reference": "ll spinning his tale, unaware or uncaring of the chaos, maintain", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
223
  {"doc_id": "news_1166@gpt-4o-2024-08-06", "prompt": "But rather than dismissing this sentiment as merely the nostalgic ramblings of a bygone generation, it might serve us well to d", "reference": "elve deeper into the root of such concerns. The discourse surrou", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_107>P<extra_id_110><extra_id_88>Q;w\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107>P*\rx<extra_id_63>\u001f^<extra_id_101><extra_id_93>(<extra_id_115>\u001c&<extra_id_64>"}
224
  {"doc_id": "spok_1150@chunk_2", "prompt": "And they have a new proprietary blend which is their metabolic ferment, which I can't explain to you the science behind it, but", "reference": " I will tell you that it's supposed to help boost your natural c", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_16><extra_id_58>:<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
225
+ {"doc_id": "blog_0026@gpt-4o-2024-08-06", "prompt": "I slipped, my feet skidding out from under me like an awkward rendition of a cartoon character slipping on a banana peel. As I ", "reference": "tumbled, I could almost hear the universe snickering. My backsid", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
226
  {"doc_id": "blog_0819@Meta-Llama-3-8B-Instruct", "prompt": "You see, it's all about power, man. The War on Drugs is just a thinly veiled excuse for the government to exert its control ove", "reference": "r our lives. They're like, \"Oh, we're doing this to protect you ", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
227
  {"doc_id": "news_0251@chunk_2", "prompt": "A scan of the Carolingian pot was conducted by Dr John Reid, consultant radiographer at the BGH, who is also a keen amateur arc", "reference": "haeologist. He was approached by Richard Welander, head of colle", "prediction": "i\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017i2<extra_id_56>(<extra_id_115>\u001c&<extra_id_64>"}
228
  {"doc_id": "fic_0017@gpt-4o-2024-08-06", "prompt": "He was a man of rigorous discipline, a stalwart presence in the realm of naval journalism, and one who carried the weight of hi", "reference": "s responsibilities with a steely resolve. His eyes, accustomed t", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_4><extra_id_88>\u0019\u0000<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
 
241
  {"doc_id": "fic_0964@chunk_2", "prompt": "The preacher, half amused, half embarrassed, stood still, as a woman came out of a tiny hut near by, and peered about her with ", "reference": "angry, short-sighted eyes. Mrs. Brazybone was a vast woman, with", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
242
  {"doc_id": "blog_0946@Meta-Llama-3-8B-Instruct", "prompt": "I'm still trying to process everything that's going on. I feel like I'm stuck in some kind of never-ending nightmare. My dad's ", "reference": "financial situation is getting worse and worse, and I'm starting", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
243
  {"doc_id": "blog_1429@Meta-Llama-3-8B", "prompt": "It was a great experience! And we were very happy with the outcome. However, when we came back from our break and met again yes", "reference": "terday, I felt like there was something missing. We had a lot of", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_16><extra_id_58><extra_id_25><extra_id_16><extra_id_58>&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
244
+ {"doc_id": "blog_1429@chunk_2", "prompt": "Out of those 3 days, our staff formed into 'action teams' to work together the rest of the year on specific areas we felt neede", "reference": "d redefining, exploration, clarity . I chose the numeracy group.", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P<extra_id_114>\u0017i2<extra_id_56>(<extra_id_115>(<extra_id_115><extra_id_64>"}
245
  {"doc_id": "fic_1250@Meta-Llama-3-8B-Instruct", "prompt": "As he descended into the canyon, the silence was broken only by the soft rustling of leaves and the gentle crunch of gravel ben", "reference": "eath his hooves. Spread Horn's ears twitched ever so slightly, a", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
246
  {"doc_id": "fic_0546@Meta-Llama-3-8B", "prompt": "It was evident that she was used to asking questions, and receiving answers, and that she did not think it worth while to show ", "reference": "emotion about either process. She laid down the paper, and took ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
247
  {"doc_id": "spok_1404@chunk_1", "prompt": "Is take it away ryos extra. Can I call you Riles? Yeah, so cute as well my friends. Call me. How you that? All right. Well my e", "reference": "lementary school friends. No one here does except for trials. Oh", "prediction": "b<extra_id_64>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_16><extra_id_16><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
muon/predictions_01.jsonl CHANGED
The diff for this file is too large to render. See raw diff
 
muon/predictions_02.jsonl CHANGED
The diff for this file is too large to render. See raw diff
 
muon/predictions_03.jsonl CHANGED
The diff for this file is too large to render. See raw diff
 
muon/predictions_04.jsonl CHANGED
The diff for this file is too large to render. See raw diff
 
muon/predictions_05.jsonl CHANGED
The diff for this file is too large to render. See raw diff
 
muon/predictions_06.jsonl CHANGED
The diff for this file is too large to render. See raw diff
 
muon/predictions_07.jsonl CHANGED
The diff for this file is too large to render. See raw diff
 
muon/predictions_08.jsonl CHANGED
The diff for this file is too large to render. See raw diff
 
muon/predictions_09.jsonl CHANGED
The diff for this file is too large to render. See raw diff
 
muon/predictions_10.jsonl CHANGED
The diff for this file is too large to render. See raw diff
 
muon/settings.json CHANGED
@@ -12,27 +12,30 @@
12
  "dimension_model": 256,
13
  "attention_heads": 8,
14
  "decoder_blocks": 4,
15
- "learning_rate": 0.02,
16
  "weight_decay": 0.01,
17
  "beta1": 0.9,
18
- "beta2": 0.999,
19
  "epsilon": 1e-08,
20
  "gamma": 0.025,
21
  "momentum": 0.95,
22
  "ns_steps": 5,
23
  "ns_epsilon": 1e-07,
24
- "aux_learning_rate": 0.0003,
 
 
 
25
  "max_grad_norm": 1.0,
26
  "prompt_tokens": 128,
27
  "max_new_tokens": 64,
28
- "device": "cuda:1",
29
  "mixed_precision": false,
30
- "tokenizer_name": "/kaggle/working/part2/tokenizer",
31
- "cache_dir": "/kaggle/working/part2/cache",
32
- "output_dir": "/kaggle/working/part2/output/muon",
33
  "max_train_tokens": 0,
34
  "prepare_only": false,
35
- "resume": "part2/output/muon/latest.pt"
36
  },
37
  "model": {
38
  "vocab_size": 388,
@@ -44,34 +47,53 @@
44
  "parameters": 3355012,
45
  "optimizer_groups": [
46
  {
47
- "lr": 0.02,
48
  "use_muon": true,
49
  "momentum": 0.95,
50
  "ns_steps": 5,
51
  "betas": [
52
  0.9,
53
- 0.999
54
  ],
55
  "eps": 1e-08,
56
  "weight_decay": 0.01,
57
- "ns_eps": 1e-07
 
 
58
  },
59
  {
60
- "lr": 0.0003,
61
  "use_muon": false,
62
  "momentum": 0.95,
63
  "ns_steps": 5,
64
  "betas": [
65
  0.9,
66
- 0.999
67
  ],
68
  "eps": 1e-08,
69
  "weight_decay": 0.01,
70
- "ns_eps": 1e-07
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
  }
72
  ],
73
  "data": {
74
- "dataset_fingerprint": "c19c55fa232416f4",
75
  "split_method": "90/5/5 by source doc_id before @, then seeded row subsampling",
76
  "full_split_documents": {
77
  "train": 59688,
@@ -15518,14 +15540,8 @@
15518
  "validation_target_tokens": 720136,
15519
  "test_target_tokens": 708110
15520
  },
15521
- "learning_rate_schedule": "constant",
15522
  "evaluation_interval": "0.1 of selected training budget",
15523
- "precision_changes": [
15524
- {
15525
- "train_tokens": 21366755,
15526
- "previous_mixed_precision": true,
15527
- "mixed_precision": false
15528
- }
15529
- ],
15530
  "bleu_definition": "Greedy continuation after prompt_tokens; reference is the next max_new_tokens tokens."
15531
  }
 
12
  "dimension_model": 256,
13
  "attention_heads": 8,
14
  "decoder_blocks": 4,
15
+ "learning_rate": 0.001,
16
  "weight_decay": 0.01,
17
  "beta1": 0.9,
18
+ "beta2": 0.95,
19
  "epsilon": 1e-08,
20
  "gamma": 0.025,
21
  "momentum": 0.95,
22
  "ns_steps": 5,
23
  "ns_epsilon": 1e-07,
24
+ "aux_learning_rate": 0.0001,
25
+ "lr_schedule": "cosine",
26
+ "warmup_fraction": 0.05,
27
+ "min_lr_ratio": 0.1,
28
  "max_grad_norm": 1.0,
29
  "prompt_tokens": 128,
30
  "max_new_tokens": 64,
31
+ "device": "auto",
32
  "mixed_precision": false,
33
+ "tokenizer_name": "/Users/samawalke/Project/Masters/ANLP/Assignment2/2022101025_assignment2/part2/tokenizer",
34
+ "cache_dir": "/Users/samawalke/Project/Masters/ANLP/Assignment2/2022101025_assignment2/part2/cache",
35
+ "output_dir": "part2/part2_results/output/muon",
36
  "max_train_tokens": 0,
37
  "prepare_only": false,
38
+ "resume": null
39
  },
40
  "model": {
41
  "vocab_size": 388,
 
47
  "parameters": 3355012,
48
  "optimizer_groups": [
49
  {
50
+ "lr": 0.0,
51
  "use_muon": true,
52
  "momentum": 0.95,
53
  "ns_steps": 5,
54
  "betas": [
55
  0.9,
56
+ 0.95
57
  ],
58
  "eps": 1e-08,
59
  "weight_decay": 0.01,
60
+ "ns_eps": 1e-07,
61
+ "attention_layout": "full_projection",
62
+ "peak_lr": 0.001
63
  },
64
  {
65
+ "lr": 0.0,
66
  "use_muon": false,
67
  "momentum": 0.95,
68
  "ns_steps": 5,
69
  "betas": [
70
  0.9,
71
+ 0.95
72
  ],
73
  "eps": 1e-08,
74
  "weight_decay": 0.01,
75
+ "ns_eps": 1e-07,
76
+ "attention_layout": "full_projection",
77
+ "peak_lr": 0.0001
78
+ },
79
+ {
80
+ "lr": 0.0,
81
+ "use_muon": false,
82
+ "weight_decay": 0.0,
83
+ "momentum": 0.95,
84
+ "ns_steps": 5,
85
+ "betas": [
86
+ 0.9,
87
+ 0.95
88
+ ],
89
+ "eps": 1e-08,
90
+ "ns_eps": 1e-07,
91
+ "attention_layout": "full_projection",
92
+ "peak_lr": 0.0001
93
  }
94
  ],
95
  "data": {
96
+ "dataset_fingerprint": "abc9253b0d88d598",
97
  "split_method": "90/5/5 by source doc_id before @, then seeded row subsampling",
98
  "full_split_documents": {
99
  "train": 59688,
 
15540
  "validation_target_tokens": 720136,
15541
  "test_target_tokens": 708110
15542
  },
15543
+ "learning_rate_schedule": "cosine",
15544
  "evaluation_interval": "0.1 of selected training budget",
15545
+ "precision_changes": [],
 
 
 
 
 
 
15546
  "bleu_definition": "Greedy continuation after prompt_tokens; reference is the next max_new_tokens tokens."
15547
  }
muon/training_steps.csv CHANGED
The diff for this file is too large to render. See raw diff
 
source/part2/README.md CHANGED
@@ -1,252 +1,73 @@
1
- # Part 2: optimizer experiments
2
 
3
- The model architecture and utility files are copied from Part 1. Part 2 defaults
4
- to a model dimension of 256, 8 attention heads, and 4 decoder blocks. All four
5
- scripts instantiate the dense variant with fresh weights and the same initialization seed.
6
- The saved byte-level tokenizer is copied too; its vocabulary matches Part 1.
 
 
 
 
7
 
8
- | Script | Category | Optimizer |
9
- | --- | --- | --- |
10
- | `train_adamw.py` | AdamW baseline | AdamW |
11
- | `train_mars.py` | Variance-reduced AdamW | Approximate MARS-AdamW |
12
- | `train_lion.py` | Memory-efficient | Lion |
13
- | `train_muon.py` | Matrix-based | Muon plus custom AdamW for other parameters |
14
 
15
- These choices correspond to Algorithms 1, 5, 3, and 8 in
16
- [Fantastic Pretraining Optimizers and Where to Find Them, v2](https://arxiv.org/pdf/2509.02046v2),
17
- Appendix A, pages 23-26. The updates are implemented explicitly, and each optimizer
18
- inherits only from `torch.optim.Optimizer`; no predefined optimizer is used.
19
-
20
- Two apparent errors in the appendix are corrected in the implementation:
21
-
22
- - Gradient clipping uses `min(1, limit / norm)`, not the printed `max`, which would
23
- enlarge small gradients and leave large gradients unclipped. AdamW, Lion, and
24
- Muon receive globally clipped gradients from the training loop. MARS receives
25
- raw gradients, applies its variance correction, and then globally clips the
26
- corrected gradients, following Algorithm 5's order. Its previous gradient starts
27
- at zero and is saved before clipping for the next update.
28
- - Muon's Newton-Schulz iteration uses `B = -4.7750 A + 2.0315 A^2` followed by
29
- `X = 3.4445 X + B X`, matching the
30
- [original Muon implementation](https://github.com/KellerJordan/Muon/blob/master/muon.py).
31
- The polynomial printed in Algorithm 8 is not that iteration. Momentum and the
32
- rectangular-matrix scale follow Algorithm 8. Each 3D attention tensor is converted
33
- from `[heads, input, output_per_head]` to the full `[output, input]` projection
34
- before orthogonalization, then converted back. Q, K, and V remain separate.
35
- The default 256-dimensional attention projections are square, with scale 1;
36
- the earlier per-head implementation applied scale sqrt(8). Newton-Schulz uses FP32 during
37
- training and a separate `--ns-epsilon` (default `1e-7`).
38
-
39
- AdamW and MARS use bias-corrected first/second moments and decoupled weight decay.
40
- Lion uses the old momentum for its sign update before updating the stored momentum.
41
- Muon uses the same custom AdamW update helper for embeddings, the output head,
42
- LayerNorm parameters, and other biases. All persistent optimizer state is saved
43
- in `state_dict()` for checkpoint resume. This is approximate MARS-AdamW with the
44
- preceding batch's gradient, not an extra backward pass on the current batch.
45
-
46
- ## Data and evaluation
47
-
48
- Defaults use 25% of the training documents. Source documents are divided into
49
- 90% training, 5% validation, and 5% test before subsampling. All author versions
50
- sharing the same `doc_id` prefix before `@` stay together, avoiding leakage between
51
- parallel versions. A fixed seed selects and orders the same text for every run.
52
- Every selected document is used in full, with BOS/EOS markers, and packed into
53
- 256-token sequences. Only the last partial sequence is padded.
54
-
55
- Evaluation limits are also scaled by 0.25: the default 1,000 validation documents
56
- and 1,000 test documents become 250 each, or fewer if a split is smaller.
57
- Evaluation batches default to 64 for both loss scoring and BLEU generation;
58
- use `--eval-batch-size` to change this.
59
- All selected validation/test tokens are scored for loss. BLEU uses the same fixed
60
- test documents every time: the first 128 tokens are the prompt, and the following
61
- 64 tokens are the reference continuation. Predictions use greedy decoding for up
62
- to 64 tokens, stopping at EOS. Prompt text is excluded from BLEU. This is a
63
- continuation BLEU definition for next-token pretraining, not translation BLEU.
64
-
65
- There is an initial evaluation, then one at each 10% of the selected training token
66
- budget. Final batches are shortened to hit those token boundaries exactly. The
67
- logs distinguish progress through the selected subset from progress relative to
68
- the full training split. Selecting 25% of documents may yield slightly more or
69
- less than 25% of tokens because document lengths differ.
70
-
71
- ## Run
72
-
73
- From `2022101025_assignment2`, prepare the shared cache once:
74
-
75
- ```bash
76
- python -m part2.train_adamw --prepare-only
77
- ```
78
-
79
- Run two experiments on Kaggle's two GPUs:
80
-
81
- ```bash
82
- python -m part2.train_adamw --device cuda:0 > part2/adamw.log 2>&1 &
83
- python -m part2.train_mars --device cuda:1 > part2/mars.log 2>&1 &
84
- wait
85
- python -m part2.train_lion --device cuda:0 > part2/lion.log 2>&1 &
86
- python -m part2.train_muon --device cuda:1 > part2/muon.log 2>&1 &
87
- wait
88
- ```
89
-
90
- Direct script execution works too: `python part2/train_adamw.py --help`.
91
- FP16 mixed precision and gradient scaling are enabled on CUDA for AdamW, Lion,
92
- and MARS. Muon defaults to FP32 and matrix learning rate 0.005. CPU/MPS use FP32.
93
- Use `--no-mixed-precision` to disable it. The learning rate is constant; defaults
94
- are starting values rather than tuned optimizer comparisons. Muon has separate
95
- learning rates for matrix parameters and its auxiliary custom AdamW group.
96
-
97
- Resume an interrupted run with the same settings:
98
 
99
  ```bash
100
- python -m part2.train_adamw --device cuda:0 --resume part2/output/adamw/latest.pt
101
  ```
102
 
103
- Checkpoints are saved after each evaluation, including the exact token position,
104
- optimizer/scaler state, metrics, and configuration. Resume continues from that
105
- position, so an interruption can require repeating up to one evaluation interval.
106
- Preserve/download `part2/output` before a Kaggle session ends.
107
-
108
- If an FP16 run stops with a non-finite training loss, first try resuming its last
109
- evaluation checkpoint with `--no-mixed-precision`. FP32 provides a wider numerical
110
- range; this is a recovery attempt, not a guarantee against optimizer divergence.
111
- Changing numerical precision on resume is allowed and recorded, with the token
112
- position of the change, in `settings.json` and the checkpoint state. Other training
113
- hyperparameters still need to match the saved run. For example:
 
 
 
 
114
 
115
  ```bash
116
- python -m part2.train_adamw --device cuda:0 --batch-size 64 \
117
- --no-mixed-precision --resume part2/output/adamw/latest.pt
118
  ```
119
 
120
- The corrected Muon implementation requires a fresh run if the old checkpoint
121
- used per-head attention updates. It rejects those optimizer states rather than
122
- silently changing the update rule halfway through training. Keep the old result
123
- and run the retry in a separate folder:
124
 
125
  ```bash
126
- python -u -m part2.train_muon --device cuda:0 --batch-size 64 \
127
- --eval-batch-size 64 --learning-rate 0.005 --no-mixed-precision \
128
- --output-dir part2/output/muon_retry
129
- python -m part2.plot_comparison --muon-dir part2/output/muon_retry
130
  ```
131
 
132
- This uses the same data, model, seed, and 11 evaluation points. The comparison
133
- includes both Muon runs. Inspect validation loss and saved predictions before
134
- claiming an improvement; changing the layout, precision, and learning rate together
135
- does not establish which change caused a difference. For uploads, add
136
- `--muon-dir PATH/TO/muon_retry` to `upload_part2_results.py` commands. The helper
137
- then uploads the retry as Muon, with its actual settings, while retaining the old
138
- local run. The new W&B run is named `muon_retry`; the other runs can remain in their
139
- existing project. Use a new Hugging Face repository if you already uploaded the
140
- earlier comparison and want to keep both uploaded comparisons separately.
141
-
142
- ## Recorded outputs
143
-
144
- Each optimizer has its own directory under `part2/output`:
145
-
146
- - `training_steps.csv`: batch loss, target tokens seen, batch token count,
147
- gradient norm, skipped updates, and step timing.
148
- - `metrics.csv` and `metrics.json`: interval and cumulative training loss,
149
- validation loss, test BLEU, train/validation/test perplexity, test loss,
150
- token counts, selected/full dataset fractions, learning rates, successful/skipped
151
- updates, training/evaluation times, throughput, BLEU signature, and sample count.
152
- - `predictions_*.jsonl`: test document IDs, prompts, references, and generated text
153
- for each evaluation point.
154
- - `settings.json`: all hyperparameters, model size, data fingerprint, split counts,
155
- selected row indices, and evaluation definitions. Separate output directories
156
- preserve separate hyperparameter runs.
157
- - `loss_vs_tokens.png` and `bleu_vs_tokens.png`: plots from recorded metrics.
158
- - `latest.pt`, `final.pt`, and `tokenizer/`: resumable/latest and final artifacts.
159
-
160
- After training, make the cross-optimizer plots:
161
 
162
  ```bash
163
- python -m part2.plot_comparison
 
 
 
 
164
  ```
165
 
166
- Run the optimizer numerical checks from the same directory:
167
-
168
- ```bash
169
- python -m unittest discover -s part2/tests -v
170
- ```
171
-
172
- For a full-data run, use `--data-fraction 1 --evaluation-fraction 1` and new cache
173
- and output directories. The current 25% setting follows the requested reduced
174
- experiment; it does not claim to reach the assignment's 1x full-dataset target.
175
-
176
- ## Upload finished runs
177
-
178
- The standalone `upload_part2_results.py` in the workspace root uploads saved
179
- results without repeating training. On this laptop it automatically reads
180
- `part2/part2_results/output`, independent of the shell's current directory.
181
- It checks that each selected optimizer reached its recorded training budget.
182
-
183
- From the workspace root (`Assignment2`):
184
-
185
- ```bash
186
- source .venv/bin/activate
187
- python -m pip install --upgrade wandb huggingface_hub
188
-
189
- # Validate locally; these commands do not log in or upload anything.
190
- python upload_part2_results.py wandb --dry-run
191
- python upload_part2_results.py huggingface --dry-run
192
-
193
- wandb login
194
- hf auth login
195
-
196
- python upload_part2_results.py wandb \
197
- --project anlp-assignment2-part2 --entity YOUR_WANDB_USER_OR_TEAM
198
- python upload_part2_results.py huggingface \
199
- --repo-id YOUR_HF_USERNAME/anlp-assignment2-part2
200
- ```
201
-
202
- Replace the account placeholders. The Hugging Face token needs write access.
203
- Alternatively, set `WANDB_API_KEY` and `HF_TOKEN` in your environment.
204
-
205
- W&B receives one run per optimizer, with batch loss, evaluation metrics,
206
- perplexity, BLEU, gradient norms, learning rates, throughput, and recorded timing
207
- plotted against training tokens. Raw CSV/JSON files, all prediction files,
208
- training logs, and plots are uploaded too. Saved plots are also displayed as
209
- images. Run IDs in each optimizer's `.wandb_upload.json` allow retries to resume
210
- without logging the same metric events again. Keep these files when retrying.
211
- If metrics or settings change after uploading, the script rejects resuming that
212
- run; use a fresh results directory for a different experiment. Historical GPU
213
- utilization was not recorded, and upload-time system monitoring is disabled.
214
-
215
- Hugging Face receives a single model repository with an optimizer folder for
216
- AdamW, MARS, Lion, and Muon. By default it includes both `final.pt` and `latest.pt`,
217
- plus tokenizers, settings, metrics, predictions, logs, plots, and custom model
218
- source. The generated model card provides a loading example; these are custom
219
- PyTorch checkpoints rather than Transformers AutoModel artifacts. New repositories
220
- are private by default; add `--public` to create a public repository. This flag
221
- does not change the visibility of an already existing repository. Add
222
- `--final-only` to omit other checkpoints. Re-running an upload updates the same
223
- Hugging Face repository.
224
-
225
- To upload only selected optimizers, add e.g. `--optimizers adamw muon` to either
226
- command. To use another results directory, supply `--output-dir PATH` and, if
227
- needed, `--source-dir PATH`. For Kaggle, use explicit paths:
228
-
229
- ```bash
230
- python /kaggle/working/upload_part2_results.py wandb \
231
- --output-dir /kaggle/working/part2/output \
232
- --project anlp-assignment2-part2 --entity YOUR_WANDB_USER_OR_TEAM
233
- python /kaggle/working/upload_part2_results.py huggingface \
234
- --output-dir /kaggle/working/part2/output \
235
- --source-dir /kaggle/working/part2 \
236
- --repo-id YOUR_HF_USERNAME/anlp-assignment2-part2
237
- ```
238
-
239
- Generate comparison plots if they are missing, from `2022101025_assignment2`:
240
-
241
- ```bash
242
- python -m part2.plot_comparison --output-dir part2/part2_results/output
243
- ```
244
 
245
- The uploader's offline checks run from the workspace root:
246
 
247
  ```bash
248
- python -m unittest test_upload_part2_results -v
 
249
  ```
250
 
251
- These uploads preserve the completed reduced experiment; the full-dataset
252
- training regime and the written analysis remain separate assignment deliverables.
 
 
1
+ # Part 2 optimizer training
2
 
3
+ Muon uses peak learning rates of **0.001** for hidden matrices and **0.0001**
4
+ for auxiliary AdamW. Both rates warm up over 5% of training tokens, then follow
5
+ cosine decay to 10% of their peaks. Training uses FP32, momentum 0.95, five
6
+ Newton–Schulz iterations, gradient clipping at 1.0 and auxiliary AdamW betas
7
+ (0.9, 0.95). Attention matrices use full projection updates. Weight decay is
8
+ 0.01 on matrices and embeddings/head weights; biases and normalization gains
9
+ have zero decay. These settings are a conservative training configuration;
10
+ their performance must be measured on held-out data.
11
 
12
+ The dense architecture, seed, token order, 25% data selection, and eleven
13
+ evaluation points match the optimizer comparison. Batch size is 64. Learning
14
+ rates depend on tokens consumed, so resuming preserves the schedule.
 
 
 
15
 
16
+ From the Assignment2 directory, run:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
 
18
  ```bash
19
+ bash run_muon_local.sh
20
  ```
21
 
22
+ The script uses the project's `.venv`, selects MPS/CUDA when available, and
23
+ otherwise uses CPU. It downloads/caches the data on first use, writes the single
24
+ Muon run to `2022101025_assignment2/part2/part2_results/output/muon`, and logs to
25
+ `2022101025_assignment2/part2/part2_results/muon.log`. The initial evaluation runs
26
+ before the training progress bar. When training completes, the script refreshes
27
+ the comparison figures and the Muon row in `report.md` using measured results.
28
+ The launcher then uploads Muon metrics/plots to W&B project
29
+ `anlp-assignment2-part2` and checkpoints/source/results to your Hugging Face
30
+ repository `USERNAME/anlp-assignment2-part2`, and adds the returned links to the
31
+ report. The repository defaults to private when first created. Both services
32
+ use your saved login or `WANDB_API_KEY` / `HF_TOKEN` environment variables.
33
+ Run the same command after an interruption to resume `latest.pt`, or after
34
+ an upload error to upload the completed run.
35
+
36
+ If you are not already logged in, run these once from Assignment2:
37
 
38
  ```bash
39
+ .venv/bin/wandb login
40
+ .venv/bin/hf auth login
41
  ```
42
 
43
+ To install compatible dependencies in a fresh environment:
 
 
 
44
 
45
  ```bash
46
+ .venv/bin/python -m pip install -r 2022101025_assignment2/part2/requirements.txt
 
 
 
47
  ```
48
 
49
+ To invoke the trainer directly with its default parameters:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
 
51
  ```bash
52
+ cd 2022101025_assignment2
53
+ ../.venv/bin/python -u -m part2.train_muon --device auto
54
+ # Resume using identical training arguments:
55
+ ../.venv/bin/python -u -m part2.train_muon --device auto \
56
+ --resume part2/part2_results/output/muon/latest.pt
57
  ```
58
 
59
+ For a quick separate run, cap training with `--max-train-tokens` and supply a
60
+ temporary `--output-dir` directly to the trainer. A capped run is not a final
61
+ optimizer-comparison result. The full experiment trains on 42,733,510 target
62
+ tokens and scores 250 validation and 250 test documents.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
 
64
+ To validate the uploads separately, the helper uses the same `muon` folder:
65
 
66
  ```bash
67
+ .venv/bin/python upload_part2_results.py wandb --optimizers muon --dry-run
68
+ .venv/bin/python upload_part2_results.py huggingface --optimizers muon --dry-run
69
  ```
70
 
71
+ Remove `--dry-run` when you want to upload the completed results.
72
+ Muon parameter grouping and momentum defaults follow the
73
+ [Muon reference implementation](https://github.com/KellerJordan/Muon).
source/part2/plot_comparison.py CHANGED
@@ -1,6 +1,7 @@
1
  """Compare the four runs using recorded training token counts."""
2
 
3
  import argparse
 
4
  import json
5
  from pathlib import Path
6
 
@@ -11,29 +12,69 @@ import matplotlib.pyplot as plt
11
 
12
  def main():
13
  parser = argparse.ArgumentParser()
14
- parser.add_argument("--output-dir", type=Path, default=Path(__file__).resolve().parent / "output")
15
- parser.add_argument("--muon-dir", type=Path, help="Use a fresh Muon retry folder in the comparison.")
 
 
 
16
  args = parser.parse_args()
17
  runs = {}
 
18
  for name in ("adamw", "mars", "lion", "muon"):
19
- directory = args.muon_dir if name == "muon" and args.muon_dir is not None else args.output_dir / name
20
  path = directory / "metrics.json"
21
  if path.exists():
22
  runs[name] = json.loads(path.read_text())
23
- if args.muon_dir is not None:
24
- original = args.output_dir / "muon" / "metrics.json"
25
- if original.exists():
26
- runs["muon (original)"] = json.loads(original.read_text())
27
  if not runs:
28
  raise FileNotFoundError("No metrics.json files found under the output directory.")
29
- for metric, label in (("validation_loss", "Validation cross entropy"), ("test_bleu", "Test BLEU (0-100)")):
30
- figure, axis = plt.subplots(figsize=(8, 5), layout="constrained")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31
  for name, history in runs.items():
32
- axis.plot([row["train_tokens"] for row in history], [row[metric] for row in history],
33
- marker="o", label=name)
34
- axis.set(xlabel="Training target tokens seen", ylabel=label, title="Optimizer comparison")
 
 
 
 
 
 
 
 
35
  axis.legend()
36
- figure.savefig(args.output_dir / f"comparison_{metric}.png", dpi=180)
 
 
 
 
 
 
 
 
 
 
 
37
  plt.close(figure)
38
 
39
 
 
1
  """Compare the four runs using recorded training token counts."""
2
 
3
  import argparse
4
+ import csv
5
  import json
6
  from pathlib import Path
7
 
 
12
 
13
  def main():
14
  parser = argparse.ArgumentParser()
15
+ here = Path(__file__).resolve().parent
16
+ output = here / "part2_results" / "output"
17
+ parser.add_argument("--output-dir", type=Path, default=output if output.is_dir() else here / "output")
18
+ parser.add_argument("--x-axis", choices=("tokens", "dataset"), default="tokens",
19
+ help="dataset uses one pass over the selected training subset as 1x.")
20
  args = parser.parse_args()
21
  runs = {}
22
+ directories = {}
23
  for name in ("adamw", "mars", "lion", "muon"):
24
+ directory = args.output_dir / name
25
  path = directory / "metrics.json"
26
  if path.exists():
27
  runs[name] = json.loads(path.read_text())
28
+ directories[name] = directory
 
 
 
29
  if not runs:
30
  raise FileNotFoundError("No metrics.json files found under the output directory.")
31
+ dataset_sizes = {}
32
+ if args.x_axis == "dataset":
33
+ records = []
34
+ for name, history in runs.items():
35
+ settings = json.loads((directories[name] / "settings.json").read_text())
36
+ size = settings["data"]["train_target_tokens"]
37
+ if size <= 0:
38
+ raise ValueError(f"{name}: selected training dataset must contain target tokens.")
39
+ dataset_sizes[name] = size
40
+ for row in history:
41
+ records.append({**row, "optimizer": name, "experiment_dataset_tokens": size,
42
+ "experiment_dataset_passes": row["train_tokens"] / size,
43
+ "original_train_split_tokens": settings["data"]["full_train_target_tokens"]})
44
+ print(f"{name}: 1x dataset = {size:,} selected target tokens; "
45
+ f"final coverage = {history[-1]['train_tokens'] / size:.5f}x")
46
+ fields = list(dict.fromkeys(key for row in records for key in row))
47
+ with (args.output_dir / "comparison_dataset_metrics.csv").open("w", newline="") as stream:
48
+ writer = csv.DictWriter(stream, fieldnames=fields)
49
+ writer.writeheader()
50
+ writer.writerows(records)
51
+
52
+ def draw(axis, metric, label):
53
  for name, history in runs.items():
54
+ denominator = dataset_sizes[name] if args.x_axis == "dataset" else 1
55
+ axis.plot([row["train_tokens"] / denominator for row in history],
56
+ [row[metric] for row in history], marker="o", label=name)
57
+ if args.x_axis == "dataset":
58
+ axis.set_xlabel("Training tokens / selected training dataset tokens (passes)")
59
+ axis.set_xticks([i / 10 for i in range(11)])
60
+ axis.set_title("Optimizer comparison: selected subset is the experiment dataset")
61
+ else:
62
+ axis.set(xlabel="Training target tokens seen", title="Optimizer comparison")
63
+ axis.set_ylabel(label)
64
+ axis.grid(alpha=0.25)
65
  axis.legend()
66
+
67
+ for metric, label in (("validation_loss", "Validation cross entropy"), ("test_bleu", "Test BLEU (0-100)")):
68
+ figure, axis = plt.subplots(figsize=(8, 5), layout="constrained")
69
+ draw(axis, metric, label)
70
+ suffix = "_vs_dataset" if args.x_axis == "dataset" else ""
71
+ figure.savefig(args.output_dir / f"comparison_{metric}{suffix}.png", dpi=180)
72
+ plt.close(figure)
73
+ if args.x_axis == "dataset":
74
+ figure, axes = plt.subplots(1, 2, figsize=(14, 5), layout="constrained")
75
+ draw(axes[0], "validation_loss", "Validation cross entropy")
76
+ draw(axes[1], "test_bleu", "Test BLEU (0-100)")
77
+ figure.savefig(args.output_dir / "comparison_dataset_dynamics.png", dpi=180)
78
  plt.close(figure)
79
 
80
 
source/part2/requirements.txt CHANGED
@@ -1,6 +1,8 @@
1
  torch>=2.3
2
  datasets>=3
3
- transformers>=4.40
 
4
  sacrebleu>=2.4
5
  matplotlib>=3.8
6
  tqdm>=4.66
 
 
1
  torch>=2.3
2
  datasets>=3
3
+ transformers>=5.17,<6
4
+ huggingface-hub>=1.5.0,<2.0
5
  sacrebleu>=2.4
6
  matplotlib>=3.8
7
  tqdm>=4.66
8
+ wandb>=0.19
source/part2/train.py CHANGED
@@ -45,6 +45,11 @@ def get_arguments(name):
45
  parser.add_argument("--ns-epsilon", type=float, default=1e-7, help="Muon matrix normalization epsilon.")
46
  parser.add_argument("--aux-learning-rate", type=float, default=3e-4,
47
  help="Muon's custom AdamW learning rate for embeddings/head/biases.")
 
 
 
 
 
48
  parser.add_argument("--max-grad-norm", type=float, default=1.0)
49
  parser.add_argument("--prompt-tokens", type=int, default=128)
50
  parser.add_argument("--max-new-tokens", type=int, default=64)
@@ -62,7 +67,11 @@ def get_arguments(name):
62
  if name == "lion":
63
  parser.set_defaults(learning_rate=1e-4, beta2=0.99)
64
  elif name == "muon":
65
- parser.set_defaults(learning_rate=0.005, mixed_precision=False)
 
 
 
 
66
  args = parser.parse_args()
67
  if args.evaluation_fraction is None:
68
  args.evaluation_fraction = args.data_fraction
@@ -79,9 +88,26 @@ def get_arguments(name):
79
  parser.error("Token caps must be nonnegative and learning rates positive.")
80
  if args.max_grad_norm <= 0 or args.weight_decay < 0:
81
  parser.error("Use positive max-grad-norm and nonnegative weight-decay.")
 
 
82
  return args
83
 
84
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
85
  def get_device(name):
86
  if name == "auto":
87
  if torch.cuda.is_available():
@@ -226,8 +252,10 @@ def run_training(name, make_optimizer, args):
226
  raise ValueError("Resume checkpoint has a different optimizer or dataset selection.")
227
  ignored_args = {"device", "output_dir", "resume", "prepare_only", "mixed_precision"}
228
  current_args = {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}
229
- for key, value in checkpoint["args"].items():
230
- if key not in ignored_args and current_args[key] != value:
 
 
231
  raise ValueError(f"Resume argument changed: {key}.")
232
  model.load_state_dict(checkpoint["model_state_dict"])
233
  optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
@@ -242,6 +270,7 @@ def run_training(name, make_optimizer, args):
242
  })
243
  history = checkpoint["history"]
244
 
 
245
  tokenizer.save_pretrained(directory / "tokenizer")
246
  groups = [{key: value for key, value in group.items() if key != "params"}
247
  for group in optimizer.param_groups]
@@ -249,7 +278,8 @@ def run_training(name, make_optimizer, args):
249
  for key, value in vars(args).items()},
250
  "model": asdict(config), "parameters": state["parameters"],
251
  "optimizer_groups": groups, "data": prepared["metadata"],
252
- "learning_rate_schedule": "constant", "evaluation_interval": "0.1 of selected training budget",
 
253
  "precision_changes": state.get("precision_changes", []),
254
  "bleu_definition": "Greedy continuation after prompt_tokens; reference is the next max_new_tokens tokens."}
255
  (directory / "settings.json").write_text(json.dumps(settings, indent=2))
@@ -264,7 +294,8 @@ def run_training(name, make_optimizer, args):
264
  if args.resume and log_path.exists():
265
  with log_path.open() as stream:
266
  previous_rows = [row for row in csv.DictReader(stream) if int(row["step"]) <= state["step"]]
267
- columns = ["step", "train_tokens", "batch_tokens", "loss", "gradient_norm", "update_skipped", "step_seconds"]
 
268
  boundaries = sorted(set(math.ceil(budget * fraction / 10) for fraction in range(1, 11)))
269
  progress = tqdm(total=budget, initial=state["position"], desc=name, unit="tok")
270
  with log_path.open("w", newline="") as stream:
@@ -303,6 +334,7 @@ def run_training(name, make_optimizer, args):
303
  if not math.isfinite(gradient_norm) and not scaler.is_enabled():
304
  raise RuntimeError("Gradient norm is not finite; the last evaluation checkpoint is saved.")
305
  previous_scale = scaler.get_scale()
 
306
  scaler.step(optimizer)
307
  scaler.update()
308
  skipped = scaler.get_scale() < previous_scale
@@ -319,7 +351,8 @@ def run_training(name, make_optimizer, args):
319
  writer.writerow({"step": state["step"], "train_tokens": state["position"],
320
  "batch_tokens": count, "loss": loss_value,
321
  "gradient_norm": state["last_gradient_norm"], "update_skipped": skipped,
322
- "step_seconds": seconds})
 
323
  progress.update(count)
324
  progress.set_postfix(loss=f"{loss_value:.4f}")
325
  stream.flush()
 
45
  parser.add_argument("--ns-epsilon", type=float, default=1e-7, help="Muon matrix normalization epsilon.")
46
  parser.add_argument("--aux-learning-rate", type=float, default=3e-4,
47
  help="Muon's custom AdamW learning rate for embeddings/head/biases.")
48
+ parser.add_argument("--lr-schedule", choices=("constant", "cosine"), default="constant")
49
+ parser.add_argument("--warmup-fraction", type=float, default=0.05,
50
+ help="Fraction of training tokens used for linear warmup with cosine scheduling.")
51
+ parser.add_argument("--min-lr-ratio", type=float, default=0.1,
52
+ help="Final learning rate / peak learning rate for cosine scheduling.")
53
  parser.add_argument("--max-grad-norm", type=float, default=1.0)
54
  parser.add_argument("--prompt-tokens", type=int, default=128)
55
  parser.add_argument("--max-new-tokens", type=int, default=64)
 
67
  if name == "lion":
68
  parser.set_defaults(learning_rate=1e-4, beta2=0.99)
69
  elif name == "muon":
70
+ parser.set_defaults(learning_rate=1e-3, aux_learning_rate=1e-4,
71
+ beta2=0.95, lr_schedule="cosine", mixed_precision=False,
72
+ batch_size=64,
73
+ output_dir=str(HERE / "part2_results" / "output" / name)
74
+ if (HERE / "part2_results").is_dir() else str(HERE / "output" / name))
75
  args = parser.parse_args()
76
  if args.evaluation_fraction is None:
77
  args.evaluation_fraction = args.data_fraction
 
88
  parser.error("Token caps must be nonnegative and learning rates positive.")
89
  if args.max_grad_norm <= 0 or args.weight_decay < 0:
90
  parser.error("Use positive max-grad-norm and nonnegative weight-decay.")
91
+ if not 0 <= args.warmup_fraction < 1 or not 0 < args.min_lr_ratio <= 1:
92
+ parser.error("Use warmup-fraction in [0, 1) and min-lr-ratio in (0, 1].")
93
  return args
94
 
95
 
96
+ def set_learning_rates(optimizer, args, position, budget):
97
+ """Schedule each group's peak rate by tokens, including after a checkpoint resume."""
98
+ if args.lr_schedule == "constant":
99
+ return
100
+ progress = min(max(position / budget, 0.0), 1.0)
101
+ if args.warmup_fraction and progress < args.warmup_fraction:
102
+ factor = progress / args.warmup_fraction
103
+ else:
104
+ decay = (progress - args.warmup_fraction) / (1 - args.warmup_fraction)
105
+ factor = args.min_lr_ratio + (1 - args.min_lr_ratio) * 0.5 * (1 + math.cos(math.pi * decay))
106
+ for group in optimizer.param_groups:
107
+ group.setdefault("peak_lr", group["lr"])
108
+ group["lr"] = group["peak_lr"] * factor
109
+
110
+
111
  def get_device(name):
112
  if name == "auto":
113
  if torch.cuda.is_available():
 
252
  raise ValueError("Resume checkpoint has a different optimizer or dataset selection.")
253
  ignored_args = {"device", "output_dir", "resume", "prepare_only", "mixed_precision"}
254
  current_args = {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}
255
+ schedule_defaults = {"lr_schedule": "constant", "warmup_fraction": 0.05, "min_lr_ratio": 0.1}
256
+ for key, value in current_args.items():
257
+ saved_value = checkpoint["args"].get(key, schedule_defaults.get(key))
258
+ if key not in ignored_args and saved_value != value:
259
  raise ValueError(f"Resume argument changed: {key}.")
260
  model.load_state_dict(checkpoint["model_state_dict"])
261
  optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
 
270
  })
271
  history = checkpoint["history"]
272
 
273
+ set_learning_rates(optimizer, args, state["position"], budget)
274
  tokenizer.save_pretrained(directory / "tokenizer")
275
  groups = [{key: value for key, value in group.items() if key != "params"}
276
  for group in optimizer.param_groups]
 
278
  for key, value in vars(args).items()},
279
  "model": asdict(config), "parameters": state["parameters"],
280
  "optimizer_groups": groups, "data": prepared["metadata"],
281
+ "learning_rate_schedule": args.lr_schedule,
282
+ "evaluation_interval": "0.1 of selected training budget",
283
  "precision_changes": state.get("precision_changes", []),
284
  "bleu_definition": "Greedy continuation after prompt_tokens; reference is the next max_new_tokens tokens."}
285
  (directory / "settings.json").write_text(json.dumps(settings, indent=2))
 
294
  if args.resume and log_path.exists():
295
  with log_path.open() as stream:
296
  previous_rows = [row for row in csv.DictReader(stream) if int(row["step"]) <= state["step"]]
297
+ rate_columns = [f"learning_rate_{index}" for index in range(len(optimizer.param_groups))]
298
+ columns = ["step", "train_tokens", "batch_tokens", "loss", "gradient_norm", "update_skipped", "step_seconds"] + rate_columns
299
  boundaries = sorted(set(math.ceil(budget * fraction / 10) for fraction in range(1, 11)))
300
  progress = tqdm(total=budget, initial=state["position"], desc=name, unit="tok")
301
  with log_path.open("w", newline="") as stream:
 
334
  if not math.isfinite(gradient_norm) and not scaler.is_enabled():
335
  raise RuntimeError("Gradient norm is not finite; the last evaluation checkpoint is saved.")
336
  previous_scale = scaler.get_scale()
337
+ set_learning_rates(optimizer, args, state["position"] + count, budget)
338
  scaler.step(optimizer)
339
  scaler.update()
340
  skipped = scaler.get_scale() < previous_scale
 
351
  writer.writerow({"step": state["step"], "train_tokens": state["position"],
352
  "batch_tokens": count, "loss": loss_value,
353
  "gradient_norm": state["last_gradient_norm"], "update_skipped": skipped,
354
+ "step_seconds": seconds,
355
+ **{column: group["lr"] for column, group in zip(rate_columns, optimizer.param_groups)}})
356
  progress.update(count)
357
  progress.set_postfix(loss=f"{loss_value:.4f}")
358
  stream.flush()
source/part2/train_muon.py CHANGED
@@ -1,4 +1,4 @@
1
- """Muon: matrix-based optimizer category (paper Algorithm 8)."""
2
 
3
  import math
4
  import sys
@@ -27,7 +27,7 @@ def newton_schulz(matrix, steps=5, eps=1e-7):
27
 
28
  for _ in range(steps):
29
  gram = x @ x.transpose(-2, -1)
30
- # The PDF's polynomial has a typo; these are the original Muon coefficients.
31
  correction = -4.7750 * gram + 2.0315 * (gram @ gram)
32
  x = 3.4445 * x + correction @ x
33
 
@@ -59,8 +59,7 @@ class Muon(Optimizer):
59
  for current, saved in zip(self.param_groups, state_dict["param_groups"]):
60
  if current["use_muon"] and any(p.ndim == 3 for p in current["params"]):
61
  if saved.get("attention_layout") != "full_projection":
62
- raise ValueError("This Muon checkpoint used per-head updates. Start a fresh run "
63
- "with the corrected attention layout.")
64
  super().load_state_dict(state_dict)
65
 
66
  @torch.no_grad()
@@ -105,16 +104,21 @@ class Muon(Optimizer):
105
 
106
  def make_optimizer(model, args):
107
  matrix_parameters = []
108
- other_parameters = []
 
109
  for name, parameter in model.named_parameters():
110
  is_embedding_or_head = name.startswith("embedding.") or name.startswith("unembedding.")
111
  if parameter.ndim >= 2 and not is_embedding_or_head:
112
  matrix_parameters.append(parameter)
 
 
113
  else:
114
- other_parameters.append(parameter)
115
  groups = [
116
  {"params": matrix_parameters, "lr": args.learning_rate, "use_muon": True},
117
- {"params": other_parameters, "lr": args.aux_learning_rate, "use_muon": False},
 
 
118
  ]
119
  return Muon(groups, momentum=args.momentum, ns_steps=args.ns_steps,
120
  betas=(args.beta1, args.beta2), eps=args.epsilon,
 
1
+ """Muon with conservative rates, full attention projections and auxiliary AdamW."""
2
 
3
  import math
4
  import sys
 
27
 
28
  for _ in range(steps):
29
  gram = x @ x.transpose(-2, -1)
30
+ # Muon's quintic Newton-Schulz polynomial.
31
  correction = -4.7750 * gram + 2.0315 * (gram @ gram)
32
  x = 3.4445 * x + correction @ x
33
 
 
59
  for current, saved in zip(self.param_groups, state_dict["param_groups"]):
60
  if current["use_muon"] and any(p.ndim == 3 for p in current["params"]):
61
  if saved.get("attention_layout") != "full_projection":
62
+ raise ValueError("Muon checkpoints must use the full_projection attention layout.")
 
63
  super().load_state_dict(state_dict)
64
 
65
  @torch.no_grad()
 
104
 
105
  def make_optimizer(model, args):
106
  matrix_parameters = []
107
+ auxiliary_weights = []
108
+ gains_and_biases = []
109
  for name, parameter in model.named_parameters():
110
  is_embedding_or_head = name.startswith("embedding.") or name.startswith("unembedding.")
111
  if parameter.ndim >= 2 and not is_embedding_or_head:
112
  matrix_parameters.append(parameter)
113
+ elif parameter.ndim >= 2:
114
+ auxiliary_weights.append(parameter)
115
  else:
116
+ gains_and_biases.append(parameter)
117
  groups = [
118
  {"params": matrix_parameters, "lr": args.learning_rate, "use_muon": True},
119
+ {"params": auxiliary_weights, "lr": args.aux_learning_rate, "use_muon": False},
120
+ {"params": gains_and_biases, "lr": args.aux_learning_rate, "use_muon": False,
121
+ "weight_decay": 0.0},
122
  ]
123
  return Muon(groups, momentum=args.momentum, ns_steps=args.ns_steps,
124
  betas=(args.beta1, args.beta2), eps=args.epsilon,
source/part2/update_report.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Fill the Muon report row from a completed full-budget run."""
2
+
3
+ import argparse
4
+ import json
5
+ from pathlib import Path
6
+ import shutil
7
+
8
+
9
+ def main():
10
+ here = Path(__file__).resolve().parent
11
+ parser = argparse.ArgumentParser(description=__doc__)
12
+ parser.add_argument("--output-dir", type=Path, default=here / "part2_results" / "output")
13
+ args = parser.parse_args()
14
+ directory = args.output_dir / "muon"
15
+ history = json.loads((directory / "metrics.json").read_text())
16
+ settings = json.loads((directory / "settings.json").read_text())
17
+ final = history[-1]
18
+ if (final["train_tokens"] != settings["data"]["train_target_tokens"]
19
+ or not (directory / "final.pt").is_file()):
20
+ raise ValueError("The report requires a completed pass over the selected dataset.")
21
+ report = here.parent / "report.md"
22
+ if not report.is_file():
23
+ print("No report.md found; recorded results are available in", directory)
24
+ return
25
+ peak = settings["arguments"]["learning_rate"]
26
+ auxiliary = settings["arguments"]["aux_learning_rate"]
27
+ checkpoint = directory.resolve().relative_to(here.parent.resolve()) / "final.pt"
28
+ run_link = f"[Local checkpoint]({checkpoint.as_posix()})"
29
+ wandb_file = directory / ".wandb_upload.json"
30
+ if wandb_file.is_file():
31
+ uploaded = json.loads(wandb_file.read_text())
32
+ run_link = f"[W&B]({uploaded['url']})"
33
+ row = (f"| Muon | {peak:g} / {auxiliary:g} auxiliary | "
34
+ f"{final['validation_loss']:.4f} | {final['test_perplexity']:.3f} | "
35
+ f"{final['test_bleu']:.4f} | {final['training_seconds']:.1f} | "
36
+ f"{run_link} |")
37
+ text = report.read_text()
38
+ text = "\n".join(row if line.startswith("| Muon |") else line for line in text.splitlines()) + "\n"
39
+ text = text.replace("Muon results are pending completion of its configured training budget.",
40
+ "Muon completed its configured training budget.")
41
+ huggingface_file = directory / ".huggingface_upload.json"
42
+ if huggingface_file.is_file():
43
+ uploaded = json.loads(huggingface_file.read_text())
44
+ lines = text.splitlines()
45
+ for index, line in enumerate(lines):
46
+ if line.startswith("Checkpoints:") and "[AdamW]" in line:
47
+ # Regenerate only Muon's checkpoint link; preserve the other optimizers.
48
+ line = line.split(", [Muon]")[0].split(". Each folder")[0]
49
+ lines[index] = line.rstrip(". ") + f", [Muon]({uploaded['url']}). Each folder has `final.pt` and `latest.pt`."
50
+ break
51
+ text = "\n".join(lines) + "\n"
52
+ report.write_text(text)
53
+ shutil.copy2(args.output_dir / "comparison_dataset_dynamics.png", here.parent / "assets/part2_dynamics.png")
54
+ print(f"Updated {report} from {final['train_tokens']:,} Muon training tokens.")
55
+
56
+
57
+ if __name__ == "__main__":
58
+ main()