Upload Part 2 checkpoints and recorded results
Browse files- .gitattributes +2 -0
- README.md +2 -2
- comparison_dataset_dynamics.png +3 -0
- comparison_dataset_metrics.csv +45 -0
- comparison_test_bleu.png +2 -2
- comparison_test_bleu_vs_dataset.png +3 -0
- comparison_validation_loss.png +0 -0
- comparison_validation_loss_vs_dataset.png +0 -0
- muon/bleu_vs_tokens.png +0 -0
- muon/final.pt +2 -2
- muon/latest.pt +2 -2
- muon/loss_vs_tokens.png +0 -0
- muon/metrics.csv +11 -11
- muon/metrics.json +190 -179
- muon/muon.log +0 -0
- muon/predictions_00.jsonl +20 -20
- muon/predictions_01.jsonl +0 -0
- muon/predictions_02.jsonl +0 -0
- muon/predictions_03.jsonl +0 -0
- muon/predictions_04.jsonl +0 -0
- muon/predictions_05.jsonl +0 -0
- muon/predictions_06.jsonl +0 -0
- muon/predictions_07.jsonl +0 -0
- muon/predictions_08.jsonl +0 -0
- muon/predictions_09.jsonl +0 -0
- muon/predictions_10.jsonl +0 -0
- muon/settings.json +39 -23
- muon/training_steps.csv +0 -0
- source/part2/README.md +49 -228
- source/part2/plot_comparison.py +54 -13
- source/part2/requirements.txt +3 -1
- source/part2/train.py +39 -6
- source/part2/train_muon.py +11 -7
- source/part2/update_report.py +58 -0
.gitattributes
CHANGED
|
@@ -34,3 +34,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
comparison_test_bleu.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
comparison_test_bleu.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
comparison_dataset_dynamics.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
comparison_test_bleu_vs_dataset.png filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -23,11 +23,11 @@ sys.path.insert(0, str(folder / "source"))
|
|
| 23 |
from part2.model import DecoderOnlyTransformer, TransformerConfig
|
| 24 |
from part2.data import load_tokenizer
|
| 25 |
|
| 26 |
-
checkpoint = torch.load(folder / '
|
| 27 |
model = DecoderOnlyTransformer(TransformerConfig(**checkpoint["config"]), False, 0, 0, False)
|
| 28 |
model.load_state_dict(checkpoint["model_state_dict"])
|
| 29 |
model.eval()
|
| 30 |
-
tokenizer = load_tokenizer(str(folder / '
|
| 31 |
```
|
| 32 |
|
| 33 |
The final checkpoints include optimizer/scaler state for resuming training.
|
|
|
|
| 23 |
from part2.model import DecoderOnlyTransformer, TransformerConfig
|
| 24 |
from part2.data import load_tokenizer
|
| 25 |
|
| 26 |
+
checkpoint = torch.load(folder / 'muon' / "final.pt", map_location="cpu", weights_only=True)
|
| 27 |
model = DecoderOnlyTransformer(TransformerConfig(**checkpoint["config"]), False, 0, 0, False)
|
| 28 |
model.load_state_dict(checkpoint["model_state_dict"])
|
| 29 |
model.eval()
|
| 30 |
+
tokenizer = load_tokenizer(str(folder / 'muon' / "tokenizer"))
|
| 31 |
```
|
| 32 |
|
| 33 |
The final checkpoints include optimizer/scaler state for resuming training.
|
comparison_dataset_dynamics.png
ADDED
|
Git LFS Details
|
comparison_dataset_metrics.csv
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
optimizer,step,successful_updates,skipped_updates,train_tokens,training_budget_tokens,selected_dataset_fraction,full_dataset_token_fraction,tokens_per_parameter,train_loss,train_perplexity,cumulative_train_loss,validation_loss,validation_perplexity,validation_tokens,test_loss,test_perplexity,test_tokens,test_bleu,bleu_signature,bleu_samples,generated_tokens,learning_rates,last_gradient_norm,training_seconds,evaluation_seconds,total_evaluation_seconds,train_tokens_per_second,prediction_file,experiment_dataset_tokens,experiment_dataset_passes,original_train_split_tokens
|
| 2 |
+
adamw,0,0,0,0,42733510,0.0,0.0,0.0,,,,6.152310108351061,469.80142637427707,720136,6.153817186992487,470.5099878645091,708110,0.002109752683516993,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],,0.0,12.185807590999957,12.185807590999957,,predictions_00.jsonl,42733510,0.0,170790691
|
| 3 |
+
adamw,261,261,0,4273351,42733510,0.1,0.025020983140117397,1.273721524692013,2.640498514812532,14.02019113885227,2.640498514812532,2.442611973858063,11.503047196263836,720136,2.44764719613434,11.561113661849923,708110,0.05532590836351658,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],0.45159417390823364,40.88506411700109,12.159935426000004,24.34574301699996,104521.07859659751,predictions_01.jsonl,42733510,0.1,170790691
|
| 4 |
+
adamw,522,522,0,8546702,42733510,0.2,0.050041966280234794,2.547443049384026,2.2929823701285956,9.904432360498767,2.466740442470564,2.1418738580716004,8.515379300241122,720136,2.1500780604575915,8.585528561309054,708110,0.10125930673494067,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.0839279890060425,82.78312815800115,12.447991385000023,36.793734401999984,103242.07589362453,predictions_02.jsonl,42733510,0.2,170790691
|
| 5 |
+
adamw,783,783,0,12820053,42733510,0.3,0.07506294942035219,3.821164574076039,2.025029414033044,7.576333791891917,2.319503432991391,1.9197091547638223,6.818974914481506,720136,1.9306420518837115,6.8939350849472545,708110,0.1722466981765809,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.8015629053115845,124.17741831200016,12.350141832000077,49.14387623400006,103239.80941356957,predictions_03.jsonl,42733510,0.3,170790691
|
| 6 |
+
adamw,1044,1044,0,17093404,42733510,0.4,0.10008393256046959,5.094886098768052,1.8591893168135667,6.418531266327286,2.2044249039469346,1.76821006672264,5.860354324675273,720136,1.7770830621081117,5.912584598370209,708110,0.24831101149416698,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.264988899230957,165.69403289600137,12.310649127000033,61.454525361000094,103162.45975332591,predictions_04.jsonl,42733510,0.4,170790691
|
| 7 |
+
adamw,1305,1305,0,21366755,42733510,0.5,0.125104915700587,6.368607623460065,1.731667797280856,5.650069224916237,2.109873482613719,1.66693166833244,5.295893285074366,720136,1.6760529018266936,5.344419336577045,708110,0.26162517431091903,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],2.0656886100769043,207.10014180200125,12.336321245000022,73.79084660600012,103171.12684755067,predictions_05.jsonl,42733510,0.5,170790691
|
| 8 |
+
adamw,1566,1566,0,25640106,42733510,0.6,0.15012589884070437,7.642329148152078,1.6438578430091555,5.175095758349662,2.0322042093462915,1.6001972305404812,4.954009409999732,720136,1.6092371785126076,4.998996431121064,708110,0.410308644723639,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.3277252912521362,248.49769949300202,12.298542177000058,86.08938878300017,103180.45620668636,predictions_06.jsonl,42733510,0.6,170790691
|
| 9 |
+
adamw,1827,1827,0,29913457,42733510,0.7,0.1751468819808218,8.916050672844092,1.5925767979894636,4.916401192870087,1.9694002934381734,1.5608394358409383,4.762817647378607,720136,1.566571245496374,4.790195599205834,708110,0.3047428372679637,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.0786640644073486,289.87049320100147,12.298826415000008,98.38821519800018,103195.93646690166,predictions_07.jsonl,42733510,0.7,170790691
|
| 10 |
+
adamw,2088,2088,0,34186808,42733510,0.8,0.20016786512093918,10.189772197536104,1.5402283544145818,4.665655572397232,1.9157538010602244,1.523367808302876,4.587649531642508,720136,1.5293414451188727,4.615136500630765,708110,0.5040661650051995,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.9299545288085938,331.26377932600093,12.284402999000008,110.67261819700019,103201.16515472198,predictions_08.jsonl,42733510,0.8,170790691
|
| 11 |
+
adamw,2349,2349,0,38460159,42733510,0.9,0.22518884826105656,11.463493722228117,1.5047946790418072,4.503228928082886,1.8700916763915114,1.485235500980891,4.416005263902061,720136,1.491143989921234,4.442174416106604,708110,0.4872224185479862,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.259429931640625,372.6202616480012,12.283753058000002,122.95637125500019,103215.42588666771,predictions_09.jsonl,42733510,0.9,170790691
|
| 12 |
+
adamw,2610,2610,0,42733510,42733510,1.0,0.250209831401174,12.73721524692013,1.4782456321713036,4.385245594709652,1.8309070719694907,1.4478944097668704,4.254147586534556,720136,1.452432387551678,4.273496683456494,708110,0.47516893154716505,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.2657051086425781,413.9625683530009,12.29498626499992,135.2513575200001,103230.37218079965,predictions_10.jsonl,42733510,1.0,170790691
|
| 13 |
+
mars,0,0,0,0,42733510,0.0,0.0,0.0,,,,6.152309945621383,469.80134992364816,720136,6.1538172007836,470.5099943533652,708110,0.002109752683516993,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],,0.0,12.137210432000018,12.137210432000018,,predictions_00.jsonl,42733510,0.0,170790691
|
| 14 |
+
mars,261,261,0,4273351,42733510,0.1,0.025020983140117397,1.273721524692013,2.6350216016477592,13.943613664940504,2.6350216016477592,2.43761234857027,11.445679797860048,720136,2.4428096350341932,11.505321126828223,708110,0.06660427037878557,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],0.8482398390769958,42.77929676000008,12.14897425700002,24.286184689000038,99892.96981608428,predictions_01.jsonl,42733510,0.1,170790691
|
| 15 |
+
mars,522,522,0,8546702,42733510,0.2,0.050041966280234794,2.547443049384026,2.2968188837789993,9.942503834617087,2.4659202427133793,2.1463869595899245,8.553896923083718,720136,2.1558107833925875,8.634888365662338,708110,0.12884781642453785,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.0802031755447388,86.43895843200028,12.409839562999991,36.69602425200003,98875.57826976261,predictions_02.jsonl,42733510,0.2,170790691
|
| 16 |
+
mars,783,783,0,12820053,42733510,0.3,0.07506294942035219,3.821164574076039,2.0407106237667065,7.6960762708417425,2.3241837030644885,1.957633121990152,7.08254369397268,720136,1.9675556565187613,7.153170296305804,708110,0.15496398107655524,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.1142001152038574,129.72636143600107,12.415474795000023,49.11149904700005,98823.80772950775,predictions_03.jsonl,42733510,0.3,170790691
|
| 17 |
+
mars,1044,1044,0,17093404,42733510,0.4,0.10008393256046959,5.094886098768052,1.8945741906420932,6.6497162902814395,2.2167813249588897,1.81029685169711,6.112261598176767,720136,1.820918442198908,6.177529549019345,708110,0.17599086736363748,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.237196922302246,173.21508395600074,12.36834270899999,61.47984175600004,98683.1147127001,predictions_04.jsonl,42733510,0.4,170790691
|
| 18 |
+
mars,1305,1305,0,21366755,42733510,0.5,0.125104915700587,6.368607623460065,1.7801481340870975,5.930734897523064,2.129454686784531,1.714784321121288,5.555477184049246,720136,1.724313380479816,5.608668685895299,708110,0.24534585874439038,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.592305302619934,216.62708447400064,12.436493875999986,73.91633563200003,98633.81142704903,predictions_05.jsonl,42733510,0.5,170790691
|
| 19 |
+
mars,1566,1566,0,25640106,42733510,0.6,0.15012589884070437,7.642329148152078,1.6931412306945999,5.43653131019262,2.0567357774362094,1.657125382610021,5.244214047188049,720136,1.6676527921542468,5.299713657195096,708110,0.2440907782990909,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.0193804502487183,260.1115367720008,12.393600711999966,86.309936344,98573.50549766148,predictions_06.jsonl,42733510,0.6,170790691
|
| 20 |
+
mars,1827,1827,0,29913457,42733510,0.7,0.1751468819808218,8.916050672844092,1.6378689231330386,5.144195147502627,1.9968976553928992,1.601110594257543,4.958536289477835,720136,1.609088186913271,4.99825167813006,708110,0.27954084634942006,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.314588189125061,303.5638545300005,12.412496107999914,98.72243245199991,98540.90516248772,predictions_07.jsonl,42733510,0.7,170790691
|
| 21 |
+
mars,2088,2088,0,34186808,42733510,0.8,0.20016786512093918,10.189772197536104,1.5845113037937653,4.876907469877936,1.9453493614430073,1.5681297796961269,4.7976671036559315,720136,1.5752427994392644,4.831914661867808,708110,0.291937689998352,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.9565479755401611,347.0294731490004,12.407213007999985,111.12964545999989,98512.69314327538,predictions_08.jsonl,42733510,0.8,170790691
|
| 22 |
+
mars,2349,2349,0,38460159,42733510,0.9,0.22518884826105656,11.463493722228117,1.5487290109711367,4.705485759552475,1.9012804336127995,1.5326619945746358,4.630486761171459,720136,1.5411480120993295,4.669948352040108,708110,0.46672146049280167,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],2.234539747238159,390.5177544279994,12.427969979000068,123.55761543899996,98485.04597782885,predictions_09.jsonl,42733510,0.9,170790691
|
| 23 |
+
mars,2610,2610,0,42733510,42733510,1.0,0.250209831401174,12.73721524692013,1.5211646478296674,4.577553329390168,1.8632688550344862,1.4913745594799108,4.443198764388519,720136,1.49689860280028,4.4678111040721165,708110,0.4529043189821306,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0003],1.104906678199768,433.93933552499936,12.412074367999935,135.9696898069999,98478.07401073487,predictions_10.jsonl,42733510,1.0,170790691
|
| 24 |
+
lion,0,0,0,0,42733510,0.0,0.0,0.0,,,,6.152310151745643,469.8014467611137,720136,6.153817203541823,470.5099956511367,708110,0.002109752683516993,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],,0.0,12.305204914,12.305204914,,predictions_00.jsonl,42733510,0.0,170790691
|
| 25 |
+
lion,261,261,0,4273351,42733510,0.1,0.025020983140117397,1.273721524692013,2.8078204490104692,16.57375547868066,2.8078204490104692,2.493524042246881,12.103855548831,720136,2.497530412989772,12.152445350965934,708110,0.023491054883226305,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],0.5989791750907898,40.75469877799992,12.05658539000001,24.36179030400001,104855.41859302927,predictions_01.jsonl,42733510,0.1,170790691
|
| 26 |
+
lion,522,522,0,8546702,42733510,0.2,0.050041966280234794,2.547443049384026,2.438017814129468,11.450321567795019,2.6229191315699687,2.335367805091677,10.33325985609919,720136,2.344312593283088,10.426103290238618,708110,0.06359788640230295,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],0.7433375120162964,81.26354952799852,12.179590484000073,36.54138078800008,105172.63951232308,predictions_02.jsonl,42733510,0.2,170790691
|
| 27 |
+
lion,783,783,0,12820053,42733510,0.3,0.07506294942035219,3.821164574076039,2.192337917007256,8.95612733973809,2.4793920600490646,2.0457414521788246,7.734891464431085,720136,2.0569859330923603,7.822357135321954,708110,0.1473893318542731,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],1.2614325284957886,121.98602876099653,12.17860739400021,48.71998818200029,105094.43688111148,predictions_03.jsonl,42733510,0.3,170790691
|
| 28 |
+
lion,1044,1044,0,17093404,42733510,0.4,0.10008393256046959,5.094886098768052,1.9468686484814643,7.0067127125346875,2.3462612071571645,1.822833350879209,6.189370287323649,720136,1.8345315415441281,6.262199877713038,708110,0.25862362069881323,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],1.393475890159607,162.67717684499303,12.244096438000042,60.964084620000335,105075.61251992622,predictions_04.jsonl,42733510,0.4,170790691
|
| 29 |
+
lion,1305,1305,0,21366755,42733510,0.5,0.125104915700587,6.368607623460065,1.758935989300471,5.806256187842715,2.2287961635858258,1.6703211946467404,5.313874311087103,720136,1.6780224823826806,5.354955973957277,708110,0.23976036675370394,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],1.1012609004974365,203.45306142699064,12.25387805499986,73.2179626750002,105020.56272899822,predictions_05.jsonl,42733510,0.5,170790691
|
| 30 |
+
lion,1566,1566,0,25640106,42733510,0.6,0.15012589884070437,7.642329148152078,1.6225743659270604,5.066115578803247,2.127759197309365,1.5729091530757733,4.820651828634709,720136,1.5809962055959261,4.859794755275936,708110,0.4276805014235463,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],1.2251908779144287,244.21673679899163,12.240612259000045,85.45857493400024,104989.14339807798,predictions_06.jsonl,42733510,0.6,170790691
|
| 31 |
+
lion,1827,1827,0,29913457,42733510,0.7,0.1751468819808218,8.916050672844092,1.5423915518299942,4.6757592306364,2.044135247955169,1.501913337593524,4.490272263173101,720136,1.5057976073632628,4.507747609490873,708110,0.5148404590813516,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],1.1267120838165283,285.19143254099345,12.46664583200004,97.92522076600028,104889.04499506744,predictions_07.jsonl,42733510,0.7,170790691
|
| 32 |
+
lion,2088,2088,0,34186808,42733510,0.8,0.20016786512093918,10.189772197536104,1.4766037938191805,4.378051637582159,1.9731938161881706,1.456802736848491,4.292214227993766,720136,1.4601326796088354,4.306530879273558,708110,0.6183258743967537,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],1.731221318244934,326.1571126909937,12.370582222999928,110.29580298900021,104816.9936198482,predictions_08.jsonl,42733510,0.8,170790691
|
| 33 |
+
lion,2349,2349,0,38460159,42733510,0.9,0.22518884826105656,11.463493722228117,1.4372478736383814,4.2090958983168925,1.9136442670159717,1.4123747599845724,4.105693873750749,720136,1.4161928332164582,4.121399677503366,708110,0.47246291242006627,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],0.9133048057556152,367.225464117995,12.380042230000072,122.67584521900028,104731.73229523697,predictions_09.jsonl,42733510,0.9,170790691
|
| 34 |
+
lion,2610,2610,0,42733510,42733510,1.0,0.250209831401174,12.73721524692013,1.4083791706187379,4.089321936519697,1.8631177573762483,1.3890832556281036,4.011171148369319,720136,1.391464327824464,4.020733416172768,708110,0.5714227299959309,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,[0.0001],0.9298375248908997,408.1954670909936,12.43709894199992,135.1129441610002,104688.84013960397,predictions_10.jsonl,42733510,1.0,170790691
|
| 35 |
+
muon,0,0,0,0,42733510,0.0,0.0,0.0,,,,6.152294551393765,469.7941177503992,720136,6.153802047108677,470.50286445188505,708110,0.002089086296127387,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0, 0.0, 0.0]",,0.0,82.7089025840396,82.7089025840396,,predictions_00.jsonl,42733510,0.0,170790691
|
| 36 |
+
muon,261,261,0,4273351,42733510,0.1,0.025020983140117397,1.273721524692013,4.0481406036315155,57.290831582475136,4.0481406036315155,2.5621703693382223,12.963923305966714,720136,2.5636621853640325,12.98327752739135,708110,0.03083398708662475,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0009938625865312252, 9.938625865312251e-05, 9.938625865312251e-05]",0.3203253746032715,281.1368540887488,78.27012733393349,160.9790299179731,15200.25189814138,predictions_01.jsonl,42733510,0.1,170790691
|
| 37 |
+
muon,522,522,0,8546702,42733510,0.2,0.050041966280234794,2.547443049384026,2.5045045028839676,12.237493820398619,3.2763225532577414,2.463011343300085,11.740111869820579,720136,2.467185018978778,11.789213685445926,708110,0.05582133505440844,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0009457631880429201, 9.4576318804292e-05, 9.4576318804292e-05]",0.774559497833252,568.6849648010684,79.9007168749813,240.8797467929544,15028.886868830303,predictions_02.jsonl,42733510,0.2,170790691
|
| 38 |
+
muon,783,783,0,12820053,42733510,0.3,0.07506294942035219,3.821164574076039,2.430226130058437,11.36145095510898,2.9942904121913068,2.3827435894053344,10.834587783949875,720136,2.39104282934943,10.92488079106834,708110,0.05654392092884844,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0008547767072315835, 8.547767072315835e-05, 8.547767072315835e-05]",0.8050723671913147,840.1531296785688,72.94383920903783,313.8235860019922,15259.186149678188,predictions_03.jsonl,42733510,0.3,170790691
|
| 39 |
+
muon,1044,1044,0,17093404,42733510,0.4,0.10008393256046959,5.094886098768052,2.3242174275854817,10.218680098756618,2.8267721660398504,2.223032372531022,9.235293297912508,720136,2.2355781408321977,9.351886997309776,708110,0.100897461835075,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0007307629410938362, 7.307629410938362e-05, 7.307629410938362e-05]",0.6698967218399048,1115.7324636415578,77.46016829099972,391.28375429299194,15320.342964844893,predictions_04.jsonl,42733510,0.4,170790691
|
| 40 |
+
muon,1305,1305,0,21366755,42733510,0.5,0.125104915700587,6.368607623460065,2.1430165465797297,8.525115287851063,2.6900210421478263,2.0409670501908668,7.6980500012074335,720136,2.0537061002671284,7.7967431393926105,708110,0.12345122653106588,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0005871607054625496, 5.8716070546254966e-05, 5.8716070546254966e-05]",0.6055232286453247,1406.0170264714397,78.36569400003646,469.6494482930284,15196.654519627198,predictions_05.jsonl,42733510,0.5,170790691
|
| 41 |
+
muon,1566,1566,0,25640106,42733510,0.6,0.15012589884070437,7.642329148152078,1.9805903106765663,7.24701971584848,2.5717825869026165,1.9150039671325971,6.786965721738856,720136,1.9279956241901859,6.875714904050608,708110,0.12877604487102923,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00043953153078664056, 4.3953153078664054e-05, 4.3953153078664054e-05]",0.8913306593894958,1680.1966308491537,73.43593741592485,543.0853857089533,15260.18177232135,predictions_06.jsonl,42733510,0.6,170790691
|
| 42 |
+
muon,1827,1827,0,29913457,42733510,0.7,0.1751468819808218,8.916050672844092,1.884803411397829,6.585059767065248,2.473642704687647,1.8385603145838079,6.287479748988225,720136,1.8502426677341355,6.361363032867387,708110,0.1802229209222577,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00030387332884490803, 3.0387332884490805e-05, 3.0387332884490805e-05]",0.75834721326828,1950.8896681739716,73.07733799994458,616.1627237088978,15333.23872077242,predictions_07.jsonl,42733510,0.7,170790691
|
| 43 |
+
muon,2088,2088,0,34186808,42733510,0.8,0.20016786512093918,10.189772197536104,1.8216860253402747,6.182273136872596,2.3921481197692254,1.7927702604742368,6.006067813606906,720136,1.8042369539932885,6.075333920134396,708110,0.2571039476796964,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00019488677077162295, 1.9488677077162295e-05, 1.9488677077162295e-05]",1.9932366609573364,2219.8103655477753,77.00118562497664,693.1639093338745,15400.778611808957,predictions_08.jsonl,42733510,0.8,170790691
|
| 44 |
+
muon,2349,2349,0,38460159,42733510,0.9,0.22518884826105656,11.463493722228117,1.7873370917559692,5.9735243210402365,2.324946894434419,1.7655405758426186,5.844731024658187,720136,1.7764862606588754,5.909057012052186,708110,0.22627215440136425,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00012438224123471442, 1.2438224123471442e-05, 1.2438224123471442e-05]",0.6744703054428101,2485.9783932112623,73.41921604203526,766.5831253759097,15470.833980306279,predictions_09.jsonl,42733510,0.9,170790691
|
| 45 |
+
muon,2610,2610,0,42733510,42733510,1.0,0.250209831401174,12.73721524692013,1.7692924620928165,5.86670097923902,2.2693814512002586,1.747523807568987,5.740370800119521,720136,1.7582855103506165,5.802480568528633,708110,0.24637446793816298,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0001, 1e-05, 1e-05]",0.7609756588935852,2754.439210804994,73.39668541704305,839.9798107929528,15514.41390769012,predictions_10.jsonl,42733510,1.0,170790691
|
comparison_test_bleu.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
comparison_test_bleu_vs_dataset.png
ADDED
|
Git LFS Details
|
comparison_validation_loss.png
CHANGED
|
|
comparison_validation_loss_vs_dataset.png
ADDED
|
muon/bleu_vs_tokens.png
CHANGED
|
|
muon/final.pt
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b543c13675576ec83ff68c8e175d230b721cf88e3338ed4c2c52b8f3657f2919
|
| 3 |
+
size 28036667
|
muon/latest.pt
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6a5f8e5f2225e0d49620500ac824cf1254e2853103e645e725d851ad731d166e
|
| 3 |
+
size 28039439
|
muon/loss_vs_tokens.png
CHANGED
|
|
muon/metrics.csv
CHANGED
|
@@ -1,12 +1,12 @@
|
|
| 1 |
optimizer,step,successful_updates,skipped_updates,train_tokens,training_budget_tokens,selected_dataset_fraction,full_dataset_token_fraction,tokens_per_parameter,train_loss,train_perplexity,cumulative_train_loss,validation_loss,validation_perplexity,validation_tokens,test_loss,test_perplexity,test_tokens,test_bleu,bleu_signature,bleu_samples,generated_tokens,learning_rates,last_gradient_norm,training_seconds,evaluation_seconds,total_evaluation_seconds,train_tokens_per_second,prediction_file
|
| 2 |
-
muon,0,0,0,0,42733510,0.0,0.0,0.0,,,,6.
|
| 3 |
-
muon,261,
|
| 4 |
-
muon,522,
|
| 5 |
-
muon,783,
|
| 6 |
-
muon,1044,
|
| 7 |
-
muon,1305,
|
| 8 |
-
muon,1566,
|
| 9 |
-
muon,1827,
|
| 10 |
-
muon,2088,
|
| 11 |
-
muon,2349,
|
| 12 |
-
muon,2610,
|
|
|
|
| 1 |
optimizer,step,successful_updates,skipped_updates,train_tokens,training_budget_tokens,selected_dataset_fraction,full_dataset_token_fraction,tokens_per_parameter,train_loss,train_perplexity,cumulative_train_loss,validation_loss,validation_perplexity,validation_tokens,test_loss,test_perplexity,test_tokens,test_bleu,bleu_signature,bleu_samples,generated_tokens,learning_rates,last_gradient_norm,training_seconds,evaluation_seconds,total_evaluation_seconds,train_tokens_per_second,prediction_file
|
| 2 |
+
muon,0,0,0,0,42733510,0.0,0.0,0.0,,,,6.152294551393765,469.7941177503992,720136,6.153802047108677,470.50286445188505,708110,0.002089086296127387,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0, 0.0, 0.0]",,0.0,82.7089025840396,82.7089025840396,,predictions_00.jsonl
|
| 3 |
+
muon,261,261,0,4273351,42733510,0.1,0.025020983140117397,1.273721524692013,4.0481406036315155,57.290831582475136,4.0481406036315155,2.5621703693382223,12.963923305966714,720136,2.5636621853640325,12.98327752739135,708110,0.03083398708662475,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0009938625865312252, 9.938625865312251e-05, 9.938625865312251e-05]",0.3203253746032715,281.1368540887488,78.27012733393349,160.9790299179731,15200.25189814138,predictions_01.jsonl
|
| 4 |
+
muon,522,522,0,8546702,42733510,0.2,0.050041966280234794,2.547443049384026,2.5045045028839676,12.237493820398619,3.2763225532577414,2.463011343300085,11.740111869820579,720136,2.467185018978778,11.789213685445926,708110,0.05582133505440844,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0009457631880429201, 9.4576318804292e-05, 9.4576318804292e-05]",0.774559497833252,568.6849648010684,79.9007168749813,240.8797467929544,15028.886868830303,predictions_02.jsonl
|
| 5 |
+
muon,783,783,0,12820053,42733510,0.3,0.07506294942035219,3.821164574076039,2.430226130058437,11.36145095510898,2.9942904121913068,2.3827435894053344,10.834587783949875,720136,2.39104282934943,10.92488079106834,708110,0.05654392092884844,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0008547767072315835, 8.547767072315835e-05, 8.547767072315835e-05]",0.8050723671913147,840.1531296785688,72.94383920903783,313.8235860019922,15259.186149678188,predictions_03.jsonl
|
| 6 |
+
muon,1044,1044,0,17093404,42733510,0.4,0.10008393256046959,5.094886098768052,2.3242174275854817,10.218680098756618,2.8267721660398504,2.223032372531022,9.235293297912508,720136,2.2355781408321977,9.351886997309776,708110,0.100897461835075,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0007307629410938362, 7.307629410938362e-05, 7.307629410938362e-05]",0.6698967218399048,1115.7324636415578,77.46016829099972,391.28375429299194,15320.342964844893,predictions_04.jsonl
|
| 7 |
+
muon,1305,1305,0,21366755,42733510,0.5,0.125104915700587,6.368607623460065,2.1430165465797297,8.525115287851063,2.6900210421478263,2.0409670501908668,7.6980500012074335,720136,2.0537061002671284,7.7967431393926105,708110,0.12345122653106588,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0005871607054625496, 5.8716070546254966e-05, 5.8716070546254966e-05]",0.6055232286453247,1406.0170264714397,78.36569400003646,469.6494482930284,15196.654519627198,predictions_05.jsonl
|
| 8 |
+
muon,1566,1566,0,25640106,42733510,0.6,0.15012589884070437,7.642329148152078,1.9805903106765663,7.24701971584848,2.5717825869026165,1.9150039671325971,6.786965721738856,720136,1.9279956241901859,6.875714904050608,708110,0.12877604487102923,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00043953153078664056, 4.3953153078664054e-05, 4.3953153078664054e-05]",0.8913306593894958,1680.1966308491537,73.43593741592485,543.0853857089533,15260.18177232135,predictions_06.jsonl
|
| 9 |
+
muon,1827,1827,0,29913457,42733510,0.7,0.1751468819808218,8.916050672844092,1.884803411397829,6.585059767065248,2.473642704687647,1.8385603145838079,6.287479748988225,720136,1.8502426677341355,6.361363032867387,708110,0.1802229209222577,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00030387332884490803, 3.0387332884490805e-05, 3.0387332884490805e-05]",0.75834721326828,1950.8896681739716,73.07733799994458,616.1627237088978,15333.23872077242,predictions_07.jsonl
|
| 10 |
+
muon,2088,2088,0,34186808,42733510,0.8,0.20016786512093918,10.189772197536104,1.8216860253402747,6.182273136872596,2.3921481197692254,1.7927702604742368,6.006067813606906,720136,1.8042369539932885,6.075333920134396,708110,0.2571039476796964,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00019488677077162295, 1.9488677077162295e-05, 1.9488677077162295e-05]",1.9932366609573364,2219.8103655477753,77.00118562497664,693.1639093338745,15400.778611808957,predictions_08.jsonl
|
| 11 |
+
muon,2349,2349,0,38460159,42733510,0.9,0.22518884826105656,11.463493722228117,1.7873370917559692,5.9735243210402365,2.324946894434419,1.7655405758426186,5.844731024658187,720136,1.7764862606588754,5.909057012052186,708110,0.22627215440136425,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.00012438224123471442, 1.2438224123471442e-05, 1.2438224123471442e-05]",0.6744703054428101,2485.9783932112623,73.41921604203526,766.5831253759097,15470.833980306279,predictions_09.jsonl
|
| 12 |
+
muon,2610,2610,0,42733510,42733510,1.0,0.250209831401174,12.73721524692013,1.7692924620928165,5.86670097923902,2.2693814512002586,1.747523807568987,5.740370800119521,720136,1.7582855103506165,5.802480568528633,708110,0.24637446793816298,nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0,250,16000,"[0.0001, 1e-05, 1e-05]",0.7609756588935852,2754.439210804994,73.39668541704305,839.9798107929528,15514.41390769012,predictions_10.jsonl
|
muon/metrics.json
CHANGED
|
@@ -12,365 +12,376 @@
|
|
| 12 |
"train_loss": null,
|
| 13 |
"train_perplexity": null,
|
| 14 |
"cumulative_train_loss": null,
|
| 15 |
-
"validation_loss": 6.
|
| 16 |
-
"validation_perplexity": 469.
|
| 17 |
"validation_tokens": 720136,
|
| 18 |
-
"test_loss": 6.
|
| 19 |
-
"test_perplexity": 470.
|
| 20 |
"test_tokens": 708110,
|
| 21 |
-
"test_bleu": 0.
|
| 22 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 23 |
"bleu_samples": 250,
|
| 24 |
"generated_tokens": 16000,
|
| 25 |
"learning_rates": [
|
| 26 |
-
0.
|
| 27 |
-
0.
|
|
|
|
| 28 |
],
|
| 29 |
"last_gradient_norm": null,
|
| 30 |
"training_seconds": 0.0,
|
| 31 |
-
"evaluation_seconds":
|
| 32 |
-
"total_evaluation_seconds":
|
| 33 |
"train_tokens_per_second": null,
|
| 34 |
"prediction_file": "predictions_00.jsonl"
|
| 35 |
},
|
| 36 |
{
|
| 37 |
"optimizer": "muon",
|
| 38 |
"step": 261,
|
| 39 |
-
"successful_updates":
|
| 40 |
-
"skipped_updates":
|
| 41 |
"train_tokens": 4273351,
|
| 42 |
"training_budget_tokens": 42733510,
|
| 43 |
"selected_dataset_fraction": 0.1,
|
| 44 |
"full_dataset_token_fraction": 0.025020983140117397,
|
| 45 |
"tokens_per_parameter": 1.273721524692013,
|
| 46 |
-
"train_loss":
|
| 47 |
-
"train_perplexity":
|
| 48 |
-
"cumulative_train_loss":
|
| 49 |
-
"validation_loss": 2.
|
| 50 |
-
"validation_perplexity":
|
| 51 |
"validation_tokens": 720136,
|
| 52 |
-
"test_loss": 2.
|
| 53 |
-
"test_perplexity":
|
| 54 |
"test_tokens": 708110,
|
| 55 |
-
"test_bleu": 0.
|
| 56 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 57 |
"bleu_samples": 250,
|
| 58 |
"generated_tokens": 16000,
|
| 59 |
"learning_rates": [
|
| 60 |
-
0.
|
| 61 |
-
|
|
|
|
| 62 |
],
|
| 63 |
-
"last_gradient_norm":
|
| 64 |
-
"training_seconds":
|
| 65 |
-
"evaluation_seconds":
|
| 66 |
-
"total_evaluation_seconds":
|
| 67 |
-
"train_tokens_per_second":
|
| 68 |
"prediction_file": "predictions_01.jsonl"
|
| 69 |
},
|
| 70 |
{
|
| 71 |
"optimizer": "muon",
|
| 72 |
"step": 522,
|
| 73 |
-
"successful_updates":
|
| 74 |
-
"skipped_updates":
|
| 75 |
"train_tokens": 8546702,
|
| 76 |
"training_budget_tokens": 42733510,
|
| 77 |
"selected_dataset_fraction": 0.2,
|
| 78 |
"full_dataset_token_fraction": 0.050041966280234794,
|
| 79 |
"tokens_per_parameter": 2.547443049384026,
|
| 80 |
-
"train_loss": 2.
|
| 81 |
-
"train_perplexity":
|
| 82 |
-
"cumulative_train_loss":
|
| 83 |
-
"validation_loss": 2.
|
| 84 |
-
"validation_perplexity":
|
| 85 |
"validation_tokens": 720136,
|
| 86 |
-
"test_loss": 2.
|
| 87 |
-
"test_perplexity":
|
| 88 |
"test_tokens": 708110,
|
| 89 |
-
"test_bleu": 0.
|
| 90 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 91 |
"bleu_samples": 250,
|
| 92 |
"generated_tokens": 16000,
|
| 93 |
"learning_rates": [
|
| 94 |
-
0.
|
| 95 |
-
|
|
|
|
| 96 |
],
|
| 97 |
-
"last_gradient_norm":
|
| 98 |
-
"training_seconds":
|
| 99 |
-
"evaluation_seconds":
|
| 100 |
-
"total_evaluation_seconds":
|
| 101 |
-
"train_tokens_per_second":
|
| 102 |
"prediction_file": "predictions_02.jsonl"
|
| 103 |
},
|
| 104 |
{
|
| 105 |
"optimizer": "muon",
|
| 106 |
"step": 783,
|
| 107 |
-
"successful_updates":
|
| 108 |
-
"skipped_updates":
|
| 109 |
"train_tokens": 12820053,
|
| 110 |
"training_budget_tokens": 42733510,
|
| 111 |
"selected_dataset_fraction": 0.3,
|
| 112 |
"full_dataset_token_fraction": 0.07506294942035219,
|
| 113 |
"tokens_per_parameter": 3.821164574076039,
|
| 114 |
-
"train_loss":
|
| 115 |
-
"train_perplexity":
|
| 116 |
-
"cumulative_train_loss": 2.
|
| 117 |
-
"validation_loss":
|
| 118 |
-
"validation_perplexity":
|
| 119 |
"validation_tokens": 720136,
|
| 120 |
-
"test_loss":
|
| 121 |
-
"test_perplexity":
|
| 122 |
"test_tokens": 708110,
|
| 123 |
-
"test_bleu": 0.
|
| 124 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 125 |
"bleu_samples": 250,
|
| 126 |
"generated_tokens": 16000,
|
| 127 |
"learning_rates": [
|
| 128 |
-
0.
|
| 129 |
-
|
|
|
|
| 130 |
],
|
| 131 |
-
"last_gradient_norm": 0.
|
| 132 |
-
"training_seconds":
|
| 133 |
-
"evaluation_seconds":
|
| 134 |
-
"total_evaluation_seconds":
|
| 135 |
-
"train_tokens_per_second":
|
| 136 |
"prediction_file": "predictions_03.jsonl"
|
| 137 |
},
|
| 138 |
{
|
| 139 |
"optimizer": "muon",
|
| 140 |
"step": 1044,
|
| 141 |
-
"successful_updates":
|
| 142 |
-
"skipped_updates":
|
| 143 |
"train_tokens": 17093404,
|
| 144 |
"training_budget_tokens": 42733510,
|
| 145 |
"selected_dataset_fraction": 0.4,
|
| 146 |
"full_dataset_token_fraction": 0.10008393256046959,
|
| 147 |
"tokens_per_parameter": 5.094886098768052,
|
| 148 |
-
"train_loss":
|
| 149 |
-
"train_perplexity":
|
| 150 |
-
"cumulative_train_loss": 2.
|
| 151 |
-
"validation_loss":
|
| 152 |
-
"validation_perplexity":
|
| 153 |
"validation_tokens": 720136,
|
| 154 |
-
"test_loss":
|
| 155 |
-
"test_perplexity":
|
| 156 |
"test_tokens": 708110,
|
| 157 |
-
"test_bleu": 0.
|
| 158 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 159 |
"bleu_samples": 250,
|
| 160 |
"generated_tokens": 16000,
|
| 161 |
"learning_rates": [
|
| 162 |
-
0.
|
| 163 |
-
|
|
|
|
| 164 |
],
|
| 165 |
-
"last_gradient_norm":
|
| 166 |
-
"training_seconds":
|
| 167 |
-
"evaluation_seconds":
|
| 168 |
-
"total_evaluation_seconds":
|
| 169 |
-
"train_tokens_per_second":
|
| 170 |
"prediction_file": "predictions_04.jsonl"
|
| 171 |
},
|
| 172 |
{
|
| 173 |
"optimizer": "muon",
|
| 174 |
"step": 1305,
|
| 175 |
-
"successful_updates":
|
| 176 |
-
"skipped_updates":
|
| 177 |
"train_tokens": 21366755,
|
| 178 |
"training_budget_tokens": 42733510,
|
| 179 |
"selected_dataset_fraction": 0.5,
|
| 180 |
"full_dataset_token_fraction": 0.125104915700587,
|
| 181 |
"tokens_per_parameter": 6.368607623460065,
|
| 182 |
-
"train_loss":
|
| 183 |
-
"train_perplexity":
|
| 184 |
-
"cumulative_train_loss": 2.
|
| 185 |
-
"validation_loss":
|
| 186 |
-
"validation_perplexity":
|
| 187 |
"validation_tokens": 720136,
|
| 188 |
-
"test_loss":
|
| 189 |
-
"test_perplexity":
|
| 190 |
"test_tokens": 708110,
|
| 191 |
-
"test_bleu": 0.
|
| 192 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 193 |
"bleu_samples": 250,
|
| 194 |
"generated_tokens": 16000,
|
| 195 |
"learning_rates": [
|
| 196 |
-
0.
|
| 197 |
-
|
|
|
|
| 198 |
],
|
| 199 |
-
"last_gradient_norm": 0.
|
| 200 |
-
"training_seconds":
|
| 201 |
-
"evaluation_seconds":
|
| 202 |
-
"total_evaluation_seconds":
|
| 203 |
-
"train_tokens_per_second":
|
| 204 |
"prediction_file": "predictions_05.jsonl"
|
| 205 |
},
|
| 206 |
{
|
| 207 |
"optimizer": "muon",
|
| 208 |
"step": 1566,
|
| 209 |
-
"successful_updates":
|
| 210 |
-
"skipped_updates":
|
| 211 |
"train_tokens": 25640106,
|
| 212 |
"training_budget_tokens": 42733510,
|
| 213 |
"selected_dataset_fraction": 0.6,
|
| 214 |
"full_dataset_token_fraction": 0.15012589884070437,
|
| 215 |
"tokens_per_parameter": 7.642329148152078,
|
| 216 |
-
"train_loss":
|
| 217 |
-
"train_perplexity":
|
| 218 |
-
"cumulative_train_loss": 2.
|
| 219 |
-
"validation_loss":
|
| 220 |
-
"validation_perplexity":
|
| 221 |
"validation_tokens": 720136,
|
| 222 |
-
"test_loss":
|
| 223 |
-
"test_perplexity":
|
| 224 |
"test_tokens": 708110,
|
| 225 |
-
"test_bleu": 0.
|
| 226 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 227 |
"bleu_samples": 250,
|
| 228 |
"generated_tokens": 16000,
|
| 229 |
"learning_rates": [
|
| 230 |
-
0.
|
| 231 |
-
|
|
|
|
| 232 |
],
|
| 233 |
-
"last_gradient_norm": 0.
|
| 234 |
-
"training_seconds":
|
| 235 |
-
"evaluation_seconds":
|
| 236 |
-
"total_evaluation_seconds":
|
| 237 |
-
"train_tokens_per_second":
|
| 238 |
"prediction_file": "predictions_06.jsonl"
|
| 239 |
},
|
| 240 |
{
|
| 241 |
"optimizer": "muon",
|
| 242 |
"step": 1827,
|
| 243 |
-
"successful_updates":
|
| 244 |
-
"skipped_updates":
|
| 245 |
"train_tokens": 29913457,
|
| 246 |
"training_budget_tokens": 42733510,
|
| 247 |
"selected_dataset_fraction": 0.7,
|
| 248 |
"full_dataset_token_fraction": 0.1751468819808218,
|
| 249 |
"tokens_per_parameter": 8.916050672844092,
|
| 250 |
-
"train_loss":
|
| 251 |
-
"train_perplexity":
|
| 252 |
-
"cumulative_train_loss": 2.
|
| 253 |
-
"validation_loss":
|
| 254 |
-
"validation_perplexity":
|
| 255 |
"validation_tokens": 720136,
|
| 256 |
-
"test_loss":
|
| 257 |
-
"test_perplexity":
|
| 258 |
"test_tokens": 708110,
|
| 259 |
-
"test_bleu": 0.
|
| 260 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 261 |
"bleu_samples": 250,
|
| 262 |
"generated_tokens": 16000,
|
| 263 |
"learning_rates": [
|
| 264 |
-
0.
|
| 265 |
-
|
|
|
|
| 266 |
],
|
| 267 |
-
"last_gradient_norm": 0.
|
| 268 |
-
"training_seconds":
|
| 269 |
-
"evaluation_seconds":
|
| 270 |
-
"total_evaluation_seconds":
|
| 271 |
-
"train_tokens_per_second":
|
| 272 |
"prediction_file": "predictions_07.jsonl"
|
| 273 |
},
|
| 274 |
{
|
| 275 |
"optimizer": "muon",
|
| 276 |
"step": 2088,
|
| 277 |
-
"successful_updates":
|
| 278 |
-
"skipped_updates":
|
| 279 |
"train_tokens": 34186808,
|
| 280 |
"training_budget_tokens": 42733510,
|
| 281 |
"selected_dataset_fraction": 0.8,
|
| 282 |
"full_dataset_token_fraction": 0.20016786512093918,
|
| 283 |
"tokens_per_parameter": 10.189772197536104,
|
| 284 |
-
"train_loss":
|
| 285 |
-
"train_perplexity":
|
| 286 |
-
"cumulative_train_loss":
|
| 287 |
-
"validation_loss":
|
| 288 |
-
"validation_perplexity":
|
| 289 |
"validation_tokens": 720136,
|
| 290 |
-
"test_loss":
|
| 291 |
-
"test_perplexity":
|
| 292 |
"test_tokens": 708110,
|
| 293 |
-
"test_bleu": 0.
|
| 294 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 295 |
"bleu_samples": 250,
|
| 296 |
"generated_tokens": 16000,
|
| 297 |
"learning_rates": [
|
| 298 |
-
0.
|
| 299 |
-
|
|
|
|
| 300 |
],
|
| 301 |
-
"last_gradient_norm":
|
| 302 |
-
"training_seconds":
|
| 303 |
-
"evaluation_seconds":
|
| 304 |
-
"total_evaluation_seconds":
|
| 305 |
-
"train_tokens_per_second":
|
| 306 |
"prediction_file": "predictions_08.jsonl"
|
| 307 |
},
|
| 308 |
{
|
| 309 |
"optimizer": "muon",
|
| 310 |
"step": 2349,
|
| 311 |
-
"successful_updates":
|
| 312 |
-
"skipped_updates":
|
| 313 |
"train_tokens": 38460159,
|
| 314 |
"training_budget_tokens": 42733510,
|
| 315 |
"selected_dataset_fraction": 0.9,
|
| 316 |
"full_dataset_token_fraction": 0.22518884826105656,
|
| 317 |
"tokens_per_parameter": 11.463493722228117,
|
| 318 |
-
"train_loss":
|
| 319 |
-
"train_perplexity":
|
| 320 |
-
"cumulative_train_loss":
|
| 321 |
-
"validation_loss":
|
| 322 |
-
"validation_perplexity":
|
| 323 |
"validation_tokens": 720136,
|
| 324 |
-
"test_loss":
|
| 325 |
-
"test_perplexity":
|
| 326 |
"test_tokens": 708110,
|
| 327 |
-
"test_bleu": 0.
|
| 328 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 329 |
"bleu_samples": 250,
|
| 330 |
"generated_tokens": 16000,
|
| 331 |
"learning_rates": [
|
| 332 |
-
0.
|
| 333 |
-
|
|
|
|
| 334 |
],
|
| 335 |
-
"last_gradient_norm": 0.
|
| 336 |
-
"training_seconds":
|
| 337 |
-
"evaluation_seconds":
|
| 338 |
-
"total_evaluation_seconds":
|
| 339 |
-
"train_tokens_per_second":
|
| 340 |
"prediction_file": "predictions_09.jsonl"
|
| 341 |
},
|
| 342 |
{
|
| 343 |
"optimizer": "muon",
|
| 344 |
"step": 2610,
|
| 345 |
-
"successful_updates":
|
| 346 |
-
"skipped_updates":
|
| 347 |
"train_tokens": 42733510,
|
| 348 |
"training_budget_tokens": 42733510,
|
| 349 |
"selected_dataset_fraction": 1.0,
|
| 350 |
"full_dataset_token_fraction": 0.250209831401174,
|
| 351 |
"tokens_per_parameter": 12.73721524692013,
|
| 352 |
-
"train_loss":
|
| 353 |
-
"train_perplexity":
|
| 354 |
-
"cumulative_train_loss":
|
| 355 |
-
"validation_loss":
|
| 356 |
-
"validation_perplexity":
|
| 357 |
"validation_tokens": 720136,
|
| 358 |
-
"test_loss":
|
| 359 |
-
"test_perplexity":
|
| 360 |
"test_tokens": 708110,
|
| 361 |
-
"test_bleu": 0.
|
| 362 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 363 |
"bleu_samples": 250,
|
| 364 |
"generated_tokens": 16000,
|
| 365 |
"learning_rates": [
|
| 366 |
-
0.
|
| 367 |
-
|
|
|
|
| 368 |
],
|
| 369 |
-
"last_gradient_norm": 0.
|
| 370 |
-
"training_seconds":
|
| 371 |
-
"evaluation_seconds":
|
| 372 |
-
"total_evaluation_seconds":
|
| 373 |
-
"train_tokens_per_second":
|
| 374 |
"prediction_file": "predictions_10.jsonl"
|
| 375 |
}
|
| 376 |
]
|
|
|
|
| 12 |
"train_loss": null,
|
| 13 |
"train_perplexity": null,
|
| 14 |
"cumulative_train_loss": null,
|
| 15 |
+
"validation_loss": 6.152294551393765,
|
| 16 |
+
"validation_perplexity": 469.7941177503992,
|
| 17 |
"validation_tokens": 720136,
|
| 18 |
+
"test_loss": 6.153802047108677,
|
| 19 |
+
"test_perplexity": 470.50286445188505,
|
| 20 |
"test_tokens": 708110,
|
| 21 |
+
"test_bleu": 0.002089086296127387,
|
| 22 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 23 |
"bleu_samples": 250,
|
| 24 |
"generated_tokens": 16000,
|
| 25 |
"learning_rates": [
|
| 26 |
+
0.0,
|
| 27 |
+
0.0,
|
| 28 |
+
0.0
|
| 29 |
],
|
| 30 |
"last_gradient_norm": null,
|
| 31 |
"training_seconds": 0.0,
|
| 32 |
+
"evaluation_seconds": 82.7089025840396,
|
| 33 |
+
"total_evaluation_seconds": 82.7089025840396,
|
| 34 |
"train_tokens_per_second": null,
|
| 35 |
"prediction_file": "predictions_00.jsonl"
|
| 36 |
},
|
| 37 |
{
|
| 38 |
"optimizer": "muon",
|
| 39 |
"step": 261,
|
| 40 |
+
"successful_updates": 261,
|
| 41 |
+
"skipped_updates": 0,
|
| 42 |
"train_tokens": 4273351,
|
| 43 |
"training_budget_tokens": 42733510,
|
| 44 |
"selected_dataset_fraction": 0.1,
|
| 45 |
"full_dataset_token_fraction": 0.025020983140117397,
|
| 46 |
"tokens_per_parameter": 1.273721524692013,
|
| 47 |
+
"train_loss": 4.0481406036315155,
|
| 48 |
+
"train_perplexity": 57.290831582475136,
|
| 49 |
+
"cumulative_train_loss": 4.0481406036315155,
|
| 50 |
+
"validation_loss": 2.5621703693382223,
|
| 51 |
+
"validation_perplexity": 12.963923305966714,
|
| 52 |
"validation_tokens": 720136,
|
| 53 |
+
"test_loss": 2.5636621853640325,
|
| 54 |
+
"test_perplexity": 12.98327752739135,
|
| 55 |
"test_tokens": 708110,
|
| 56 |
+
"test_bleu": 0.03083398708662475,
|
| 57 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 58 |
"bleu_samples": 250,
|
| 59 |
"generated_tokens": 16000,
|
| 60 |
"learning_rates": [
|
| 61 |
+
0.0009938625865312252,
|
| 62 |
+
9.938625865312251e-05,
|
| 63 |
+
9.938625865312251e-05
|
| 64 |
],
|
| 65 |
+
"last_gradient_norm": 0.3203253746032715,
|
| 66 |
+
"training_seconds": 281.1368540887488,
|
| 67 |
+
"evaluation_seconds": 78.27012733393349,
|
| 68 |
+
"total_evaluation_seconds": 160.9790299179731,
|
| 69 |
+
"train_tokens_per_second": 15200.25189814138,
|
| 70 |
"prediction_file": "predictions_01.jsonl"
|
| 71 |
},
|
| 72 |
{
|
| 73 |
"optimizer": "muon",
|
| 74 |
"step": 522,
|
| 75 |
+
"successful_updates": 522,
|
| 76 |
+
"skipped_updates": 0,
|
| 77 |
"train_tokens": 8546702,
|
| 78 |
"training_budget_tokens": 42733510,
|
| 79 |
"selected_dataset_fraction": 0.2,
|
| 80 |
"full_dataset_token_fraction": 0.050041966280234794,
|
| 81 |
"tokens_per_parameter": 2.547443049384026,
|
| 82 |
+
"train_loss": 2.5045045028839676,
|
| 83 |
+
"train_perplexity": 12.237493820398619,
|
| 84 |
+
"cumulative_train_loss": 3.2763225532577414,
|
| 85 |
+
"validation_loss": 2.463011343300085,
|
| 86 |
+
"validation_perplexity": 11.740111869820579,
|
| 87 |
"validation_tokens": 720136,
|
| 88 |
+
"test_loss": 2.467185018978778,
|
| 89 |
+
"test_perplexity": 11.789213685445926,
|
| 90 |
"test_tokens": 708110,
|
| 91 |
+
"test_bleu": 0.05582133505440844,
|
| 92 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 93 |
"bleu_samples": 250,
|
| 94 |
"generated_tokens": 16000,
|
| 95 |
"learning_rates": [
|
| 96 |
+
0.0009457631880429201,
|
| 97 |
+
9.4576318804292e-05,
|
| 98 |
+
9.4576318804292e-05
|
| 99 |
],
|
| 100 |
+
"last_gradient_norm": 0.774559497833252,
|
| 101 |
+
"training_seconds": 568.6849648010684,
|
| 102 |
+
"evaluation_seconds": 79.9007168749813,
|
| 103 |
+
"total_evaluation_seconds": 240.8797467929544,
|
| 104 |
+
"train_tokens_per_second": 15028.886868830303,
|
| 105 |
"prediction_file": "predictions_02.jsonl"
|
| 106 |
},
|
| 107 |
{
|
| 108 |
"optimizer": "muon",
|
| 109 |
"step": 783,
|
| 110 |
+
"successful_updates": 783,
|
| 111 |
+
"skipped_updates": 0,
|
| 112 |
"train_tokens": 12820053,
|
| 113 |
"training_budget_tokens": 42733510,
|
| 114 |
"selected_dataset_fraction": 0.3,
|
| 115 |
"full_dataset_token_fraction": 0.07506294942035219,
|
| 116 |
"tokens_per_parameter": 3.821164574076039,
|
| 117 |
+
"train_loss": 2.430226130058437,
|
| 118 |
+
"train_perplexity": 11.36145095510898,
|
| 119 |
+
"cumulative_train_loss": 2.9942904121913068,
|
| 120 |
+
"validation_loss": 2.3827435894053344,
|
| 121 |
+
"validation_perplexity": 10.834587783949875,
|
| 122 |
"validation_tokens": 720136,
|
| 123 |
+
"test_loss": 2.39104282934943,
|
| 124 |
+
"test_perplexity": 10.92488079106834,
|
| 125 |
"test_tokens": 708110,
|
| 126 |
+
"test_bleu": 0.05654392092884844,
|
| 127 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 128 |
"bleu_samples": 250,
|
| 129 |
"generated_tokens": 16000,
|
| 130 |
"learning_rates": [
|
| 131 |
+
0.0008547767072315835,
|
| 132 |
+
8.547767072315835e-05,
|
| 133 |
+
8.547767072315835e-05
|
| 134 |
],
|
| 135 |
+
"last_gradient_norm": 0.8050723671913147,
|
| 136 |
+
"training_seconds": 840.1531296785688,
|
| 137 |
+
"evaluation_seconds": 72.94383920903783,
|
| 138 |
+
"total_evaluation_seconds": 313.8235860019922,
|
| 139 |
+
"train_tokens_per_second": 15259.186149678188,
|
| 140 |
"prediction_file": "predictions_03.jsonl"
|
| 141 |
},
|
| 142 |
{
|
| 143 |
"optimizer": "muon",
|
| 144 |
"step": 1044,
|
| 145 |
+
"successful_updates": 1044,
|
| 146 |
+
"skipped_updates": 0,
|
| 147 |
"train_tokens": 17093404,
|
| 148 |
"training_budget_tokens": 42733510,
|
| 149 |
"selected_dataset_fraction": 0.4,
|
| 150 |
"full_dataset_token_fraction": 0.10008393256046959,
|
| 151 |
"tokens_per_parameter": 5.094886098768052,
|
| 152 |
+
"train_loss": 2.3242174275854817,
|
| 153 |
+
"train_perplexity": 10.218680098756618,
|
| 154 |
+
"cumulative_train_loss": 2.8267721660398504,
|
| 155 |
+
"validation_loss": 2.223032372531022,
|
| 156 |
+
"validation_perplexity": 9.235293297912508,
|
| 157 |
"validation_tokens": 720136,
|
| 158 |
+
"test_loss": 2.2355781408321977,
|
| 159 |
+
"test_perplexity": 9.351886997309776,
|
| 160 |
"test_tokens": 708110,
|
| 161 |
+
"test_bleu": 0.100897461835075,
|
| 162 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 163 |
"bleu_samples": 250,
|
| 164 |
"generated_tokens": 16000,
|
| 165 |
"learning_rates": [
|
| 166 |
+
0.0007307629410938362,
|
| 167 |
+
7.307629410938362e-05,
|
| 168 |
+
7.307629410938362e-05
|
| 169 |
],
|
| 170 |
+
"last_gradient_norm": 0.6698967218399048,
|
| 171 |
+
"training_seconds": 1115.7324636415578,
|
| 172 |
+
"evaluation_seconds": 77.46016829099972,
|
| 173 |
+
"total_evaluation_seconds": 391.28375429299194,
|
| 174 |
+
"train_tokens_per_second": 15320.342964844893,
|
| 175 |
"prediction_file": "predictions_04.jsonl"
|
| 176 |
},
|
| 177 |
{
|
| 178 |
"optimizer": "muon",
|
| 179 |
"step": 1305,
|
| 180 |
+
"successful_updates": 1305,
|
| 181 |
+
"skipped_updates": 0,
|
| 182 |
"train_tokens": 21366755,
|
| 183 |
"training_budget_tokens": 42733510,
|
| 184 |
"selected_dataset_fraction": 0.5,
|
| 185 |
"full_dataset_token_fraction": 0.125104915700587,
|
| 186 |
"tokens_per_parameter": 6.368607623460065,
|
| 187 |
+
"train_loss": 2.1430165465797297,
|
| 188 |
+
"train_perplexity": 8.525115287851063,
|
| 189 |
+
"cumulative_train_loss": 2.6900210421478263,
|
| 190 |
+
"validation_loss": 2.0409670501908668,
|
| 191 |
+
"validation_perplexity": 7.6980500012074335,
|
| 192 |
"validation_tokens": 720136,
|
| 193 |
+
"test_loss": 2.0537061002671284,
|
| 194 |
+
"test_perplexity": 7.7967431393926105,
|
| 195 |
"test_tokens": 708110,
|
| 196 |
+
"test_bleu": 0.12345122653106588,
|
| 197 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 198 |
"bleu_samples": 250,
|
| 199 |
"generated_tokens": 16000,
|
| 200 |
"learning_rates": [
|
| 201 |
+
0.0005871607054625496,
|
| 202 |
+
5.8716070546254966e-05,
|
| 203 |
+
5.8716070546254966e-05
|
| 204 |
],
|
| 205 |
+
"last_gradient_norm": 0.6055232286453247,
|
| 206 |
+
"training_seconds": 1406.0170264714397,
|
| 207 |
+
"evaluation_seconds": 78.36569400003646,
|
| 208 |
+
"total_evaluation_seconds": 469.6494482930284,
|
| 209 |
+
"train_tokens_per_second": 15196.654519627198,
|
| 210 |
"prediction_file": "predictions_05.jsonl"
|
| 211 |
},
|
| 212 |
{
|
| 213 |
"optimizer": "muon",
|
| 214 |
"step": 1566,
|
| 215 |
+
"successful_updates": 1566,
|
| 216 |
+
"skipped_updates": 0,
|
| 217 |
"train_tokens": 25640106,
|
| 218 |
"training_budget_tokens": 42733510,
|
| 219 |
"selected_dataset_fraction": 0.6,
|
| 220 |
"full_dataset_token_fraction": 0.15012589884070437,
|
| 221 |
"tokens_per_parameter": 7.642329148152078,
|
| 222 |
+
"train_loss": 1.9805903106765663,
|
| 223 |
+
"train_perplexity": 7.24701971584848,
|
| 224 |
+
"cumulative_train_loss": 2.5717825869026165,
|
| 225 |
+
"validation_loss": 1.9150039671325971,
|
| 226 |
+
"validation_perplexity": 6.786965721738856,
|
| 227 |
"validation_tokens": 720136,
|
| 228 |
+
"test_loss": 1.9279956241901859,
|
| 229 |
+
"test_perplexity": 6.875714904050608,
|
| 230 |
"test_tokens": 708110,
|
| 231 |
+
"test_bleu": 0.12877604487102923,
|
| 232 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 233 |
"bleu_samples": 250,
|
| 234 |
"generated_tokens": 16000,
|
| 235 |
"learning_rates": [
|
| 236 |
+
0.00043953153078664056,
|
| 237 |
+
4.3953153078664054e-05,
|
| 238 |
+
4.3953153078664054e-05
|
| 239 |
],
|
| 240 |
+
"last_gradient_norm": 0.8913306593894958,
|
| 241 |
+
"training_seconds": 1680.1966308491537,
|
| 242 |
+
"evaluation_seconds": 73.43593741592485,
|
| 243 |
+
"total_evaluation_seconds": 543.0853857089533,
|
| 244 |
+
"train_tokens_per_second": 15260.18177232135,
|
| 245 |
"prediction_file": "predictions_06.jsonl"
|
| 246 |
},
|
| 247 |
{
|
| 248 |
"optimizer": "muon",
|
| 249 |
"step": 1827,
|
| 250 |
+
"successful_updates": 1827,
|
| 251 |
+
"skipped_updates": 0,
|
| 252 |
"train_tokens": 29913457,
|
| 253 |
"training_budget_tokens": 42733510,
|
| 254 |
"selected_dataset_fraction": 0.7,
|
| 255 |
"full_dataset_token_fraction": 0.1751468819808218,
|
| 256 |
"tokens_per_parameter": 8.916050672844092,
|
| 257 |
+
"train_loss": 1.884803411397829,
|
| 258 |
+
"train_perplexity": 6.585059767065248,
|
| 259 |
+
"cumulative_train_loss": 2.473642704687647,
|
| 260 |
+
"validation_loss": 1.8385603145838079,
|
| 261 |
+
"validation_perplexity": 6.287479748988225,
|
| 262 |
"validation_tokens": 720136,
|
| 263 |
+
"test_loss": 1.8502426677341355,
|
| 264 |
+
"test_perplexity": 6.361363032867387,
|
| 265 |
"test_tokens": 708110,
|
| 266 |
+
"test_bleu": 0.1802229209222577,
|
| 267 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 268 |
"bleu_samples": 250,
|
| 269 |
"generated_tokens": 16000,
|
| 270 |
"learning_rates": [
|
| 271 |
+
0.00030387332884490803,
|
| 272 |
+
3.0387332884490805e-05,
|
| 273 |
+
3.0387332884490805e-05
|
| 274 |
],
|
| 275 |
+
"last_gradient_norm": 0.75834721326828,
|
| 276 |
+
"training_seconds": 1950.8896681739716,
|
| 277 |
+
"evaluation_seconds": 73.07733799994458,
|
| 278 |
+
"total_evaluation_seconds": 616.1627237088978,
|
| 279 |
+
"train_tokens_per_second": 15333.23872077242,
|
| 280 |
"prediction_file": "predictions_07.jsonl"
|
| 281 |
},
|
| 282 |
{
|
| 283 |
"optimizer": "muon",
|
| 284 |
"step": 2088,
|
| 285 |
+
"successful_updates": 2088,
|
| 286 |
+
"skipped_updates": 0,
|
| 287 |
"train_tokens": 34186808,
|
| 288 |
"training_budget_tokens": 42733510,
|
| 289 |
"selected_dataset_fraction": 0.8,
|
| 290 |
"full_dataset_token_fraction": 0.20016786512093918,
|
| 291 |
"tokens_per_parameter": 10.189772197536104,
|
| 292 |
+
"train_loss": 1.8216860253402747,
|
| 293 |
+
"train_perplexity": 6.182273136872596,
|
| 294 |
+
"cumulative_train_loss": 2.3921481197692254,
|
| 295 |
+
"validation_loss": 1.7927702604742368,
|
| 296 |
+
"validation_perplexity": 6.006067813606906,
|
| 297 |
"validation_tokens": 720136,
|
| 298 |
+
"test_loss": 1.8042369539932885,
|
| 299 |
+
"test_perplexity": 6.075333920134396,
|
| 300 |
"test_tokens": 708110,
|
| 301 |
+
"test_bleu": 0.2571039476796964,
|
| 302 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 303 |
"bleu_samples": 250,
|
| 304 |
"generated_tokens": 16000,
|
| 305 |
"learning_rates": [
|
| 306 |
+
0.00019488677077162295,
|
| 307 |
+
1.9488677077162295e-05,
|
| 308 |
+
1.9488677077162295e-05
|
| 309 |
],
|
| 310 |
+
"last_gradient_norm": 1.9932366609573364,
|
| 311 |
+
"training_seconds": 2219.8103655477753,
|
| 312 |
+
"evaluation_seconds": 77.00118562497664,
|
| 313 |
+
"total_evaluation_seconds": 693.1639093338745,
|
| 314 |
+
"train_tokens_per_second": 15400.778611808957,
|
| 315 |
"prediction_file": "predictions_08.jsonl"
|
| 316 |
},
|
| 317 |
{
|
| 318 |
"optimizer": "muon",
|
| 319 |
"step": 2349,
|
| 320 |
+
"successful_updates": 2349,
|
| 321 |
+
"skipped_updates": 0,
|
| 322 |
"train_tokens": 38460159,
|
| 323 |
"training_budget_tokens": 42733510,
|
| 324 |
"selected_dataset_fraction": 0.9,
|
| 325 |
"full_dataset_token_fraction": 0.22518884826105656,
|
| 326 |
"tokens_per_parameter": 11.463493722228117,
|
| 327 |
+
"train_loss": 1.7873370917559692,
|
| 328 |
+
"train_perplexity": 5.9735243210402365,
|
| 329 |
+
"cumulative_train_loss": 2.324946894434419,
|
| 330 |
+
"validation_loss": 1.7655405758426186,
|
| 331 |
+
"validation_perplexity": 5.844731024658187,
|
| 332 |
"validation_tokens": 720136,
|
| 333 |
+
"test_loss": 1.7764862606588754,
|
| 334 |
+
"test_perplexity": 5.909057012052186,
|
| 335 |
"test_tokens": 708110,
|
| 336 |
+
"test_bleu": 0.22627215440136425,
|
| 337 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 338 |
"bleu_samples": 250,
|
| 339 |
"generated_tokens": 16000,
|
| 340 |
"learning_rates": [
|
| 341 |
+
0.00012438224123471442,
|
| 342 |
+
1.2438224123471442e-05,
|
| 343 |
+
1.2438224123471442e-05
|
| 344 |
],
|
| 345 |
+
"last_gradient_norm": 0.6744703054428101,
|
| 346 |
+
"training_seconds": 2485.9783932112623,
|
| 347 |
+
"evaluation_seconds": 73.41921604203526,
|
| 348 |
+
"total_evaluation_seconds": 766.5831253759097,
|
| 349 |
+
"train_tokens_per_second": 15470.833980306279,
|
| 350 |
"prediction_file": "predictions_09.jsonl"
|
| 351 |
},
|
| 352 |
{
|
| 353 |
"optimizer": "muon",
|
| 354 |
"step": 2610,
|
| 355 |
+
"successful_updates": 2610,
|
| 356 |
+
"skipped_updates": 0,
|
| 357 |
"train_tokens": 42733510,
|
| 358 |
"training_budget_tokens": 42733510,
|
| 359 |
"selected_dataset_fraction": 1.0,
|
| 360 |
"full_dataset_token_fraction": 0.250209831401174,
|
| 361 |
"tokens_per_parameter": 12.73721524692013,
|
| 362 |
+
"train_loss": 1.7692924620928165,
|
| 363 |
+
"train_perplexity": 5.86670097923902,
|
| 364 |
+
"cumulative_train_loss": 2.2693814512002586,
|
| 365 |
+
"validation_loss": 1.747523807568987,
|
| 366 |
+
"validation_perplexity": 5.740370800119521,
|
| 367 |
"validation_tokens": 720136,
|
| 368 |
+
"test_loss": 1.7582855103506165,
|
| 369 |
+
"test_perplexity": 5.802480568528633,
|
| 370 |
"test_tokens": 708110,
|
| 371 |
+
"test_bleu": 0.24637446793816298,
|
| 372 |
"bleu_signature": "nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0",
|
| 373 |
"bleu_samples": 250,
|
| 374 |
"generated_tokens": 16000,
|
| 375 |
"learning_rates": [
|
| 376 |
+
0.0001,
|
| 377 |
+
1e-05,
|
| 378 |
+
1e-05
|
| 379 |
],
|
| 380 |
+
"last_gradient_norm": 0.7609756588935852,
|
| 381 |
+
"training_seconds": 2754.439210804994,
|
| 382 |
+
"evaluation_seconds": 73.39668541704305,
|
| 383 |
+
"total_evaluation_seconds": 839.9798107929528,
|
| 384 |
+
"train_tokens_per_second": 15514.41390769012,
|
| 385 |
"prediction_file": "predictions_10.jsonl"
|
| 386 |
}
|
| 387 |
]
|
muon/muon.log
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
muon/predictions_00.jsonl
CHANGED
|
@@ -1,13 +1,13 @@
|
|
| 1 |
{"doc_id": "spok_1146@gpt-4o-mini-2024-07-18", "prompt": "Hello Pete, and thank you for having me on this journey today. I'm thrilled to share my story with you and the audience. I bega", "reference": "n my career in recruitment much like many, inspired by a vision ", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 2 |
{"doc_id": "blog_0819@gpt-4o-mini-2024-07-18", "prompt": "Why do they do this? It's as if the higher-ups are blind to the essence of what gatherings like this mean to people. The festiv", "reference": "al wasn't a den of iniquity; it was an oasis of community and cr", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 3 |
-
{"doc_id": "acad_1084@chunk_2", "prompt": "The functional relevance of these resting-state networks (RSNs) can be inferred from their spatial congruence with networks of ", "reference": "regions co-activated in association with specific cognitive and ", "prediction": "<extra_id_115><extra_id_115>\u001c\
|
| 4 |
{"doc_id": "spok_0062@Meta-Llama-3-70B", "prompt": "We talked about networking and what that means and what that looks like and what her experience has been with networking and th", "reference": "en also what her experience has been with finding jobs online an", "prediction": "Թ<extra_id_51>\rx4<extra_id_25>\u001c&\u0012<extra_id_68>\u0017S<extra_id_27>)<extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>i2<extra_id_56><extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 5 |
{"doc_id": "fic_1082@Meta-Llama-3-70B", "prompt": "The latter species attains greater height than the former, while the coastal variety is distinguished by its larger diameter, a", "reference": "lthough some specimens of both types exceed three hundred feet i", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 6 |
{"doc_id": "fic_0946@chunk_2", "prompt": "Linda paused and waited, wondering whether she might be of any help. \"Doctor!\" gasped the woman, frantically. \"My baby swallowe", "reference": "d a pin! And I'm sure it's in her lungs now. She breathes so que", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 7 |
{"doc_id": "acad_1085@chunk_1", "prompt": "Attention Deficit Hyperactivity Disorder (ADHD) is one of the most prevalent developmental disorders, affecting between 5 and 9", "reference": "% of school-aged children (Dittmann et al., 2009; Willcutt, 2012", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_27>)s<extra_id_33><extra_id_33><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 8 |
{"doc_id": "acad_1085@Meta-Llama-3-8B", "prompt": "Since the NA system plays an important role in regulating arousal levels, it has been proposed that ATX may improve attention b", "reference": "y increasing arousal (Del Campo et al., 2011; Posner and Peterse", "prediction": "<extra_id_64>\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_107>P\u0017\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
| 9 |
{"doc_id": "acad_0646@chunk_1", "prompt": "A plethora of studies in the economics/operational research literature is engaged with the measurement of producers' inefficien", "reference": "cy, thus relaxing the behavioral assumption that they are perfec", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 10 |
-
{"doc_id": "news_0186@chunk_1", "prompt": "The suspected gunman . arrested for shooting a newspaper photographer in Paris had lived in Britain for several years before re", "reference": "turning to France in July, the Paris prosecutor says. Abdelhakim", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`
|
| 11 |
{"doc_id": "tvm_0398@chunk_2", "prompt": "Hey Holden, now my compass has gone crazy. It's spinning like a son of a bitch! Mine too, we must have hit a magnetic field or ", "reference": "something !! Mine too, we must have hit a magnetic field or some", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 12 |
{"doc_id": "spok_1591@gpt-4o-2024-08-06", "prompt": "Every instinct screamed at me to move, to alert someone, to do anything but sit frozen in terror at the back of the bus. Yet, I", "reference": " was trapped in a spellbinding gaze with the creature-a gaze tha", "prediction": "<extra_id_16><extra_id_58>:Z\u001c\u0012\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 13 |
{"doc_id": "fic_0027@chunk_2", "prompt": "Charmed with the discovery he followed his host's lead, and, with a languid air, replied to his rival in monosyllables. The cap", "reference": "tain watched with quiet satisfaction, and at each rebuff his opi", "prediction": "D\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
|
@@ -15,7 +15,7 @@
|
|
| 15 |
{"doc_id": "tvm_0567@chunk_1", "prompt": "In order to show you how a big symphony orchestra is put together, Benjamin Britten has written a big piece of music, which is ", "reference": "made up of smaller pieces that show you all the separate parts o", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 16 |
{"doc_id": "blog_0329@chunk_1", "prompt": "Tonight, my summer roommate Alex and I bonded over asian cuisine, broken hearts and a buried friend. I always knew we had poten", "reference": "tial. We have an assignment in Computer-Assisted Reporting class", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 17 |
{"doc_id": "fic_1288@chunk_1", "prompt": "If you want views about the world's news, read what Cottswold Ommony calls the views papers; there is plenty in them that thoro", "reference": "ughly zealous people believe. But remember the wise old ambassad", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 18 |
-
{"doc_id": "acad_1041@Meta-Llama-3-8B", "prompt": "On the contrary, WMS sequencing provides much larger amount of data allowing for deeper insights into microbiome structure and ", "reference": "function. As the complete set of DNA molecules present in the sa", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*
|
| 19 |
{"doc_id": "news_0407@Meta-Llama-3-8B-Instruct", "prompt": "As the wedding days approached, Chelsea's anxiety about her hair grew. She had always been proud of her strawberry blonde locks", "reference": ", and the thought of losing them was daunting. But with the scal", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 20 |
{"doc_id": "blog_0959@gpt-4o-mini-2024-07-18", "prompt": "4) We must prioritize education and vocational training specifically designed for individuals like myself, those who are physic", "reference": "ally handicapped, ensuring that we are not left behind in the pr", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
| 21 |
{"doc_id": "acad_0444@gpt-4o-mini-2024-07-18", "prompt": "This variability in developmental timing is vital for the successful establishment of clonal lines, facilitating their adaptati", "reference": "on and survival in fluctuating environments. The plasticity inhe", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
|
@@ -31,7 +31,7 @@
|
|
| 31 |
{"doc_id": "spok_0508@chunk_1", "prompt": "Hello, everyone. Today is yodh 9 12 15. Yodh also known as August 9th 2019. The reason I honor the old calendar is because I be", "reference": "lieve this the calendar that more honors Yahweh the Gregorian ca", "prediction": "b<extra_id_64><extra_id_115>\u001c\u0012\u0001\b9ú<extra_id_113><extra_id_107>P<extra_id_110><extra_id_88>Q;w\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
| 32 |
{"doc_id": "acad_0567@gpt-4o-2024-08-06", "prompt": "The establishment of such procedures can be fundamentally linked to a deeper understanding of the decision-making context and t", "reference": "he intrinsic characteristics that govern it. A methodical approa", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_63>)XI<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 33 |
{"doc_id": "tvm_0881@Meta-Llama-3-70B", "prompt": "In the Pentagon. You've got the wrong man. You're my only hope. The admiral laughs. You're a great friend, James Greer. What's ", "reference": "this all about? Did you bring the book? Yes. Good. We've been wo", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 34 |
-
{"doc_id": "spok_1694@Meta-Llama-3-70B-Instruct", "prompt": "Yeah, sorry about that, folks. My bad. So, as I was saying, with the internet, fame has become more accessible. I mean, you can", "reference": " be famous for just about anything these days. You can be a soci", "prediction": "<extra_id_115><extra_id_115>\u001c\
|
| 35 |
{"doc_id": "blog_1389@Meta-Llama-3-8B-Instruct", "prompt": "Here is the next 500 words in the same style and tone as the original text: I got off the train at Central Station, and walked ", "reference": "to the food court to grab some lunch. I was thinking about that ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 36 |
{"doc_id": "news_0673@gpt-4o-mini-2024-07-18", "prompt": "Many students find themselves navigating the sometimes choppy waters of internship life, where the balance between goals and as", "reference": "signments can feel precarious at times. However, embracing the s", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 37 |
{"doc_id": "spok_1589@chunk_1", "prompt": "All right, welcome to the show if we get started here. I just want to talk a little bit about anchor dot. F m -- and if you hav", "reference": "en't heard about anchor dot f m -- it's literally the easiest wa", "prediction": "C<extra_id_115>\u001c\rx\rx<extra_id_113><extra_id_4><extra_id_88>\u0019\u0000<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>i2<extra_id_56><extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
|
@@ -40,7 +40,7 @@
|
|
| 40 |
{"doc_id": "acad_0258@chunk_2", "prompt": "For example, the production of packaging intended to come in contact with food and drink (known as food contact materials, FCMs", "reference": ") needs to comply with the EU food contact legislation (Regulati", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 41 |
{"doc_id": "acad_0702@Meta-Llama-3-8B", "prompt": "While some scholars have highlighted the importance of communication (e.g., Bessant and Rush, 1995; Zahra and Covin, 1995), oth", "reference": "ers have questioned its relevance (e.g., Miller and Friesen, 198", "prediction": "Թ<extra_id_51>\rx\u001c\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_63>)XI<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 42 |
{"doc_id": "tvm_0398@Meta-Llama-3-8B-Instruct", "prompt": "Here is the continuation of the text: Holden's voice was laced with panic as he frantically scanned the horizon. \"What the hell", "reference": " is going on, Skipper? We're lost!\" Stepek's gruff tone only add", "prediction": "<extra_id_16><extra_id_58>:Zﹷ<extra_id_51>\rx4<extra_id_25>\u001c\b<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017i2<extra_id_56>(<extra_id_115>\u001c&<extra_id_64>"}
|
| 43 |
-
{"doc_id": "blog_0081@chunk_2", "prompt": "The views on the road weren't too picturesque, either. We should've taken the Gatwick Express , which costs pretty much the sam", "reference": "e; the saddest part is that the hotel is just a 2-minute walk fr", "prediction": "\u0012\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><
|
| 44 |
{"doc_id": "acad_0820@Meta-Llama-3-8B", "prompt": "Through this new representation, Muslim women have gained greater access to a wider range of resources, and the potential to es", "reference": "tablish their own businesses (Bakr, 2016; El-Ghobashy, 2011; She", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 45 |
{"doc_id": "blog_1364@gpt-4o-2024-08-06", "prompt": "It reverberated through the air, words echoing off the walls as if carried by the wind itself: the force, the fervor, the raw p", "reference": "ower of his conviction. Sahir stood there, his presence filling ", "prediction": "\u001c\u0012\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 46 |
{"doc_id": "spok_1587@Meta-Llama-3-8B", "prompt": "Okay, well, I think we should just take a step back. You don't have to worry about me. I'm going to be fine. I'm not worried ab", "reference": "out you. I'm worried about the podcast. If you don't mind. I'm g", "prediction": "<extra_id_64><extra_id_115>\u001c\u0001zlú<extra_id_113><extra_id_115>)<extra_id_122>s<extra_id_122>s<extra_id_122>s<extra_id_4><extra_id_110>i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
|
@@ -50,11 +50,11 @@
|
|
| 50 |
{"doc_id": "fic_0923@Meta-Llama-3-8B-Instruct", "prompt": "As the young man continued to walk down Arcadia Street, he couldn't shake off the feeling of being out of place. The grimy faca", "reference": "des of the houses seemed to loom over him, their peeling paint a", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 51 |
{"doc_id": "spok_0047@chunk_2", "prompt": "My parents are Mortal Kombat because I was like I came was like this shit and then afterwards I know my parents bake afterward ", "reference": "the newscaster like, you know, if you let your kids play Mortal ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 52 |
{"doc_id": "acad_0077@Meta-Llama-3-70B-Instruct", "prompt": "The impact of human activities on the Earth's surface and subsurface is evident in the widespread occurrence of landslides, sub", "reference": "sidence, and sinkholes, often triggered or exacerbated by anthro", "prediction": "<extra_id_64>\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 53 |
-
{"doc_id": "news_0727@chunk_2", "prompt": "I am already in Donetsk. A - Is it what I am thinking about? the one called 'M'? B - yes . A - 'PM'? B - yes, yes, yes. BUK. A ", "reference": "- oh, BM, yes, yes, yes. B - BUK . A - I understand . B - BUK, B", "prediction": "<extra_id_115><extra_id_115>\u001c\
|
| 54 |
-
{"doc_id": "spok_0496@Meta-Llama-3-70B-Instruct", "prompt": "...but it's likely that he had an inkling, given the rising tensions between Italy and Ethiopia. In October 1935, Italy invaded", "reference": " Ethiopia, and Amedeo's unit was deployed to the front lines. Th", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_94><extra_id_77><extra_id_110><extra_id_88>Q<extra_id_54>\u0017<extra_id_88>\
|
| 55 |
{"doc_id": "blog_0710@Meta-Llama-3-70B", "prompt": "So, I figured that if I ate just a few bites, then threw the rest away, Jack wouldn't know any better. I started off slowly wit", "reference": "h some beans (which weren't bad) and then moved on to the pork. ", "prediction": "\u0019\u0000\u001aﹷ<extra_id_51>\rx4<extra_id_25>\u001c\b<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 56 |
-
{"doc_id": "fic_0704@Meta-Llama-3-8B-Instruct", "prompt": "As the steamboat, or rather, the raft, drew near to the shore, Freddie Bobbsey called out to his sister, \"All aboard, Flossie! ", "reference": "Get on, get on!\" Flossie, with her blue eyes shining with excite", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P<extra_id_114>\u0017i2<extra_id_56>(<extra_id_115>
|
| 57 |
-
{"doc_id": "blog_0945@chunk_2", "prompt": "I thought they were cute but really thought nothing of it. They then kept looking our way, mind you Brad stopped headbanging an", "reference": "d was sitting on the bench with Phil and no one was in front of ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><
|
| 58 |
{"doc_id": "fic_0923@chunk_2", "prompt": "Still appearing utterly oblivious of the questions showered upon him by the now frantic man who had constituted himself as guid", "reference": "e, the young man had got midway up the street, and was still sea", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 59 |
{"doc_id": "spok_1705@chunk_2", "prompt": "So if you can picture eight feet tall like the basket that you used to be able to dunk on when you're in like 8th grade because", "reference": " you couldn't get 10 that's almost how long his arms are which i", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 60 |
{"doc_id": "acad_0521@chunk_1", "prompt": "A wide body of literature in archaeology and anthropology discusses the concept of craft specialisation. This topic is importan", "reference": "t because it contributes to understanding the development of soc", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
|
@@ -104,7 +104,7 @@
|
|
| 104 |
{"doc_id": "acad_0483@Meta-Llama-3-8B", "prompt": "A survey of 5000 UK residents found that perceptions of visual amenity were more negative among those who had previously oppose", "reference": "d onshore wind farms, even if they lived further away from exist", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 105 |
{"doc_id": "spok_1550@chunk_1", "prompt": "Wrong, so wrong by c-pillar 42 read by litter Aryan. summary There is a kiss and it is so wrong. It turns out that even after k", "reference": "nowing each other for about six thousand years. It is possible t", "prediction": "U<extra_id_29><extra_id_120><extra_id_115>\u001c\u0012\u0001\b9ú<extra_id_113><extra_id_16><extra_id_58>:<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0i(<extra_id_119>C<extra_id_107>P*\rx<extra_id_63>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
|
| 106 |
{"doc_id": "news_0483@Meta-Llama-3-8B", "prompt": "'We are working hard to minimise any disruption to our customers and will contact them directly if their flight has been affect", "reference": "ed.' BA added that it was continuing to monitor the situation cl", "prediction": "\u0019\u0000\u001aU\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
|
| 107 |
-
{"doc_id": "blog_0065@gpt-4o-mini-2024-07-18", "prompt": "Not for You - A powerful track from Vitalogy, it echoes the frustrations of feeling trapped within the confines of expectation.", "reference": " Eddie's emotive delivery resonated deeply, especially during th", "prediction": "<extra_id_119><extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><
|
| 108 |
{"doc_id": "spok_0558@chunk_1", "prompt": "Hi everyone, and welcome to this incessant home tomorrow. I am like internally screaming right now. This is a podcast about thi", "reference": "ngs that just make isn't comfortable and that we don't want to t", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 109 |
{"doc_id": "spok_1215@chunk_1", "prompt": "Welcome to the good Christian talks podcast. I'm Levi and I'm Chris. Thank you so much for joining us this week on this podcast", "reference": ". We select one taco week to help each one of us get the Bible i", "prediction": "\u0019\u0000\u001aﹷ<extra_id_51>\rx4<extra_id_25>\u001c\b<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>i2<extra_id_56>(<extra_id_115>)I<extra_id_16><extra_id_58><extra_id_25><extra_id_64>"}
|
| 110 |
{"doc_id": "acad_0525@gpt-4o-mini-2024-07-18", "prompt": "This observation suggests a multifaceted role for p53 in modulating not only the cellular response to DNA damage but also the m", "reference": "etabolism of environmental carcinogens, thereby influencing susc", "prediction": "\u0012\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
|
@@ -115,15 +115,15 @@
|
|
| 115 |
{"doc_id": "blog_0358@gpt-4o-2024-08-06", "prompt": "67-year-old S Puhaindran was a young teacher in Raffles Institution when Mr Goh was a student there. \"I remember him well,\" Mr.", "reference": " Puhaindran recounted with an affectionate smile. \"Goh was the k", "prediction": "<extra_id_119><extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_63>)XI<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 116 |
{"doc_id": "spok_0718@chunk_2", "prompt": "Welcome to our second of two episodes on John T Scopes last week We examined the rise of the anti-evolution movement in the ear", "reference": "ly 1920s that led to the establishment of the butler act which b", "prediction": "<extra_id_25><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 117 |
{"doc_id": "spok_1582@chunk_1", "prompt": "Hello everyone and welcome to the Notre grow podcast. I'm your host. Dr. Ivan Khan, and we'll be breaking down topics around ed", "reference": "ucation growth and culture with the intention to help your growt", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 118 |
-
{"doc_id": "blog_1429@Meta-Llama-3-70B-Instruct", "prompt": "As I sat in the staff meeting, listening to our administrators outline the plan for the year, I couldn't help but feel a sense ", "reference": "of excitement and trepidation. The idea of \"writing it down and ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001z<extra_id_111>\u001c\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017
|
| 119 |
{"doc_id": "tvm_0327@chunk_2", "prompt": "Dead serious. See... the world is made up of rules. Kids don't like rules. They rebel for a while, but ultimately they start to", "reference": " fall in line and become ordinary Joe's like you and me. But som", "prediction": "C<extra_id_115>\u001c\u0012\u0001\b9ú<extra_id_113><extra_id_107>P<extra_id_110><extra_id_88>Q;w\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107>P*\rx<extra_id_63>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
|
| 120 |
{"doc_id": "tvm_0857@gpt-4o-mini-2024-07-18", "prompt": "With silent determination, MALONE casts an assured glance at the OARSMEN, who hesitate for a moment, caught between the ardor o", "reference": "f loyalty and the dread of what lies ahead. One by one, they asc", "prediction": "C\u001c&<extra_id_121>\u0001\b9ú<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 121 |
{"doc_id": "blog_0293@chunk_1", "prompt": "Michael posted more about abortion and guns on his website and, as a woman, I violentely reacted to some of his comments on abo", "reference": "rtion. My reaction was too long to post on his blog, so I'm doin", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 122 |
-
{"doc_id": "news_0530@Meta-Llama-3-8B-Instruct", "prompt": "As the controversy surrounding the botched trial of John Downey continued to simmer, the UK government's refusal to launch a ju", "reference": "dicial inquiry into the matter only added fuel to the fire. Nort", "prediction": "x<extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*
|
| 123 |
{"doc_id": "spok_0772@Meta-Llama-3-8B-Instruct", "prompt": "Here's the next 500 words in the same style and tone: David: Yeah, okay, let's move on to the next question. At wonderland unde", "reference": "rscore underscore, what's the weirdest thing that's happened to ", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 124 |
{"doc_id": "news_0390@chunk_2", "prompt": "For example, Benson said that $1 million in $100 bills weighs 22 pounds, while $1 million in 500 euro notes weighs 3.5 pounds. ", "reference": "\"It's a huge difference,\" he said. Though Europe is highly attra", "prediction": "<extra_id_115><extra_id_115>\u001c\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 125 |
{"doc_id": "spok_0704@gpt-4o-2024-08-06", "prompt": "You see, creativity intertwines itself with the rituals that one builds around it. For me, it's not so much about adhering to a", "reference": " strict schedule, as it is about weaving those moments of intros", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 126 |
-
{"doc_id": "news_0483@Meta-Llama-3-8B-Instruct", "prompt": "As the storm continues to rage on the East Coast, the travel chaos shows no signs of abating. Thousands of Britons are facing a", "reference": "nother day of uncertainty, with airlines scrambling to rebook fl", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><
|
| 127 |
{"doc_id": "fic_1229@chunk_2", "prompt": "Phillips, his valet, had heard the shot, and knowing that the prince's cousin, Prince Miguel, with Don Solado, prime minister o", "reference": "f Joyalita, had made attempts on his life before, in New York, h", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0\u0014\u0017i2<extra_id_56>(<extra_id_115><extra_id_64>\u0017\u000b"}
|
| 128 |
{"doc_id": "fic_0555@gpt-4o-2024-08-06", "prompt": "The memories of carefree childhood days woven amidst the tranquil charm of the estate danced in his mind, anchoring him to this", "reference": " beloved place with an unyielding fondness. The familiar sound o", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 129 |
{"doc_id": "blog_1347@Meta-Llama-3-70B-Instruct", "prompt": "As I gazed deeper into the mirror, my eyes seemed to bore into my own soul. The reflection staring back at me was a stranger, y", "reference": "et eerily familiar. The lines on my face told a story of pain an", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\b9ú<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
|
@@ -135,7 +135,7 @@
|
|
| 135 |
{"doc_id": "news_0135@gpt-4o-mini-2024-07-18", "prompt": "However, the events of last October reshaped those timelines and priorities, pushing Virgin Galactic into a more insular approa", "reference": "ch to the development of their space ambitions. The firm recogni", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 136 |
{"doc_id": "acad_0489@Meta-Llama-3-8B", "prompt": "Individual hackers may hack into websites or online services to deface content, change information, disrupt service, or simply ", "reference": "out of curiosity.16 Lastly, activist groups may target critical ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 137 |
{"doc_id": "news_0246@Meta-Llama-3-70B", "prompt": "That might seem an unlikely trio of activities for a man preparing for a world title fight but there were two serious reasons f", "reference": "or Manny's trip here. One was to promote his forthcoming book, a", "prediction": "<extra_id_16><extra_id_58>:Z\u001c\u0012\u0001\b\u0017\u0017*<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 138 |
-
{"doc_id": "acad_0497@gpt-4o-mini-2024-07-18", "prompt": "Moreover, the limitations inherent within the traditional diagnostic framework underscore a pressing need for an integrative, t", "reference": "ransdiagnostic approach to emotional disorders that recognizes a", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><
|
| 139 |
{"doc_id": "spok_1425@chunk_1", "prompt": "Even when you say listen, we might not be getting billboard this time around because you're a brand new artist. They don't nece", "reference": "ssarily hear that. So first off I just want to say welcome to th", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 140 |
{"doc_id": "blog_0578@Meta-Llama-3-8B-Instruct", "prompt": "As I reflect on my childhood friendships, I'm reminded of the many memories I've accumulated over the years. One of my earliest", "reference": " and most cherished friendships was with Rachel. We met in kinde", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 141 |
{"doc_id": "spok_1607@Meta-Llama-3-70B", "prompt": "We have and if you haven't had that experience yet, then I would definitely suggest you start paying attention to how you feel ", "reference": "and learning to be in tune with yourself and your intuition. So ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
|
@@ -161,7 +161,7 @@
|
|
| 161 |
{"doc_id": "tvm_0962@Meta-Llama-3-70B-Instruct", "prompt": "Walter's eyes dart around the elevator, searching for an escape from the uncomfortable silence. Ted's gaze lingers, as if darin", "reference": "g Walter to ask again. Finally, he responds, his tone measured, ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_94><extra_id_77><extra_id_110>\u0003<extra_id_16><extra_id_58>㥦\u0017\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
| 162 |
{"doc_id": "acad_0646@gpt-4o-2024-08-06", "prompt": "These studies employed Geographic Information Systems (GIS) to accurately delineate spatial relationships and pinpoint geograph", "reference": "ical proximities between producers, allowing for a nuanced analy", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 163 |
{"doc_id": "acad_0772@chunk_2", "prompt": "However, the pressure signal of the catheter is affected by the viscosity of the colonic fluids [28]. Arkwright et al. [28] sho", "reference": "wed that the predetermined amplitude of the applied pressure on ", "prediction": "C\u001c&<extra_id_121>\u0001\b\u0017\u0017*<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 164 |
-
{"doc_id": "news_0656@gpt-4o-mini-2024-07-18", "prompt": "In facing this precarious situation, the possibility of requiring a financial rescue looms large on Italy's horizon, though man", "reference": "y experts argue it lacks the structural and political framework ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>
|
| 165 |
{"doc_id": "spok_0704@Meta-Llama-3-8B", "prompt": "That's really cool. I think that's great. I mean, I'm not judging anyone or anything, but it's interesting because I remember r", "reference": "eading an article by Tony Robbins and he said that he stopped dr", "prediction": "<extra_id_25><extra_id_115>\u001cﹷ\u0017\u0017\u0017\u0017*<extra_id_16><extra_id_58>:\u0017S<extra_id_27>)<extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 166 |
{"doc_id": "news_1257@gpt-4o-2024-08-06", "prompt": "The goal is to prevent Iran from refining uranium beyond a certain threshold, drastically reducing its capacity to progress tow", "reference": "ard a nuclear weapon in the near term. In exchange, Iran will re", "prediction": "\u0017\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
|
| 167 |
{"doc_id": "spok_1443@Meta-Llama-3-8B", "prompt": "They said they couldn't stand the smell of your dead cat anymore. So they took off. You've been living alone since 1999 and no ", "reference": "one has heard from them again. I guess they just went back home ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zlú<extra_id_113><extra_id_115>)<extra_id_122>s<extra_id_122>s<extra_id_122>s<extra_id_33><extra_id_33><extra_id_33><extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
|
@@ -179,7 +179,7 @@
|
|
| 179 |
{"doc_id": "blog_0126@chunk_2", "prompt": "What took you so long? God help you become victorious. I want to say hello to Bush, to shake his hand. We came out of the grave", "reference": ".'\" The Guadian, 3-22-03 \"As hundreds of coalition troops swept ", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P<extra_id_114>\u0017i2<extra_id_56>(<extra_id_115>\u001c&9(<extra_id_115>"}
|
| 180 |
{"doc_id": "fic_0051@chunk_2", "prompt": "Two seamen, each pushing an oar before him, swam out to the nearest boat, and having got on board of her, and freed her from he", "reference": "r entanglements, they rowed out to the capsized boat, and towed ", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 181 |
{"doc_id": "tvm_0383@chunk_1", "prompt": "How do you do. My name is Penelope Ryan. This is a simple-minded play about men who enjoy killing -- and those who don't. I am ", "reference": "Harold Ryan, her husband. I have killed perhaps two hundred men ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
| 182 |
-
{"doc_id": "acad_0983@Meta-Llama-3-8B", "prompt": "Understanding how much water is available for plants in a given ecosystem is critical for managing natural resources and assess", "reference": "ing the impact of land-use/management practices on water availab", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<
|
| 183 |
{"doc_id": "fic_0034@Meta-Llama-3-70B", "prompt": "The ears were short and delicately made; the eyes were well-placed; and the distance to the angle of the jaw was such as good b", "reference": "reeders desire. Considering him closer she became aware of the l", "prediction": "<extra_id_64>\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 184 |
{"doc_id": "blog_0069@chunk_1", "prompt": "..Is the one, which tells you immediately, what the outcome will be: Good food, bad husband, horror able dinner, acing a test, ", "reference": "bad sex, insanely demented neighbours... the last thing is an is", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
|
| 185 |
{"doc_id": "blog_0747@gpt-4o-2024-08-06", "prompt": "I finally reached Plaza Singapura, and there was Darius, waiting patiently with a grin. The moment I saw him, I felt a little f", "reference": "lutter in my stomach. It's funny how a net friend can feel so fa", "prediction": "<extra_id_16><extra_id_58>:Z\u001c\u0012\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
|
@@ -199,12 +199,12 @@
|
|
| 199 |
{"doc_id": "acad_0807@gpt-4o-2024-08-06", "prompt": "As a result, the development of more adaptive operational strategies that can detect and respond to ice formation is paramount.", "reference": " Integrating advanced sensor technology can offer a real-time as", "prediction": "<extra_id_119><extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 200 |
{"doc_id": "fic_0975@Meta-Llama-3-70B", "prompt": "\"Well, what does the chief say, sir?\" he asked. \"He doesn't seem very cheerful over it.\" \"He isn't,\" replied Ramshaw grimly. \"S", "reference": "ome scoundrel has tampered with the high-pressure slide valves. ", "prediction": "<extra_id_32>÷\u0017\u0017\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 201 |
{"doc_id": "spok_1483@Meta-Llama-3-8B-Instruct", "prompt": "But we're not going to be bored, because we're going to dive into the story of the 1974 Zaire team and explore the complexities", "reference": " of colonialism and its lasting impact on the African continent.", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 202 |
-
{"doc_id": "fic_0841@chunk_1", "prompt": "One evening Mr. Courtnay Fox, the London correspondent of the Edinburgh Chronicle, was as usual in his own room in the office i", "reference": "n Fleet-street, when a card was brought to him. \"Show the gentle", "prediction": "\u001c\u0012\
|
| 203 |
{"doc_id": "news_0227@chunk_2", "prompt": "We've been writers in our own right for several years before we started to write together, and from the beginning we thought it", "reference": " was a great idea to stay secret behind a pen name. Then two thi", "prediction": "\u0019\u0000\u001aU\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_94><extra_id_77><extra_id_110>\u0003<extra_id_16><extra_id_58>Һ<extra_id_113>\u0017i2<extra_id_56>(<extra_id_115>\u001c&9(<extra_id_115>"}
|
| 204 |
{"doc_id": "tvm_0327@Meta-Llama-3-70B", "prompt": "It's the humane thing to do. We don't want them to suffer. You know how hard it is to find a decent lawyer these days? This is ", "reference": "the place? Yeah, I think so. Come on, Lizzy. Let's check it out.", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zlú<extra_id_113><extra_id_115>)<extra_id_122>s<extra_id_122>\u001fZ\u0019\u0000^<extra_id_101>0i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 205 |
{"doc_id": "blog_0026@chunk_1", "prompt": "On Monday morning at work, I slipped on a wet floor. Mondays are always heavy load days. Mondays after a holiday... ew. Twelve ", "reference": "dogs to do. And they wouldn't let me wash some of them a day ear", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 206 |
{"doc_id": "news_0187@gpt-4o-2024-08-06", "prompt": "Jimmy was grateful for Claire's presence and quickly found that her vivacious spirit brightened the household. Even Gemma, who ", "reference": "had her reservations at first about her father's burgeoning rela", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 207 |
-
{"doc_id": "fic_1122@Meta-Llama-3-8B-Instruct", "prompt": "As the days passed, the anticipation grew, and the village was abuzz with excitement. The two suitors, myself and my rival, wer", "reference": "e summoned to the farmer's estate, where we were to present ours", "prediction": "<extra_id_25><extra_id_115>\u001c\
|
| 208 |
{"doc_id": "news_0530@gpt-4o-mini-2024-07-18", "prompt": "In a shocking escalation of political tension, Mr. Robinson's outrage reverberated throughout Northern Ireland's tumultuous lan", "reference": "dscape, reigniting old wounds and fears among the populace. The ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 209 |
{"doc_id": "news_0251@chunk_1", "prompt": "Experts were unable to open the 9th century Carolingian pot (pictured), which was found on church land in Dumfries and Galloway", "reference": " in September . The mystery surrounding the contents of a Viking", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 210 |
{"doc_id": "spok_1103@chunk_2", "prompt": "It was a questionable mix at that point in history. One member of the crowd tried to storm the stage, but he was quickly subdue", "reference": "d by security after he managed to break one of the presentation ", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
|
@@ -222,7 +222,7 @@
|
|
| 222 |
{"doc_id": "tvm_0420@gpt-4o-2024-08-06", "prompt": "We simply critique the cult that surrounds them. Sirens wail in the distance, and the thrum of anticipation grows. Young is sti", "reference": "ll spinning his tale, unaware or uncaring of the chaos, maintain", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
|
| 223 |
{"doc_id": "news_1166@gpt-4o-2024-08-06", "prompt": "But rather than dismissing this sentiment as merely the nostalgic ramblings of a bygone generation, it might serve us well to d", "reference": "elve deeper into the root of such concerns. The discourse surrou", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_107>P<extra_id_110><extra_id_88>Q;w\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107>P*\rx<extra_id_63>\u001f^<extra_id_101><extra_id_93>(<extra_id_115>\u001c&<extra_id_64>"}
|
| 224 |
{"doc_id": "spok_1150@chunk_2", "prompt": "And they have a new proprietary blend which is their metabolic ferment, which I can't explain to you the science behind it, but", "reference": " I will tell you that it's supposed to help boost your natural c", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_16><extra_id_58>:<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 225 |
-
{"doc_id": "blog_0026@gpt-4o-2024-08-06", "prompt": "I slipped, my feet skidding out from under me like an awkward rendition of a cartoon character slipping on a banana peel. As I ", "reference": "tumbled, I could almost hear the universe snickering. My backsid", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>
|
| 226 |
{"doc_id": "blog_0819@Meta-Llama-3-8B-Instruct", "prompt": "You see, it's all about power, man. The War on Drugs is just a thinly veiled excuse for the government to exert its control ove", "reference": "r our lives. They're like, \"Oh, we're doing this to protect you ", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 227 |
{"doc_id": "news_0251@chunk_2", "prompt": "A scan of the Carolingian pot was conducted by Dr John Reid, consultant radiographer at the BGH, who is also a keen amateur arc", "reference": "haeologist. He was approached by Richard Welander, head of colle", "prediction": "i\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017i2<extra_id_56>(<extra_id_115>\u001c&<extra_id_64>"}
|
| 228 |
{"doc_id": "fic_0017@gpt-4o-2024-08-06", "prompt": "He was a man of rigorous discipline, a stalwart presence in the realm of naval journalism, and one who carried the weight of hi", "reference": "s responsibilities with a steely resolve. His eyes, accustomed t", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_4><extra_id_88>\u0019\u0000<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
|
@@ -241,7 +241,7 @@
|
|
| 241 |
{"doc_id": "fic_0964@chunk_2", "prompt": "The preacher, half amused, half embarrassed, stood still, as a woman came out of a tiny hut near by, and peered about her with ", "reference": "angry, short-sighted eyes. Mrs. Brazybone was a vast woman, with", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 242 |
{"doc_id": "blog_0946@Meta-Llama-3-8B-Instruct", "prompt": "I'm still trying to process everything that's going on. I feel like I'm stuck in some kind of never-ending nightmare. My dad's ", "reference": "financial situation is getting worse and worse, and I'm starting", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 243 |
{"doc_id": "blog_1429@Meta-Llama-3-8B", "prompt": "It was a great experience! And we were very happy with the outcome. However, when we came back from our break and met again yes", "reference": "terday, I felt like there was something missing. We had a lot of", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_16><extra_id_58><extra_id_25><extra_id_16><extra_id_58>&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 244 |
-
{"doc_id": "blog_1429@chunk_2", "prompt": "Out of those 3 days, our staff formed into 'action teams' to work together the rest of the year on specific areas we felt neede", "reference": "d redefining, exploration, clarity . I chose the numeracy group.", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P<extra_id_114>\u0017i2<extra_id_56>(<extra_id_115>
|
| 245 |
{"doc_id": "fic_1250@Meta-Llama-3-8B-Instruct", "prompt": "As he descended into the canyon, the silence was broken only by the soft rustling of leaves and the gentle crunch of gravel ben", "reference": "eath his hooves. Spread Horn's ears twitched ever so slightly, a", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
| 246 |
{"doc_id": "fic_0546@Meta-Llama-3-8B", "prompt": "It was evident that she was used to asking questions, and receiving answers, and that she did not think it worth while to show ", "reference": "emotion about either process. She laid down the paper, and took ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 247 |
{"doc_id": "spok_1404@chunk_1", "prompt": "Is take it away ryos extra. Can I call you Riles? Yeah, so cute as well my friends. Call me. How you that? All right. Well my e", "reference": "lementary school friends. No one here does except for trials. Oh", "prediction": "b<extra_id_64>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_16><extra_id_16><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
|
|
|
|
| 1 |
{"doc_id": "spok_1146@gpt-4o-mini-2024-07-18", "prompt": "Hello Pete, and thank you for having me on this journey today. I'm thrilled to share my story with you and the audience. I bega", "reference": "n my career in recruitment much like many, inspired by a vision ", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 2 |
{"doc_id": "blog_0819@gpt-4o-mini-2024-07-18", "prompt": "Why do they do this? It's as if the higher-ups are blind to the essence of what gatherings like this mean to people. The festiv", "reference": "al wasn't a den of iniquity; it was an oasis of community and cr", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 3 |
+
{"doc_id": "acad_1084@chunk_2", "prompt": "The functional relevance of these resting-state networks (RSNs) can be inferred from their spatial congruence with networks of ", "reference": "regions co-activated in association with specific cognitive and ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 4 |
{"doc_id": "spok_0062@Meta-Llama-3-70B", "prompt": "We talked about networking and what that means and what that looks like and what her experience has been with networking and th", "reference": "en also what her experience has been with finding jobs online an", "prediction": "Թ<extra_id_51>\rx4<extra_id_25>\u001c&\u0012<extra_id_68>\u0017S<extra_id_27>)<extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>i2<extra_id_56><extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 5 |
{"doc_id": "fic_1082@Meta-Llama-3-70B", "prompt": "The latter species attains greater height than the former, while the coastal variety is distinguished by its larger diameter, a", "reference": "lthough some specimens of both types exceed three hundred feet i", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 6 |
{"doc_id": "fic_0946@chunk_2", "prompt": "Linda paused and waited, wondering whether she might be of any help. \"Doctor!\" gasped the woman, frantically. \"My baby swallowe", "reference": "d a pin! And I'm sure it's in her lungs now. She breathes so que", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 7 |
{"doc_id": "acad_1085@chunk_1", "prompt": "Attention Deficit Hyperactivity Disorder (ADHD) is one of the most prevalent developmental disorders, affecting between 5 and 9", "reference": "% of school-aged children (Dittmann et al., 2009; Willcutt, 2012", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_27>)s<extra_id_33><extra_id_33><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 8 |
{"doc_id": "acad_1085@Meta-Llama-3-8B", "prompt": "Since the NA system plays an important role in regulating arousal levels, it has been proposed that ATX may improve attention b", "reference": "y increasing arousal (Del Campo et al., 2011; Posner and Peterse", "prediction": "<extra_id_64>\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_107>P\u0017\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
| 9 |
{"doc_id": "acad_0646@chunk_1", "prompt": "A plethora of studies in the economics/operational research literature is engaged with the measurement of producers' inefficien", "reference": "cy, thus relaxing the behavioral assumption that they are perfec", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 10 |
+
{"doc_id": "news_0186@chunk_1", "prompt": "The suspected gunman . arrested for shooting a newspaper photographer in Paris had lived in Britain for several years before re", "reference": "turning to France in July, the Paris prosecutor says. Abdelhakim", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`<extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 11 |
{"doc_id": "tvm_0398@chunk_2", "prompt": "Hey Holden, now my compass has gone crazy. It's spinning like a son of a bitch! Mine too, we must have hit a magnetic field or ", "reference": "something !! Mine too, we must have hit a magnetic field or some", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 12 |
{"doc_id": "spok_1591@gpt-4o-2024-08-06", "prompt": "Every instinct screamed at me to move, to alert someone, to do anything but sit frozen in terror at the back of the bus. Yet, I", "reference": " was trapped in a spellbinding gaze with the creature-a gaze tha", "prediction": "<extra_id_16><extra_id_58>:Z\u001c\u0012\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 13 |
{"doc_id": "fic_0027@chunk_2", "prompt": "Charmed with the discovery he followed his host's lead, and, with a languid air, replied to his rival in monosyllables. The cap", "reference": "tain watched with quiet satisfaction, and at each rebuff his opi", "prediction": "D\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
|
|
|
| 15 |
{"doc_id": "tvm_0567@chunk_1", "prompt": "In order to show you how a big symphony orchestra is put together, Benjamin Britten has written a big piece of music, which is ", "reference": "made up of smaller pieces that show you all the separate parts o", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 16 |
{"doc_id": "blog_0329@chunk_1", "prompt": "Tonight, my summer roommate Alex and I bonded over asian cuisine, broken hearts and a buried friend. I always knew we had poten", "reference": "tial. We have an assignment in Computer-Assisted Reporting class", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 17 |
{"doc_id": "fic_1288@chunk_1", "prompt": "If you want views about the world's news, read what Cottswold Ommony calls the views papers; there is plenty in them that thoro", "reference": "ughly zealous people believe. But remember the wise old ambassad", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 18 |
+
{"doc_id": "acad_1041@Meta-Llama-3-8B", "prompt": "On the contrary, WMS sequencing provides much larger amount of data allowing for deeper insights into microbiome structure and ", "reference": "function. As the complete set of DNA molecules present in the sa", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
|
| 19 |
{"doc_id": "news_0407@Meta-Llama-3-8B-Instruct", "prompt": "As the wedding days approached, Chelsea's anxiety about her hair grew. She had always been proud of her strawberry blonde locks", "reference": ", and the thought of losing them was daunting. But with the scal", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 20 |
{"doc_id": "blog_0959@gpt-4o-mini-2024-07-18", "prompt": "4) We must prioritize education and vocational training specifically designed for individuals like myself, those who are physic", "reference": "ally handicapped, ensuring that we are not left behind in the pr", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
| 21 |
{"doc_id": "acad_0444@gpt-4o-mini-2024-07-18", "prompt": "This variability in developmental timing is vital for the successful establishment of clonal lines, facilitating their adaptati", "reference": "on and survival in fluctuating environments. The plasticity inhe", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
|
|
|
| 31 |
{"doc_id": "spok_0508@chunk_1", "prompt": "Hello, everyone. Today is yodh 9 12 15. Yodh also known as August 9th 2019. The reason I honor the old calendar is because I be", "reference": "lieve this the calendar that more honors Yahweh the Gregorian ca", "prediction": "b<extra_id_64><extra_id_115>\u001c\u0012\u0001\b9ú<extra_id_113><extra_id_107>P<extra_id_110><extra_id_88>Q;w\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
| 32 |
{"doc_id": "acad_0567@gpt-4o-2024-08-06", "prompt": "The establishment of such procedures can be fundamentally linked to a deeper understanding of the decision-making context and t", "reference": "he intrinsic characteristics that govern it. A methodical approa", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_63>)XI<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 33 |
{"doc_id": "tvm_0881@Meta-Llama-3-70B", "prompt": "In the Pentagon. You've got the wrong man. You're my only hope. The admiral laughs. You're a great friend, James Greer. What's ", "reference": "this all about? Did you bring the book? Yes. Good. We've been wo", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 34 |
+
{"doc_id": "spok_1694@Meta-Llama-3-70B-Instruct", "prompt": "Yeah, sorry about that, folks. My bad. So, as I was saying, with the internet, fame has become more accessible. I mean, you can", "reference": " be famous for just about anything these days. You can be a soci", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>i2<extra_id_56>(<extra_id_115>)I<extra_id_114>(<extra_id_115>"}
|
| 35 |
{"doc_id": "blog_1389@Meta-Llama-3-8B-Instruct", "prompt": "Here is the next 500 words in the same style and tone as the original text: I got off the train at Central Station, and walked ", "reference": "to the food court to grab some lunch. I was thinking about that ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 36 |
{"doc_id": "news_0673@gpt-4o-mini-2024-07-18", "prompt": "Many students find themselves navigating the sometimes choppy waters of internship life, where the balance between goals and as", "reference": "signments can feel precarious at times. However, embracing the s", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 37 |
{"doc_id": "spok_1589@chunk_1", "prompt": "All right, welcome to the show if we get started here. I just want to talk a little bit about anchor dot. F m -- and if you hav", "reference": "en't heard about anchor dot f m -- it's literally the easiest wa", "prediction": "C<extra_id_115>\u001c\rx\rx<extra_id_113><extra_id_4><extra_id_88>\u0019\u0000<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>i2<extra_id_56><extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
|
|
|
| 40 |
{"doc_id": "acad_0258@chunk_2", "prompt": "For example, the production of packaging intended to come in contact with food and drink (known as food contact materials, FCMs", "reference": ") needs to comply with the EU food contact legislation (Regulati", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 41 |
{"doc_id": "acad_0702@Meta-Llama-3-8B", "prompt": "While some scholars have highlighted the importance of communication (e.g., Bessant and Rush, 1995; Zahra and Covin, 1995), oth", "reference": "ers have questioned its relevance (e.g., Miller and Friesen, 198", "prediction": "Թ<extra_id_51>\rx\u001c\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_63>)XI<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 42 |
{"doc_id": "tvm_0398@Meta-Llama-3-8B-Instruct", "prompt": "Here is the continuation of the text: Holden's voice was laced with panic as he frantically scanned the horizon. \"What the hell", "reference": " is going on, Skipper? We're lost!\" Stepek's gruff tone only add", "prediction": "<extra_id_16><extra_id_58>:Zﹷ<extra_id_51>\rx4<extra_id_25>\u001c\b<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017i2<extra_id_56>(<extra_id_115>\u001c&<extra_id_64>"}
|
| 43 |
+
{"doc_id": "blog_0081@chunk_2", "prompt": "The views on the road weren't too picturesque, either. We should've taken the Gatwick Express , which costs pretty much the sam", "reference": "e; the saddest part is that the hotel is just a 2-minute walk fr", "prediction": "\u0012\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 44 |
{"doc_id": "acad_0820@Meta-Llama-3-8B", "prompt": "Through this new representation, Muslim women have gained greater access to a wider range of resources, and the potential to es", "reference": "tablish their own businesses (Bakr, 2016; El-Ghobashy, 2011; She", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 45 |
{"doc_id": "blog_1364@gpt-4o-2024-08-06", "prompt": "It reverberated through the air, words echoing off the walls as if carried by the wind itself: the force, the fervor, the raw p", "reference": "ower of his conviction. Sahir stood there, his presence filling ", "prediction": "\u001c\u0012\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 46 |
{"doc_id": "spok_1587@Meta-Llama-3-8B", "prompt": "Okay, well, I think we should just take a step back. You don't have to worry about me. I'm going to be fine. I'm not worried ab", "reference": "out you. I'm worried about the podcast. If you don't mind. I'm g", "prediction": "<extra_id_64><extra_id_115>\u001c\u0001zlú<extra_id_113><extra_id_115>)<extra_id_122>s<extra_id_122>s<extra_id_122>s<extra_id_4><extra_id_110>i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
|
|
|
| 50 |
{"doc_id": "fic_0923@Meta-Llama-3-8B-Instruct", "prompt": "As the young man continued to walk down Arcadia Street, he couldn't shake off the feeling of being out of place. The grimy faca", "reference": "des of the houses seemed to loom over him, their peeling paint a", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 51 |
{"doc_id": "spok_0047@chunk_2", "prompt": "My parents are Mortal Kombat because I was like I came was like this shit and then afterwards I know my parents bake afterward ", "reference": "the newscaster like, you know, if you let your kids play Mortal ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 52 |
{"doc_id": "acad_0077@Meta-Llama-3-70B-Instruct", "prompt": "The impact of human activities on the Earth's surface and subsurface is evident in the widespread occurrence of landslides, sub", "reference": "sidence, and sinkholes, often triggered or exacerbated by anthro", "prediction": "<extra_id_64>\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 53 |
+
{"doc_id": "news_0727@chunk_2", "prompt": "I am already in Donetsk. A - Is it what I am thinking about? the one called 'M'? B - yes . A - 'PM'? B - yes, yes, yes. BUK. A ", "reference": "- oh, BM, yes, yes, yes. B - BUK . A - I understand . B - BUK, B", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0i(<extra_id_119>C<extra_id_107>P*\rx<extra_id_63>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
|
| 54 |
+
{"doc_id": "spok_0496@Meta-Llama-3-70B-Instruct", "prompt": "...but it's likely that he had an inkling, given the rising tensions between Italy and Ethiopia. In October 1935, Italy invaded", "reference": " Ethiopia, and Amedeo's unit was deployed to the front lines. Th", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_94><extra_id_77><extra_id_110><extra_id_88>Q<extra_id_54>\u0017<extra_id_88><extra_id_2>\u0017\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
| 55 |
{"doc_id": "blog_0710@Meta-Llama-3-70B", "prompt": "So, I figured that if I ate just a few bites, then threw the rest away, Jack wouldn't know any better. I started off slowly wit", "reference": "h some beans (which weren't bad) and then moved on to the pork. ", "prediction": "\u0019\u0000\u001aﹷ<extra_id_51>\rx4<extra_id_25>\u001c\b<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 56 |
+
{"doc_id": "fic_0704@Meta-Llama-3-8B-Instruct", "prompt": "As the steamboat, or rather, the raft, drew near to the shore, Freddie Bobbsey called out to his sister, \"All aboard, Flossie! ", "reference": "Get on, get on!\" Flossie, with her blue eyes shining with excite", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P<extra_id_114>\u0017i2<extra_id_56>(<extra_id_115>(<extra_id_115><extra_id_64>"}
|
| 57 |
+
{"doc_id": "blog_0945@chunk_2", "prompt": "I thought they were cute but really thought nothing of it. They then kept looking our way, mind you Brad stopped headbanging an", "reference": "d was sitting on the bench with Phil and no one was in front of ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 58 |
{"doc_id": "fic_0923@chunk_2", "prompt": "Still appearing utterly oblivious of the questions showered upon him by the now frantic man who had constituted himself as guid", "reference": "e, the young man had got midway up the street, and was still sea", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 59 |
{"doc_id": "spok_1705@chunk_2", "prompt": "So if you can picture eight feet tall like the basket that you used to be able to dunk on when you're in like 8th grade because", "reference": " you couldn't get 10 that's almost how long his arms are which i", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 60 |
{"doc_id": "acad_0521@chunk_1", "prompt": "A wide body of literature in archaeology and anthropology discusses the concept of craft specialisation. This topic is importan", "reference": "t because it contributes to understanding the development of soc", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
|
|
|
| 104 |
{"doc_id": "acad_0483@Meta-Llama-3-8B", "prompt": "A survey of 5000 UK residents found that perceptions of visual amenity were more negative among those who had previously oppose", "reference": "d onshore wind farms, even if they lived further away from exist", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 105 |
{"doc_id": "spok_1550@chunk_1", "prompt": "Wrong, so wrong by c-pillar 42 read by litter Aryan. summary There is a kiss and it is so wrong. It turns out that even after k", "reference": "nowing each other for about six thousand years. It is possible t", "prediction": "U<extra_id_29><extra_id_120><extra_id_115>\u001c\u0012\u0001\b9ú<extra_id_113><extra_id_16><extra_id_58>:<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0i(<extra_id_119>C<extra_id_107>P*\rx<extra_id_63>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
|
| 106 |
{"doc_id": "news_0483@Meta-Llama-3-8B", "prompt": "'We are working hard to minimise any disruption to our customers and will contact them directly if their flight has been affect", "reference": "ed.' BA added that it was continuing to monitor the situation cl", "prediction": "\u0019\u0000\u001aU\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
|
| 107 |
+
{"doc_id": "blog_0065@gpt-4o-mini-2024-07-18", "prompt": "Not for You - A powerful track from Vitalogy, it echoes the frustrations of feeling trapped within the confines of expectation.", "reference": " Eddie's emotive delivery resonated deeply, especially during th", "prediction": "<extra_id_119><extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 108 |
{"doc_id": "spok_0558@chunk_1", "prompt": "Hi everyone, and welcome to this incessant home tomorrow. I am like internally screaming right now. This is a podcast about thi", "reference": "ngs that just make isn't comfortable and that we don't want to t", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 109 |
{"doc_id": "spok_1215@chunk_1", "prompt": "Welcome to the good Christian talks podcast. I'm Levi and I'm Chris. Thank you so much for joining us this week on this podcast", "reference": ". We select one taco week to help each one of us get the Bible i", "prediction": "\u0019\u0000\u001aﹷ<extra_id_51>\rx4<extra_id_25>\u001c\b<extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>i2<extra_id_56>(<extra_id_115>)I<extra_id_16><extra_id_58><extra_id_25><extra_id_64>"}
|
| 110 |
{"doc_id": "acad_0525@gpt-4o-mini-2024-07-18", "prompt": "This observation suggests a multifaceted role for p53 in modulating not only the cellular response to DNA damage but also the m", "reference": "etabolism of environmental carcinogens, thereby influencing susc", "prediction": "\u0012\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
|
|
|
| 115 |
{"doc_id": "blog_0358@gpt-4o-2024-08-06", "prompt": "67-year-old S Puhaindran was a young teacher in Raffles Institution when Mr Goh was a student there. \"I remember him well,\" Mr.", "reference": " Puhaindran recounted with an affectionate smile. \"Goh was the k", "prediction": "<extra_id_119><extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_63>)XI<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 116 |
{"doc_id": "spok_0718@chunk_2", "prompt": "Welcome to our second of two episodes on John T Scopes last week We examined the rise of the anti-evolution movement in the ear", "reference": "ly 1920s that led to the establishment of the butler act which b", "prediction": "<extra_id_25><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 117 |
{"doc_id": "spok_1582@chunk_1", "prompt": "Hello everyone and welcome to the Notre grow podcast. I'm your host. Dr. Ivan Khan, and we'll be breaking down topics around ed", "reference": "ucation growth and culture with the intention to help your growt", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 118 |
+
{"doc_id": "blog_1429@Meta-Llama-3-70B-Instruct", "prompt": "As I sat in the staff meeting, listening to our administrators outline the plan for the year, I couldn't help but feel a sense ", "reference": "of excitement and trepidation. The idea of \"writing it down and ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001z<extra_id_111>\u001c\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 119 |
{"doc_id": "tvm_0327@chunk_2", "prompt": "Dead serious. See... the world is made up of rules. Kids don't like rules. They rebel for a while, but ultimately they start to", "reference": " fall in line and become ordinary Joe's like you and me. But som", "prediction": "C<extra_id_115>\u001c\u0012\u0001\b9ú<extra_id_113><extra_id_107>P<extra_id_110><extra_id_88>Q;w\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107>P*\rx<extra_id_63>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
|
| 120 |
{"doc_id": "tvm_0857@gpt-4o-mini-2024-07-18", "prompt": "With silent determination, MALONE casts an assured glance at the OARSMEN, who hesitate for a moment, caught between the ardor o", "reference": "f loyalty and the dread of what lies ahead. One by one, they asc", "prediction": "C\u001c&<extra_id_121>\u0001\b9ú<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 121 |
{"doc_id": "blog_0293@chunk_1", "prompt": "Michael posted more about abortion and guns on his website and, as a woman, I violentely reacted to some of his comments on abo", "reference": "rtion. My reaction was too long to post on his blog, so I'm doin", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 122 |
+
{"doc_id": "news_0530@Meta-Llama-3-8B-Instruct", "prompt": "As the controversy surrounding the botched trial of John Downey continued to simmer, the UK government's refusal to launch a ju", "reference": "dicial inquiry into the matter only added fuel to the fire. Nort", "prediction": "x<extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 123 |
{"doc_id": "spok_0772@Meta-Llama-3-8B-Instruct", "prompt": "Here's the next 500 words in the same style and tone: David: Yeah, okay, let's move on to the next question. At wonderland unde", "reference": "rscore underscore, what's the weirdest thing that's happened to ", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 124 |
{"doc_id": "news_0390@chunk_2", "prompt": "For example, Benson said that $1 million in $100 bills weighs 22 pounds, while $1 million in 500 euro notes weighs 3.5 pounds. ", "reference": "\"It's a huge difference,\" he said. Though Europe is highly attra", "prediction": "<extra_id_115><extra_id_115>\u001c\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 125 |
{"doc_id": "spok_0704@gpt-4o-2024-08-06", "prompt": "You see, creativity intertwines itself with the rituals that one builds around it. For me, it's not so much about adhering to a", "reference": " strict schedule, as it is about weaving those moments of intros", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 126 |
+
{"doc_id": "news_0483@Meta-Llama-3-8B-Instruct", "prompt": "As the storm continues to rage on the East Coast, the travel chaos shows no signs of abating. Thousands of Britons are facing a", "reference": "nother day of uncertainty, with airlines scrambling to rebook fl", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 127 |
{"doc_id": "fic_1229@chunk_2", "prompt": "Phillips, his valet, had heard the shot, and knowing that the prince's cousin, Prince Miguel, with Don Solado, prime minister o", "reference": "f Joyalita, had made attempts on his life before, in New York, h", "prediction": "C\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0\u0014\u0017i2<extra_id_56>(<extra_id_115><extra_id_64>\u0017\u000b"}
|
| 128 |
{"doc_id": "fic_0555@gpt-4o-2024-08-06", "prompt": "The memories of carefree childhood days woven amidst the tranquil charm of the estate danced in his mind, anchoring him to this", "reference": " beloved place with an unyielding fondness. The familiar sound o", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 129 |
{"doc_id": "blog_1347@Meta-Llama-3-70B-Instruct", "prompt": "As I gazed deeper into the mirror, my eyes seemed to bore into my own soul. The reflection staring back at me was a stranger, y", "reference": "et eerily familiar. The lines on my face told a story of pain an", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\b9ú<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
|
|
|
| 135 |
{"doc_id": "news_0135@gpt-4o-mini-2024-07-18", "prompt": "However, the events of last October reshaped those timelines and priorities, pushing Virgin Galactic into a more insular approa", "reference": "ch to the development of their space ambitions. The firm recogni", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 136 |
{"doc_id": "acad_0489@Meta-Llama-3-8B", "prompt": "Individual hackers may hack into websites or online services to deface content, change information, disrupt service, or simply ", "reference": "out of curiosity.16 Lastly, activist groups may target critical ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 137 |
{"doc_id": "news_0246@Meta-Llama-3-70B", "prompt": "That might seem an unlikely trio of activities for a man preparing for a world title fight but there were two serious reasons f", "reference": "or Manny's trip here. One was to promote his forthcoming book, a", "prediction": "<extra_id_16><extra_id_58>:Z\u001c\u0012\u0001\b\u0017\u0017*<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 138 |
+
{"doc_id": "acad_0497@gpt-4o-mini-2024-07-18", "prompt": "Moreover, the limitations inherent within the traditional diagnostic framework underscore a pressing need for an integrative, t", "reference": "ransdiagnostic approach to emotional disorders that recognizes a", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 139 |
{"doc_id": "spok_1425@chunk_1", "prompt": "Even when you say listen, we might not be getting billboard this time around because you're a brand new artist. They don't nece", "reference": "ssarily hear that. So first off I just want to say welcome to th", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 140 |
{"doc_id": "blog_0578@Meta-Llama-3-8B-Instruct", "prompt": "As I reflect on my childhood friendships, I'm reminded of the many memories I've accumulated over the years. One of my earliest", "reference": " and most cherished friendships was with Rachel. We met in kinde", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 141 |
{"doc_id": "spok_1607@Meta-Llama-3-70B", "prompt": "We have and if you haven't had that experience yet, then I would definitely suggest you start paying attention to how you feel ", "reference": "and learning to be in tune with yourself and your intuition. So ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
|
|
|
| 161 |
{"doc_id": "tvm_0962@Meta-Llama-3-70B-Instruct", "prompt": "Walter's eyes dart around the elevator, searching for an escape from the uncomfortable silence. Ted's gaze lingers, as if darin", "reference": "g Walter to ask again. Finally, he responds, his tone measured, ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_94><extra_id_77><extra_id_110>\u0003<extra_id_16><extra_id_58>㥦\u0017\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
| 162 |
{"doc_id": "acad_0646@gpt-4o-2024-08-06", "prompt": "These studies employed Geographic Information Systems (GIS) to accurately delineate spatial relationships and pinpoint geograph", "reference": "ical proximities between producers, allowing for a nuanced analy", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 163 |
{"doc_id": "acad_0772@chunk_2", "prompt": "However, the pressure signal of the catheter is affected by the viscosity of the colonic fluids [28]. Arkwright et al. [28] sho", "reference": "wed that the predetermined amplitude of the applied pressure on ", "prediction": "C\u001c&<extra_id_121>\u0001\b\u0017\u0017*<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 164 |
+
{"doc_id": "news_0656@gpt-4o-mini-2024-07-18", "prompt": "In facing this precarious situation, the possibility of requiring a financial rescue looms large on Italy's horizon, though man", "reference": "y experts argue it lacks the structural and political framework ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 165 |
{"doc_id": "spok_0704@Meta-Llama-3-8B", "prompt": "That's really cool. I think that's great. I mean, I'm not judging anyone or anything, but it's interesting because I remember r", "reference": "eading an article by Tony Robbins and he said that he stopped dr", "prediction": "<extra_id_25><extra_id_115>\u001cﹷ\u0017\u0017\u0017\u0017*<extra_id_16><extra_id_58>:\u0017S<extra_id_27>)<extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 166 |
{"doc_id": "news_1257@gpt-4o-2024-08-06", "prompt": "The goal is to prevent Iran from refining uranium beyond a certain threshold, drastically reducing its capacity to progress tow", "reference": "ard a nuclear weapon in the near term. In exchange, Iran will re", "prediction": "\u0017\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
|
| 167 |
{"doc_id": "spok_1443@Meta-Llama-3-8B", "prompt": "They said they couldn't stand the smell of your dead cat anymore. So they took off. You've been living alone since 1999 and no ", "reference": "one has heard from them again. I guess they just went back home ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zlú<extra_id_113><extra_id_115>)<extra_id_122>s<extra_id_122>s<extra_id_122>s<extra_id_33><extra_id_33><extra_id_33><extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
|
|
|
| 179 |
{"doc_id": "blog_0126@chunk_2", "prompt": "What took you so long? God help you become victorious. I want to say hello to Bush, to shake his hand. We came out of the grave", "reference": ".'\" The Guadian, 3-22-03 \"As hundreds of coalition troops swept ", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P<extra_id_114>\u0017i2<extra_id_56>(<extra_id_115>\u001c&9(<extra_id_115>"}
|
| 180 |
{"doc_id": "fic_0051@chunk_2", "prompt": "Two seamen, each pushing an oar before him, swam out to the nearest boat, and having got on board of her, and freed her from he", "reference": "r entanglements, they rowed out to the capsized boat, and towed ", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 181 |
{"doc_id": "tvm_0383@chunk_1", "prompt": "How do you do. My name is Penelope Ryan. This is a simple-minded play about men who enjoy killing -- and those who don't. I am ", "reference": "Harold Ryan, her husband. I have killed perhaps two hundred men ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
| 182 |
+
{"doc_id": "acad_0983@Meta-Llama-3-8B", "prompt": "Understanding how much water is available for plants in a given ecosystem is critical for managing natural resources and assess", "reference": "ing the impact of land-use/management practices on water availab", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_63>)XI<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 183 |
{"doc_id": "fic_0034@Meta-Llama-3-70B", "prompt": "The ears were short and delicately made; the eyes were well-placed; and the distance to the angle of the jaw was such as good b", "reference": "reeders desire. Considering him closer she became aware of the l", "prediction": "<extra_id_64>\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 184 |
{"doc_id": "blog_0069@chunk_1", "prompt": "..Is the one, which tells you immediately, what the outcome will be: Good food, bad husband, horror able dinner, acing a test, ", "reference": "bad sex, insanely demented neighbours... the last thing is an is", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
|
| 185 |
{"doc_id": "blog_0747@gpt-4o-2024-08-06", "prompt": "I finally reached Plaza Singapura, and there was Darius, waiting patiently with a grin. The moment I saw him, I felt a little f", "reference": "lutter in my stomach. It's funny how a net friend can feel so fa", "prediction": "<extra_id_16><extra_id_58>:Z\u001c\u0012\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
|
|
|
| 199 |
{"doc_id": "acad_0807@gpt-4o-2024-08-06", "prompt": "As a result, the development of more adaptive operational strategies that can detect and respond to ice formation is paramount.", "reference": " Integrating advanced sensor technology can offer a real-time as", "prediction": "<extra_id_119><extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 200 |
{"doc_id": "fic_0975@Meta-Llama-3-70B", "prompt": "\"Well, what does the chief say, sir?\" he asked. \"He doesn't seem very cheerful over it.\" \"He isn't,\" replied Ramshaw grimly. \"S", "reference": "ome scoundrel has tampered with the high-pressure slide valves. ", "prediction": "<extra_id_32>÷\u0017\u0017\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 201 |
{"doc_id": "spok_1483@Meta-Llama-3-8B-Instruct", "prompt": "But we're not going to be bored, because we're going to dive into the story of the 1974 Zaire team and explore the complexities", "reference": " of colonialism and its lasting impact on the African continent.", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 202 |
+
{"doc_id": "fic_0841@chunk_1", "prompt": "One evening Mr. Courtnay Fox, the London correspondent of the Edinburgh Chronicle, was as usual in his own room in the office i", "reference": "n Fleet-street, when a card was brought to him. \"Show the gentle", "prediction": "\u001c\u0012\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 203 |
{"doc_id": "news_0227@chunk_2", "prompt": "We've been writers in our own right for several years before we started to write together, and from the beginning we thought it", "reference": " was a great idea to stay secret behind a pen name. Then two thi", "prediction": "\u0019\u0000\u001aU\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_63><extra_id_107>P<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64><extra_id_101>0<extra_id_94><extra_id_77><extra_id_110>\u0003<extra_id_16><extra_id_58>Һ<extra_id_113>\u0017i2<extra_id_56>(<extra_id_115>\u001c&9(<extra_id_115>"}
|
| 204 |
{"doc_id": "tvm_0327@Meta-Llama-3-70B", "prompt": "It's the humane thing to do. We don't want them to suffer. You know how hard it is to find a decent lawyer these days? This is ", "reference": "the place? Yeah, I think so. Come on, Lizzy. Let's check it out.", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zlú<extra_id_113><extra_id_115>)<extra_id_122>s<extra_id_122>\u001fZ\u0019\u0000^<extra_id_101>0i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 205 |
{"doc_id": "blog_0026@chunk_1", "prompt": "On Monday morning at work, I slipped on a wet floor. Mondays are always heavy load days. Mondays after a holiday... ew. Twelve ", "reference": "dogs to do. And they wouldn't let me wash some of them a day ear", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58><extra_id_67><extra_id_88>Q;<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 206 |
{"doc_id": "news_0187@gpt-4o-2024-08-06", "prompt": "Jimmy was grateful for Claire's presence and quickly found that her vivacious spirit brightened the household. Even Gemma, who ", "reference": "had her reservations at first about her father's burgeoning rela", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 207 |
+
{"doc_id": "fic_1122@Meta-Llama-3-8B-Instruct", "prompt": "As the days passed, the anticipation grew, and the village was abuzz with excitement. The two suitors, myself and my rival, wer", "reference": "e summoned to the farmer's estate, where we were to present ours", "prediction": "<extra_id_25><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 208 |
{"doc_id": "news_0530@gpt-4o-mini-2024-07-18", "prompt": "In a shocking escalation of political tension, Mr. Robinson's outrage reverberated throughout Northern Ireland's tumultuous lan", "reference": "dscape, reigniting old wounds and fears among the populace. The ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 209 |
{"doc_id": "news_0251@chunk_1", "prompt": "Experts were unable to open the 9th century Carolingian pot (pictured), which was found on church land in Dumfries and Galloway", "reference": " in September . The mystery surrounding the contents of a Viking", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 210 |
{"doc_id": "spok_1103@chunk_2", "prompt": "It was a questionable mix at that point in history. One member of the crowd tried to storm the stage, but he was quickly subdue", "reference": "d by security after he managed to break one of the presentation ", "prediction": "r<extra_id_25>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014i<extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
|
|
|
| 222 |
{"doc_id": "tvm_0420@gpt-4o-2024-08-06", "prompt": "We simply critique the cult that surrounds them. Sirens wail in the distance, and the thrum of anticipation grows. Young is sti", "reference": "ll spinning his tale, unaware or uncaring of the chaos, maintain", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*f<extra_id_16><extra_id_58><extra_id_25><extra_id_64>\u0017"}
|
| 223 |
{"doc_id": "news_1166@gpt-4o-2024-08-06", "prompt": "But rather than dismissing this sentiment as merely the nostalgic ramblings of a bygone generation, it might serve us well to d", "reference": "elve deeper into the root of such concerns. The discourse surrou", "prediction": "s'I<extra_id_16><extra_id_58><extra_id_121>\u0001z<extra_id_111><extra_id_107>z<extra_id_111><extra_id_107>P<extra_id_110><extra_id_88>Q;w\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107>P*\rx<extra_id_63>\u001f^<extra_id_101><extra_id_93>(<extra_id_115>\u001c&<extra_id_64>"}
|
| 224 |
{"doc_id": "spok_1150@chunk_2", "prompt": "And they have a new proprietary blend which is their metabolic ferment, which I can't explain to you the science behind it, but", "reference": " I will tell you that it's supposed to help boost your natural c", "prediction": "\u0019\u0000ú<extra_id_113>x\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_16><extra_id_58>:<extra_id_110><extra_id_88>Q<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 225 |
+
{"doc_id": "blog_0026@gpt-4o-2024-08-06", "prompt": "I slipped, my feet skidding out from under me like an awkward rendition of a cartoon character slipping on a banana peel. As I ", "reference": "tumbled, I could almost hear the universe snickering. My backsid", "prediction": "<extra_id_115><extra_id_115>\u001c\u0012\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64>\u0017\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 226 |
{"doc_id": "blog_0819@Meta-Llama-3-8B-Instruct", "prompt": "You see, it's all about power, man. The War on Drugs is just a thinly veiled excuse for the government to exert its control ove", "reference": "r our lives. They're like, \"Oh, we're doing this to protect you ", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 227 |
{"doc_id": "news_0251@chunk_2", "prompt": "A scan of the Carolingian pot was conducted by Dr John Reid, consultant radiographer at the BGH, who is also a keen amateur arc", "reference": "haeologist. He was approached by Richard Welander, head of colle", "prediction": "i\u001c&<extra_id_121>\u0001z<extra_id_111>ú<extra_id_113><extra_id_115>)I<extra_id_16><extra_id_58>:<extra_id_63>\u001c&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017i2<extra_id_56>(<extra_id_115>\u001c&<extra_id_64>"}
|
| 228 |
{"doc_id": "fic_0017@gpt-4o-2024-08-06", "prompt": "He was a man of rigorous discipline, a stalwart presence in the realm of naval journalism, and one who carried the weight of hi", "reference": "s responsibilities with a steely resolve. His eyes, accustomed t", "prediction": "\u001c\u0012\u001cq\u001c\rx<extra_id_113><extra_id_4><extra_id_88>\u0019\u0000<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0014`0C)<extra_id_54><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
|
|
|
| 241 |
{"doc_id": "fic_0964@chunk_2", "prompt": "The preacher, half amused, half embarrassed, stood still, as a woman came out of a tiny hut near by, and peered about her with ", "reference": "angry, short-sighted eyes. Mrs. Brazybone was a vast woman, with", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 242 |
{"doc_id": "blog_0946@Meta-Llama-3-8B-Instruct", "prompt": "I'm still trying to process everything that's going on. I feel like I'm stuck in some kind of never-ending nightmare. My dad's ", "reference": "financial situation is getting worse and worse, and I'm starting", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64>\u0017<extra_id_16><extra_id_58>I<extra_id_16><extra_id_58><extra_id_118><extra_id_33>\u0017*\u0017*m<extra_id_63>(<extra_id_115><extra_id_64>"}
|
| 243 |
{"doc_id": "blog_1429@Meta-Llama-3-8B", "prompt": "It was a great experience! And we were very happy with the outcome. However, when we came back from our break and met again yes", "reference": "terday, I felt like there was something missing. We had a lot of", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_16><extra_id_58><extra_id_25><extra_id_16><extra_id_58>&<extra_id_16><extra_id_58><extra_id_25><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0<extra_id_118><extra_id_33>\u0017\u0017q<extra_id_63>)I<extra_id_114>I<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 244 |
+
{"doc_id": "blog_1429@chunk_2", "prompt": "Out of those 3 days, our staff formed into 'action teams' to work together the rest of the year on specific areas we felt neede", "reference": "d redefining, exploration, clarity . I chose the numeracy group.", "prediction": "<extra_id_115>>\rx\rx<extra_id_113><extra_id_16><extra_id_58><extra_id_25><extra_id_37><extra_id_115>)X<extra_id_47>\u0017S<extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_107><extra_id_70><extra_id_107>P<extra_id_114>\u0017i2<extra_id_56>(<extra_id_115>(<extra_id_115><extra_id_64>"}
|
| 245 |
{"doc_id": "fic_1250@Meta-Llama-3-8B-Instruct", "prompt": "As he descended into the canyon, the silence was broken only by the soft rustling of leaves and the gentle crunch of gravel ben", "reference": "eath his hooves. Spread Horn's ears twitched ever so slightly, a", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64>r<extra_id_25><extra_id_64><extra_id_101>0\u0014\u0017<extra_id_107>P<extra_id_114><extra_id_110>\u001fZ<extra_id_25><extra_id_64>\u0017"}
|
| 246 |
{"doc_id": "fic_0546@Meta-Llama-3-8B", "prompt": "It was evident that she was used to asking questions, and receiving answers, and that she did not think it worth while to show ", "reference": "emotion about either process. She laid down the paper, and took ", "prediction": "<extra_id_115><extra_id_115>\u001c\u0001zl<extra_id_111><extra_id_4><extra_id_88>\u0019\u0000\u000fQ;w\u0017S<extra_id_27>)<extra_id_54><extra_id_64><extra_id_101>0<extra_id_107>P\u0017<extra_id_16><extra_id_58><extra_id_60>\u0017<extra_id_63>)IOI<extra_id_16><extra_id_45><extra_id_45><extra_id_45><extra_id_45>"}
|
| 247 |
{"doc_id": "spok_1404@chunk_1", "prompt": "Is take it away ryos extra. Can I call you Riles? Yeah, so cute as well my friends. Call me. How you that? All right. Well my e", "reference": "lementary school friends. No one here does except for trials. Oh", "prediction": "b<extra_id_64>\u001c&<extra_id_121>\u0001\u0017\u0017\u001c\u0012<extra_id_68><extra_id_16><extra_id_58><extra_id_25><extra_id_16><extra_id_16><extra_id_16><extra_id_58><extra_id_25><extra_id_64><extra_id_101>0\u0017S<extra_id_32>ú<extra_id_39>\u001f^<extra_id_101>0\u001fZ<extra_id_25>\u0010\u0010\u0010\u0010\u0010"}
|
muon/predictions_01.jsonl
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
muon/predictions_02.jsonl
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
muon/predictions_03.jsonl
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
muon/predictions_04.jsonl
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
muon/predictions_05.jsonl
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
muon/predictions_06.jsonl
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
muon/predictions_07.jsonl
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
muon/predictions_08.jsonl
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
muon/predictions_09.jsonl
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
muon/predictions_10.jsonl
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
muon/settings.json
CHANGED
|
@@ -12,27 +12,30 @@
|
|
| 12 |
"dimension_model": 256,
|
| 13 |
"attention_heads": 8,
|
| 14 |
"decoder_blocks": 4,
|
| 15 |
-
"learning_rate": 0.
|
| 16 |
"weight_decay": 0.01,
|
| 17 |
"beta1": 0.9,
|
| 18 |
-
"beta2": 0.
|
| 19 |
"epsilon": 1e-08,
|
| 20 |
"gamma": 0.025,
|
| 21 |
"momentum": 0.95,
|
| 22 |
"ns_steps": 5,
|
| 23 |
"ns_epsilon": 1e-07,
|
| 24 |
-
"aux_learning_rate": 0.
|
|
|
|
|
|
|
|
|
|
| 25 |
"max_grad_norm": 1.0,
|
| 26 |
"prompt_tokens": 128,
|
| 27 |
"max_new_tokens": 64,
|
| 28 |
-
"device": "
|
| 29 |
"mixed_precision": false,
|
| 30 |
-
"tokenizer_name": "/
|
| 31 |
-
"cache_dir": "/
|
| 32 |
-
"output_dir": "
|
| 33 |
"max_train_tokens": 0,
|
| 34 |
"prepare_only": false,
|
| 35 |
-
"resume":
|
| 36 |
},
|
| 37 |
"model": {
|
| 38 |
"vocab_size": 388,
|
|
@@ -44,34 +47,53 @@
|
|
| 44 |
"parameters": 3355012,
|
| 45 |
"optimizer_groups": [
|
| 46 |
{
|
| 47 |
-
"lr": 0.
|
| 48 |
"use_muon": true,
|
| 49 |
"momentum": 0.95,
|
| 50 |
"ns_steps": 5,
|
| 51 |
"betas": [
|
| 52 |
0.9,
|
| 53 |
-
0.
|
| 54 |
],
|
| 55 |
"eps": 1e-08,
|
| 56 |
"weight_decay": 0.01,
|
| 57 |
-
"ns_eps": 1e-07
|
|
|
|
|
|
|
| 58 |
},
|
| 59 |
{
|
| 60 |
-
"lr": 0.
|
| 61 |
"use_muon": false,
|
| 62 |
"momentum": 0.95,
|
| 63 |
"ns_steps": 5,
|
| 64 |
"betas": [
|
| 65 |
0.9,
|
| 66 |
-
0.
|
| 67 |
],
|
| 68 |
"eps": 1e-08,
|
| 69 |
"weight_decay": 0.01,
|
| 70 |
-
"ns_eps": 1e-07
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
}
|
| 72 |
],
|
| 73 |
"data": {
|
| 74 |
-
"dataset_fingerprint": "
|
| 75 |
"split_method": "90/5/5 by source doc_id before @, then seeded row subsampling",
|
| 76 |
"full_split_documents": {
|
| 77 |
"train": 59688,
|
|
@@ -15518,14 +15540,8 @@
|
|
| 15518 |
"validation_target_tokens": 720136,
|
| 15519 |
"test_target_tokens": 708110
|
| 15520 |
},
|
| 15521 |
-
"learning_rate_schedule": "
|
| 15522 |
"evaluation_interval": "0.1 of selected training budget",
|
| 15523 |
-
"precision_changes": [
|
| 15524 |
-
{
|
| 15525 |
-
"train_tokens": 21366755,
|
| 15526 |
-
"previous_mixed_precision": true,
|
| 15527 |
-
"mixed_precision": false
|
| 15528 |
-
}
|
| 15529 |
-
],
|
| 15530 |
"bleu_definition": "Greedy continuation after prompt_tokens; reference is the next max_new_tokens tokens."
|
| 15531 |
}
|
|
|
|
| 12 |
"dimension_model": 256,
|
| 13 |
"attention_heads": 8,
|
| 14 |
"decoder_blocks": 4,
|
| 15 |
+
"learning_rate": 0.001,
|
| 16 |
"weight_decay": 0.01,
|
| 17 |
"beta1": 0.9,
|
| 18 |
+
"beta2": 0.95,
|
| 19 |
"epsilon": 1e-08,
|
| 20 |
"gamma": 0.025,
|
| 21 |
"momentum": 0.95,
|
| 22 |
"ns_steps": 5,
|
| 23 |
"ns_epsilon": 1e-07,
|
| 24 |
+
"aux_learning_rate": 0.0001,
|
| 25 |
+
"lr_schedule": "cosine",
|
| 26 |
+
"warmup_fraction": 0.05,
|
| 27 |
+
"min_lr_ratio": 0.1,
|
| 28 |
"max_grad_norm": 1.0,
|
| 29 |
"prompt_tokens": 128,
|
| 30 |
"max_new_tokens": 64,
|
| 31 |
+
"device": "auto",
|
| 32 |
"mixed_precision": false,
|
| 33 |
+
"tokenizer_name": "/Users/samawalke/Project/Masters/ANLP/Assignment2/2022101025_assignment2/part2/tokenizer",
|
| 34 |
+
"cache_dir": "/Users/samawalke/Project/Masters/ANLP/Assignment2/2022101025_assignment2/part2/cache",
|
| 35 |
+
"output_dir": "part2/part2_results/output/muon",
|
| 36 |
"max_train_tokens": 0,
|
| 37 |
"prepare_only": false,
|
| 38 |
+
"resume": null
|
| 39 |
},
|
| 40 |
"model": {
|
| 41 |
"vocab_size": 388,
|
|
|
|
| 47 |
"parameters": 3355012,
|
| 48 |
"optimizer_groups": [
|
| 49 |
{
|
| 50 |
+
"lr": 0.0,
|
| 51 |
"use_muon": true,
|
| 52 |
"momentum": 0.95,
|
| 53 |
"ns_steps": 5,
|
| 54 |
"betas": [
|
| 55 |
0.9,
|
| 56 |
+
0.95
|
| 57 |
],
|
| 58 |
"eps": 1e-08,
|
| 59 |
"weight_decay": 0.01,
|
| 60 |
+
"ns_eps": 1e-07,
|
| 61 |
+
"attention_layout": "full_projection",
|
| 62 |
+
"peak_lr": 0.001
|
| 63 |
},
|
| 64 |
{
|
| 65 |
+
"lr": 0.0,
|
| 66 |
"use_muon": false,
|
| 67 |
"momentum": 0.95,
|
| 68 |
"ns_steps": 5,
|
| 69 |
"betas": [
|
| 70 |
0.9,
|
| 71 |
+
0.95
|
| 72 |
],
|
| 73 |
"eps": 1e-08,
|
| 74 |
"weight_decay": 0.01,
|
| 75 |
+
"ns_eps": 1e-07,
|
| 76 |
+
"attention_layout": "full_projection",
|
| 77 |
+
"peak_lr": 0.0001
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"lr": 0.0,
|
| 81 |
+
"use_muon": false,
|
| 82 |
+
"weight_decay": 0.0,
|
| 83 |
+
"momentum": 0.95,
|
| 84 |
+
"ns_steps": 5,
|
| 85 |
+
"betas": [
|
| 86 |
+
0.9,
|
| 87 |
+
0.95
|
| 88 |
+
],
|
| 89 |
+
"eps": 1e-08,
|
| 90 |
+
"ns_eps": 1e-07,
|
| 91 |
+
"attention_layout": "full_projection",
|
| 92 |
+
"peak_lr": 0.0001
|
| 93 |
}
|
| 94 |
],
|
| 95 |
"data": {
|
| 96 |
+
"dataset_fingerprint": "abc9253b0d88d598",
|
| 97 |
"split_method": "90/5/5 by source doc_id before @, then seeded row subsampling",
|
| 98 |
"full_split_documents": {
|
| 99 |
"train": 59688,
|
|
|
|
| 15540 |
"validation_target_tokens": 720136,
|
| 15541 |
"test_target_tokens": 708110
|
| 15542 |
},
|
| 15543 |
+
"learning_rate_schedule": "cosine",
|
| 15544 |
"evaluation_interval": "0.1 of selected training budget",
|
| 15545 |
+
"precision_changes": [],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15546 |
"bleu_definition": "Greedy continuation after prompt_tokens; reference is the next max_new_tokens tokens."
|
| 15547 |
}
|
muon/training_steps.csv
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
source/part2/README.md
CHANGED
|
@@ -1,252 +1,73 @@
|
|
| 1 |
-
# Part 2
|
| 2 |
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
| `train_mars.py` | Variance-reduced AdamW | Approximate MARS-AdamW |
|
| 12 |
-
| `train_lion.py` | Memory-efficient | Lion |
|
| 13 |
-
| `train_muon.py` | Matrix-based | Muon plus custom AdamW for other parameters |
|
| 14 |
|
| 15 |
-
|
| 16 |
-
[Fantastic Pretraining Optimizers and Where to Find Them, v2](https://arxiv.org/pdf/2509.02046v2),
|
| 17 |
-
Appendix A, pages 23-26. The updates are implemented explicitly, and each optimizer
|
| 18 |
-
inherits only from `torch.optim.Optimizer`; no predefined optimizer is used.
|
| 19 |
-
|
| 20 |
-
Two apparent errors in the appendix are corrected in the implementation:
|
| 21 |
-
|
| 22 |
-
- Gradient clipping uses `min(1, limit / norm)`, not the printed `max`, which would
|
| 23 |
-
enlarge small gradients and leave large gradients unclipped. AdamW, Lion, and
|
| 24 |
-
Muon receive globally clipped gradients from the training loop. MARS receives
|
| 25 |
-
raw gradients, applies its variance correction, and then globally clips the
|
| 26 |
-
corrected gradients, following Algorithm 5's order. Its previous gradient starts
|
| 27 |
-
at zero and is saved before clipping for the next update.
|
| 28 |
-
- Muon's Newton-Schulz iteration uses `B = -4.7750 A + 2.0315 A^2` followed by
|
| 29 |
-
`X = 3.4445 X + B X`, matching the
|
| 30 |
-
[original Muon implementation](https://github.com/KellerJordan/Muon/blob/master/muon.py).
|
| 31 |
-
The polynomial printed in Algorithm 8 is not that iteration. Momentum and the
|
| 32 |
-
rectangular-matrix scale follow Algorithm 8. Each 3D attention tensor is converted
|
| 33 |
-
from `[heads, input, output_per_head]` to the full `[output, input]` projection
|
| 34 |
-
before orthogonalization, then converted back. Q, K, and V remain separate.
|
| 35 |
-
The default 256-dimensional attention projections are square, with scale 1;
|
| 36 |
-
the earlier per-head implementation applied scale sqrt(8). Newton-Schulz uses FP32 during
|
| 37 |
-
training and a separate `--ns-epsilon` (default `1e-7`).
|
| 38 |
-
|
| 39 |
-
AdamW and MARS use bias-corrected first/second moments and decoupled weight decay.
|
| 40 |
-
Lion uses the old momentum for its sign update before updating the stored momentum.
|
| 41 |
-
Muon uses the same custom AdamW update helper for embeddings, the output head,
|
| 42 |
-
LayerNorm parameters, and other biases. All persistent optimizer state is saved
|
| 43 |
-
in `state_dict()` for checkpoint resume. This is approximate MARS-AdamW with the
|
| 44 |
-
preceding batch's gradient, not an extra backward pass on the current batch.
|
| 45 |
-
|
| 46 |
-
## Data and evaluation
|
| 47 |
-
|
| 48 |
-
Defaults use 25% of the training documents. Source documents are divided into
|
| 49 |
-
90% training, 5% validation, and 5% test before subsampling. All author versions
|
| 50 |
-
sharing the same `doc_id` prefix before `@` stay together, avoiding leakage between
|
| 51 |
-
parallel versions. A fixed seed selects and orders the same text for every run.
|
| 52 |
-
Every selected document is used in full, with BOS/EOS markers, and packed into
|
| 53 |
-
256-token sequences. Only the last partial sequence is padded.
|
| 54 |
-
|
| 55 |
-
Evaluation limits are also scaled by 0.25: the default 1,000 validation documents
|
| 56 |
-
and 1,000 test documents become 250 each, or fewer if a split is smaller.
|
| 57 |
-
Evaluation batches default to 64 for both loss scoring and BLEU generation;
|
| 58 |
-
use `--eval-batch-size` to change this.
|
| 59 |
-
All selected validation/test tokens are scored for loss. BLEU uses the same fixed
|
| 60 |
-
test documents every time: the first 128 tokens are the prompt, and the following
|
| 61 |
-
64 tokens are the reference continuation. Predictions use greedy decoding for up
|
| 62 |
-
to 64 tokens, stopping at EOS. Prompt text is excluded from BLEU. This is a
|
| 63 |
-
continuation BLEU definition for next-token pretraining, not translation BLEU.
|
| 64 |
-
|
| 65 |
-
There is an initial evaluation, then one at each 10% of the selected training token
|
| 66 |
-
budget. Final batches are shortened to hit those token boundaries exactly. The
|
| 67 |
-
logs distinguish progress through the selected subset from progress relative to
|
| 68 |
-
the full training split. Selecting 25% of documents may yield slightly more or
|
| 69 |
-
less than 25% of tokens because document lengths differ.
|
| 70 |
-
|
| 71 |
-
## Run
|
| 72 |
-
|
| 73 |
-
From `2022101025_assignment2`, prepare the shared cache once:
|
| 74 |
-
|
| 75 |
-
```bash
|
| 76 |
-
python -m part2.train_adamw --prepare-only
|
| 77 |
-
```
|
| 78 |
-
|
| 79 |
-
Run two experiments on Kaggle's two GPUs:
|
| 80 |
-
|
| 81 |
-
```bash
|
| 82 |
-
python -m part2.train_adamw --device cuda:0 > part2/adamw.log 2>&1 &
|
| 83 |
-
python -m part2.train_mars --device cuda:1 > part2/mars.log 2>&1 &
|
| 84 |
-
wait
|
| 85 |
-
python -m part2.train_lion --device cuda:0 > part2/lion.log 2>&1 &
|
| 86 |
-
python -m part2.train_muon --device cuda:1 > part2/muon.log 2>&1 &
|
| 87 |
-
wait
|
| 88 |
-
```
|
| 89 |
-
|
| 90 |
-
Direct script execution works too: `python part2/train_adamw.py --help`.
|
| 91 |
-
FP16 mixed precision and gradient scaling are enabled on CUDA for AdamW, Lion,
|
| 92 |
-
and MARS. Muon defaults to FP32 and matrix learning rate 0.005. CPU/MPS use FP32.
|
| 93 |
-
Use `--no-mixed-precision` to disable it. The learning rate is constant; defaults
|
| 94 |
-
are starting values rather than tuned optimizer comparisons. Muon has separate
|
| 95 |
-
learning rates for matrix parameters and its auxiliary custom AdamW group.
|
| 96 |
-
|
| 97 |
-
Resume an interrupted run with the same settings:
|
| 98 |
|
| 99 |
```bash
|
| 100 |
-
|
| 101 |
```
|
| 102 |
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
|
| 115 |
```bash
|
| 116 |
-
|
| 117 |
-
|
| 118 |
```
|
| 119 |
|
| 120 |
-
|
| 121 |
-
used per-head attention updates. It rejects those optimizer states rather than
|
| 122 |
-
silently changing the update rule halfway through training. Keep the old result
|
| 123 |
-
and run the retry in a separate folder:
|
| 124 |
|
| 125 |
```bash
|
| 126 |
-
python -
|
| 127 |
-
--eval-batch-size 64 --learning-rate 0.005 --no-mixed-precision \
|
| 128 |
-
--output-dir part2/output/muon_retry
|
| 129 |
-
python -m part2.plot_comparison --muon-dir part2/output/muon_retry
|
| 130 |
```
|
| 131 |
|
| 132 |
-
|
| 133 |
-
includes both Muon runs. Inspect validation loss and saved predictions before
|
| 134 |
-
claiming an improvement; changing the layout, precision, and learning rate together
|
| 135 |
-
does not establish which change caused a difference. For uploads, add
|
| 136 |
-
`--muon-dir PATH/TO/muon_retry` to `upload_part2_results.py` commands. The helper
|
| 137 |
-
then uploads the retry as Muon, with its actual settings, while retaining the old
|
| 138 |
-
local run. The new W&B run is named `muon_retry`; the other runs can remain in their
|
| 139 |
-
existing project. Use a new Hugging Face repository if you already uploaded the
|
| 140 |
-
earlier comparison and want to keep both uploaded comparisons separately.
|
| 141 |
-
|
| 142 |
-
## Recorded outputs
|
| 143 |
-
|
| 144 |
-
Each optimizer has its own directory under `part2/output`:
|
| 145 |
-
|
| 146 |
-
- `training_steps.csv`: batch loss, target tokens seen, batch token count,
|
| 147 |
-
gradient norm, skipped updates, and step timing.
|
| 148 |
-
- `metrics.csv` and `metrics.json`: interval and cumulative training loss,
|
| 149 |
-
validation loss, test BLEU, train/validation/test perplexity, test loss,
|
| 150 |
-
token counts, selected/full dataset fractions, learning rates, successful/skipped
|
| 151 |
-
updates, training/evaluation times, throughput, BLEU signature, and sample count.
|
| 152 |
-
- `predictions_*.jsonl`: test document IDs, prompts, references, and generated text
|
| 153 |
-
for each evaluation point.
|
| 154 |
-
- `settings.json`: all hyperparameters, model size, data fingerprint, split counts,
|
| 155 |
-
selected row indices, and evaluation definitions. Separate output directories
|
| 156 |
-
preserve separate hyperparameter runs.
|
| 157 |
-
- `loss_vs_tokens.png` and `bleu_vs_tokens.png`: plots from recorded metrics.
|
| 158 |
-
- `latest.pt`, `final.pt`, and `tokenizer/`: resumable/latest and final artifacts.
|
| 159 |
-
|
| 160 |
-
After training, make the cross-optimizer plots:
|
| 161 |
|
| 162 |
```bash
|
| 163 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
```
|
| 165 |
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
```
|
| 171 |
-
|
| 172 |
-
For a full-data run, use `--data-fraction 1 --evaluation-fraction 1` and new cache
|
| 173 |
-
and output directories. The current 25% setting follows the requested reduced
|
| 174 |
-
experiment; it does not claim to reach the assignment's 1x full-dataset target.
|
| 175 |
-
|
| 176 |
-
## Upload finished runs
|
| 177 |
-
|
| 178 |
-
The standalone `upload_part2_results.py` in the workspace root uploads saved
|
| 179 |
-
results without repeating training. On this laptop it automatically reads
|
| 180 |
-
`part2/part2_results/output`, independent of the shell's current directory.
|
| 181 |
-
It checks that each selected optimizer reached its recorded training budget.
|
| 182 |
-
|
| 183 |
-
From the workspace root (`Assignment2`):
|
| 184 |
-
|
| 185 |
-
```bash
|
| 186 |
-
source .venv/bin/activate
|
| 187 |
-
python -m pip install --upgrade wandb huggingface_hub
|
| 188 |
-
|
| 189 |
-
# Validate locally; these commands do not log in or upload anything.
|
| 190 |
-
python upload_part2_results.py wandb --dry-run
|
| 191 |
-
python upload_part2_results.py huggingface --dry-run
|
| 192 |
-
|
| 193 |
-
wandb login
|
| 194 |
-
hf auth login
|
| 195 |
-
|
| 196 |
-
python upload_part2_results.py wandb \
|
| 197 |
-
--project anlp-assignment2-part2 --entity YOUR_WANDB_USER_OR_TEAM
|
| 198 |
-
python upload_part2_results.py huggingface \
|
| 199 |
-
--repo-id YOUR_HF_USERNAME/anlp-assignment2-part2
|
| 200 |
-
```
|
| 201 |
-
|
| 202 |
-
Replace the account placeholders. The Hugging Face token needs write access.
|
| 203 |
-
Alternatively, set `WANDB_API_KEY` and `HF_TOKEN` in your environment.
|
| 204 |
-
|
| 205 |
-
W&B receives one run per optimizer, with batch loss, evaluation metrics,
|
| 206 |
-
perplexity, BLEU, gradient norms, learning rates, throughput, and recorded timing
|
| 207 |
-
plotted against training tokens. Raw CSV/JSON files, all prediction files,
|
| 208 |
-
training logs, and plots are uploaded too. Saved plots are also displayed as
|
| 209 |
-
images. Run IDs in each optimizer's `.wandb_upload.json` allow retries to resume
|
| 210 |
-
without logging the same metric events again. Keep these files when retrying.
|
| 211 |
-
If metrics or settings change after uploading, the script rejects resuming that
|
| 212 |
-
run; use a fresh results directory for a different experiment. Historical GPU
|
| 213 |
-
utilization was not recorded, and upload-time system monitoring is disabled.
|
| 214 |
-
|
| 215 |
-
Hugging Face receives a single model repository with an optimizer folder for
|
| 216 |
-
AdamW, MARS, Lion, and Muon. By default it includes both `final.pt` and `latest.pt`,
|
| 217 |
-
plus tokenizers, settings, metrics, predictions, logs, plots, and custom model
|
| 218 |
-
source. The generated model card provides a loading example; these are custom
|
| 219 |
-
PyTorch checkpoints rather than Transformers AutoModel artifacts. New repositories
|
| 220 |
-
are private by default; add `--public` to create a public repository. This flag
|
| 221 |
-
does not change the visibility of an already existing repository. Add
|
| 222 |
-
`--final-only` to omit other checkpoints. Re-running an upload updates the same
|
| 223 |
-
Hugging Face repository.
|
| 224 |
-
|
| 225 |
-
To upload only selected optimizers, add e.g. `--optimizers adamw muon` to either
|
| 226 |
-
command. To use another results directory, supply `--output-dir PATH` and, if
|
| 227 |
-
needed, `--source-dir PATH`. For Kaggle, use explicit paths:
|
| 228 |
-
|
| 229 |
-
```bash
|
| 230 |
-
python /kaggle/working/upload_part2_results.py wandb \
|
| 231 |
-
--output-dir /kaggle/working/part2/output \
|
| 232 |
-
--project anlp-assignment2-part2 --entity YOUR_WANDB_USER_OR_TEAM
|
| 233 |
-
python /kaggle/working/upload_part2_results.py huggingface \
|
| 234 |
-
--output-dir /kaggle/working/part2/output \
|
| 235 |
-
--source-dir /kaggle/working/part2 \
|
| 236 |
-
--repo-id YOUR_HF_USERNAME/anlp-assignment2-part2
|
| 237 |
-
```
|
| 238 |
-
|
| 239 |
-
Generate comparison plots if they are missing, from `2022101025_assignment2`:
|
| 240 |
-
|
| 241 |
-
```bash
|
| 242 |
-
python -m part2.plot_comparison --output-dir part2/part2_results/output
|
| 243 |
-
```
|
| 244 |
|
| 245 |
-
|
| 246 |
|
| 247 |
```bash
|
| 248 |
-
python
|
|
|
|
| 249 |
```
|
| 250 |
|
| 251 |
-
|
| 252 |
-
|
|
|
|
|
|
| 1 |
+
# Part 2 optimizer training
|
| 2 |
|
| 3 |
+
Muon uses peak learning rates of **0.001** for hidden matrices and **0.0001**
|
| 4 |
+
for auxiliary AdamW. Both rates warm up over 5% of training tokens, then follow
|
| 5 |
+
cosine decay to 10% of their peaks. Training uses FP32, momentum 0.95, five
|
| 6 |
+
Newton–Schulz iterations, gradient clipping at 1.0 and auxiliary AdamW betas
|
| 7 |
+
(0.9, 0.95). Attention matrices use full projection updates. Weight decay is
|
| 8 |
+
0.01 on matrices and embeddings/head weights; biases and normalization gains
|
| 9 |
+
have zero decay. These settings are a conservative training configuration;
|
| 10 |
+
their performance must be measured on held-out data.
|
| 11 |
|
| 12 |
+
The dense architecture, seed, token order, 25% data selection, and eleven
|
| 13 |
+
evaluation points match the optimizer comparison. Batch size is 64. Learning
|
| 14 |
+
rates depend on tokens consumed, so resuming preserves the schedule.
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
+
From the Assignment2 directory, run:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
|
| 18 |
```bash
|
| 19 |
+
bash run_muon_local.sh
|
| 20 |
```
|
| 21 |
|
| 22 |
+
The script uses the project's `.venv`, selects MPS/CUDA when available, and
|
| 23 |
+
otherwise uses CPU. It downloads/caches the data on first use, writes the single
|
| 24 |
+
Muon run to `2022101025_assignment2/part2/part2_results/output/muon`, and logs to
|
| 25 |
+
`2022101025_assignment2/part2/part2_results/muon.log`. The initial evaluation runs
|
| 26 |
+
before the training progress bar. When training completes, the script refreshes
|
| 27 |
+
the comparison figures and the Muon row in `report.md` using measured results.
|
| 28 |
+
The launcher then uploads Muon metrics/plots to W&B project
|
| 29 |
+
`anlp-assignment2-part2` and checkpoints/source/results to your Hugging Face
|
| 30 |
+
repository `USERNAME/anlp-assignment2-part2`, and adds the returned links to the
|
| 31 |
+
report. The repository defaults to private when first created. Both services
|
| 32 |
+
use your saved login or `WANDB_API_KEY` / `HF_TOKEN` environment variables.
|
| 33 |
+
Run the same command after an interruption to resume `latest.pt`, or after
|
| 34 |
+
an upload error to upload the completed run.
|
| 35 |
+
|
| 36 |
+
If you are not already logged in, run these once from Assignment2:
|
| 37 |
|
| 38 |
```bash
|
| 39 |
+
.venv/bin/wandb login
|
| 40 |
+
.venv/bin/hf auth login
|
| 41 |
```
|
| 42 |
|
| 43 |
+
To install compatible dependencies in a fresh environment:
|
|
|
|
|
|
|
|
|
|
| 44 |
|
| 45 |
```bash
|
| 46 |
+
.venv/bin/python -m pip install -r 2022101025_assignment2/part2/requirements.txt
|
|
|
|
|
|
|
|
|
|
| 47 |
```
|
| 48 |
|
| 49 |
+
To invoke the trainer directly with its default parameters:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
```bash
|
| 52 |
+
cd 2022101025_assignment2
|
| 53 |
+
../.venv/bin/python -u -m part2.train_muon --device auto
|
| 54 |
+
# Resume using identical training arguments:
|
| 55 |
+
../.venv/bin/python -u -m part2.train_muon --device auto \
|
| 56 |
+
--resume part2/part2_results/output/muon/latest.pt
|
| 57 |
```
|
| 58 |
|
| 59 |
+
For a quick separate run, cap training with `--max-train-tokens` and supply a
|
| 60 |
+
temporary `--output-dir` directly to the trainer. A capped run is not a final
|
| 61 |
+
optimizer-comparison result. The full experiment trains on 42,733,510 target
|
| 62 |
+
tokens and scores 250 validation and 250 test documents.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
|
| 64 |
+
To validate the uploads separately, the helper uses the same `muon` folder:
|
| 65 |
|
| 66 |
```bash
|
| 67 |
+
.venv/bin/python upload_part2_results.py wandb --optimizers muon --dry-run
|
| 68 |
+
.venv/bin/python upload_part2_results.py huggingface --optimizers muon --dry-run
|
| 69 |
```
|
| 70 |
|
| 71 |
+
Remove `--dry-run` when you want to upload the completed results.
|
| 72 |
+
Muon parameter grouping and momentum defaults follow the
|
| 73 |
+
[Muon reference implementation](https://github.com/KellerJordan/Muon).
|
source/part2/plot_comparison.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
"""Compare the four runs using recorded training token counts."""
|
| 2 |
|
| 3 |
import argparse
|
|
|
|
| 4 |
import json
|
| 5 |
from pathlib import Path
|
| 6 |
|
|
@@ -11,29 +12,69 @@ import matplotlib.pyplot as plt
|
|
| 11 |
|
| 12 |
def main():
|
| 13 |
parser = argparse.ArgumentParser()
|
| 14 |
-
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
| 16 |
args = parser.parse_args()
|
| 17 |
runs = {}
|
|
|
|
| 18 |
for name in ("adamw", "mars", "lion", "muon"):
|
| 19 |
-
directory = args.
|
| 20 |
path = directory / "metrics.json"
|
| 21 |
if path.exists():
|
| 22 |
runs[name] = json.loads(path.read_text())
|
| 23 |
-
|
| 24 |
-
original = args.output_dir / "muon" / "metrics.json"
|
| 25 |
-
if original.exists():
|
| 26 |
-
runs["muon (original)"] = json.loads(original.read_text())
|
| 27 |
if not runs:
|
| 28 |
raise FileNotFoundError("No metrics.json files found under the output directory.")
|
| 29 |
-
|
| 30 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
for name, history in runs.items():
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
axis.legend()
|
| 36 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
plt.close(figure)
|
| 38 |
|
| 39 |
|
|
|
|
| 1 |
"""Compare the four runs using recorded training token counts."""
|
| 2 |
|
| 3 |
import argparse
|
| 4 |
+
import csv
|
| 5 |
import json
|
| 6 |
from pathlib import Path
|
| 7 |
|
|
|
|
| 12 |
|
| 13 |
def main():
|
| 14 |
parser = argparse.ArgumentParser()
|
| 15 |
+
here = Path(__file__).resolve().parent
|
| 16 |
+
output = here / "part2_results" / "output"
|
| 17 |
+
parser.add_argument("--output-dir", type=Path, default=output if output.is_dir() else here / "output")
|
| 18 |
+
parser.add_argument("--x-axis", choices=("tokens", "dataset"), default="tokens",
|
| 19 |
+
help="dataset uses one pass over the selected training subset as 1x.")
|
| 20 |
args = parser.parse_args()
|
| 21 |
runs = {}
|
| 22 |
+
directories = {}
|
| 23 |
for name in ("adamw", "mars", "lion", "muon"):
|
| 24 |
+
directory = args.output_dir / name
|
| 25 |
path = directory / "metrics.json"
|
| 26 |
if path.exists():
|
| 27 |
runs[name] = json.loads(path.read_text())
|
| 28 |
+
directories[name] = directory
|
|
|
|
|
|
|
|
|
|
| 29 |
if not runs:
|
| 30 |
raise FileNotFoundError("No metrics.json files found under the output directory.")
|
| 31 |
+
dataset_sizes = {}
|
| 32 |
+
if args.x_axis == "dataset":
|
| 33 |
+
records = []
|
| 34 |
+
for name, history in runs.items():
|
| 35 |
+
settings = json.loads((directories[name] / "settings.json").read_text())
|
| 36 |
+
size = settings["data"]["train_target_tokens"]
|
| 37 |
+
if size <= 0:
|
| 38 |
+
raise ValueError(f"{name}: selected training dataset must contain target tokens.")
|
| 39 |
+
dataset_sizes[name] = size
|
| 40 |
+
for row in history:
|
| 41 |
+
records.append({**row, "optimizer": name, "experiment_dataset_tokens": size,
|
| 42 |
+
"experiment_dataset_passes": row["train_tokens"] / size,
|
| 43 |
+
"original_train_split_tokens": settings["data"]["full_train_target_tokens"]})
|
| 44 |
+
print(f"{name}: 1x dataset = {size:,} selected target tokens; "
|
| 45 |
+
f"final coverage = {history[-1]['train_tokens'] / size:.5f}x")
|
| 46 |
+
fields = list(dict.fromkeys(key for row in records for key in row))
|
| 47 |
+
with (args.output_dir / "comparison_dataset_metrics.csv").open("w", newline="") as stream:
|
| 48 |
+
writer = csv.DictWriter(stream, fieldnames=fields)
|
| 49 |
+
writer.writeheader()
|
| 50 |
+
writer.writerows(records)
|
| 51 |
+
|
| 52 |
+
def draw(axis, metric, label):
|
| 53 |
for name, history in runs.items():
|
| 54 |
+
denominator = dataset_sizes[name] if args.x_axis == "dataset" else 1
|
| 55 |
+
axis.plot([row["train_tokens"] / denominator for row in history],
|
| 56 |
+
[row[metric] for row in history], marker="o", label=name)
|
| 57 |
+
if args.x_axis == "dataset":
|
| 58 |
+
axis.set_xlabel("Training tokens / selected training dataset tokens (passes)")
|
| 59 |
+
axis.set_xticks([i / 10 for i in range(11)])
|
| 60 |
+
axis.set_title("Optimizer comparison: selected subset is the experiment dataset")
|
| 61 |
+
else:
|
| 62 |
+
axis.set(xlabel="Training target tokens seen", title="Optimizer comparison")
|
| 63 |
+
axis.set_ylabel(label)
|
| 64 |
+
axis.grid(alpha=0.25)
|
| 65 |
axis.legend()
|
| 66 |
+
|
| 67 |
+
for metric, label in (("validation_loss", "Validation cross entropy"), ("test_bleu", "Test BLEU (0-100)")):
|
| 68 |
+
figure, axis = plt.subplots(figsize=(8, 5), layout="constrained")
|
| 69 |
+
draw(axis, metric, label)
|
| 70 |
+
suffix = "_vs_dataset" if args.x_axis == "dataset" else ""
|
| 71 |
+
figure.savefig(args.output_dir / f"comparison_{metric}{suffix}.png", dpi=180)
|
| 72 |
+
plt.close(figure)
|
| 73 |
+
if args.x_axis == "dataset":
|
| 74 |
+
figure, axes = plt.subplots(1, 2, figsize=(14, 5), layout="constrained")
|
| 75 |
+
draw(axes[0], "validation_loss", "Validation cross entropy")
|
| 76 |
+
draw(axes[1], "test_bleu", "Test BLEU (0-100)")
|
| 77 |
+
figure.savefig(args.output_dir / "comparison_dataset_dynamics.png", dpi=180)
|
| 78 |
plt.close(figure)
|
| 79 |
|
| 80 |
|
source/part2/requirements.txt
CHANGED
|
@@ -1,6 +1,8 @@
|
|
| 1 |
torch>=2.3
|
| 2 |
datasets>=3
|
| 3 |
-
transformers>=
|
|
|
|
| 4 |
sacrebleu>=2.4
|
| 5 |
matplotlib>=3.8
|
| 6 |
tqdm>=4.66
|
|
|
|
|
|
| 1 |
torch>=2.3
|
| 2 |
datasets>=3
|
| 3 |
+
transformers>=5.17,<6
|
| 4 |
+
huggingface-hub>=1.5.0,<2.0
|
| 5 |
sacrebleu>=2.4
|
| 6 |
matplotlib>=3.8
|
| 7 |
tqdm>=4.66
|
| 8 |
+
wandb>=0.19
|
source/part2/train.py
CHANGED
|
@@ -45,6 +45,11 @@ def get_arguments(name):
|
|
| 45 |
parser.add_argument("--ns-epsilon", type=float, default=1e-7, help="Muon matrix normalization epsilon.")
|
| 46 |
parser.add_argument("--aux-learning-rate", type=float, default=3e-4,
|
| 47 |
help="Muon's custom AdamW learning rate for embeddings/head/biases.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
parser.add_argument("--max-grad-norm", type=float, default=1.0)
|
| 49 |
parser.add_argument("--prompt-tokens", type=int, default=128)
|
| 50 |
parser.add_argument("--max-new-tokens", type=int, default=64)
|
|
@@ -62,7 +67,11 @@ def get_arguments(name):
|
|
| 62 |
if name == "lion":
|
| 63 |
parser.set_defaults(learning_rate=1e-4, beta2=0.99)
|
| 64 |
elif name == "muon":
|
| 65 |
-
parser.set_defaults(learning_rate=
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
args = parser.parse_args()
|
| 67 |
if args.evaluation_fraction is None:
|
| 68 |
args.evaluation_fraction = args.data_fraction
|
|
@@ -79,9 +88,26 @@ def get_arguments(name):
|
|
| 79 |
parser.error("Token caps must be nonnegative and learning rates positive.")
|
| 80 |
if args.max_grad_norm <= 0 or args.weight_decay < 0:
|
| 81 |
parser.error("Use positive max-grad-norm and nonnegative weight-decay.")
|
|
|
|
|
|
|
| 82 |
return args
|
| 83 |
|
| 84 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
def get_device(name):
|
| 86 |
if name == "auto":
|
| 87 |
if torch.cuda.is_available():
|
|
@@ -226,8 +252,10 @@ def run_training(name, make_optimizer, args):
|
|
| 226 |
raise ValueError("Resume checkpoint has a different optimizer or dataset selection.")
|
| 227 |
ignored_args = {"device", "output_dir", "resume", "prepare_only", "mixed_precision"}
|
| 228 |
current_args = {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}
|
| 229 |
-
|
| 230 |
-
|
|
|
|
|
|
|
| 231 |
raise ValueError(f"Resume argument changed: {key}.")
|
| 232 |
model.load_state_dict(checkpoint["model_state_dict"])
|
| 233 |
optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
|
|
@@ -242,6 +270,7 @@ def run_training(name, make_optimizer, args):
|
|
| 242 |
})
|
| 243 |
history = checkpoint["history"]
|
| 244 |
|
|
|
|
| 245 |
tokenizer.save_pretrained(directory / "tokenizer")
|
| 246 |
groups = [{key: value for key, value in group.items() if key != "params"}
|
| 247 |
for group in optimizer.param_groups]
|
|
@@ -249,7 +278,8 @@ def run_training(name, make_optimizer, args):
|
|
| 249 |
for key, value in vars(args).items()},
|
| 250 |
"model": asdict(config), "parameters": state["parameters"],
|
| 251 |
"optimizer_groups": groups, "data": prepared["metadata"],
|
| 252 |
-
"learning_rate_schedule":
|
|
|
|
| 253 |
"precision_changes": state.get("precision_changes", []),
|
| 254 |
"bleu_definition": "Greedy continuation after prompt_tokens; reference is the next max_new_tokens tokens."}
|
| 255 |
(directory / "settings.json").write_text(json.dumps(settings, indent=2))
|
|
@@ -264,7 +294,8 @@ def run_training(name, make_optimizer, args):
|
|
| 264 |
if args.resume and log_path.exists():
|
| 265 |
with log_path.open() as stream:
|
| 266 |
previous_rows = [row for row in csv.DictReader(stream) if int(row["step"]) <= state["step"]]
|
| 267 |
-
|
|
|
|
| 268 |
boundaries = sorted(set(math.ceil(budget * fraction / 10) for fraction in range(1, 11)))
|
| 269 |
progress = tqdm(total=budget, initial=state["position"], desc=name, unit="tok")
|
| 270 |
with log_path.open("w", newline="") as stream:
|
|
@@ -303,6 +334,7 @@ def run_training(name, make_optimizer, args):
|
|
| 303 |
if not math.isfinite(gradient_norm) and not scaler.is_enabled():
|
| 304 |
raise RuntimeError("Gradient norm is not finite; the last evaluation checkpoint is saved.")
|
| 305 |
previous_scale = scaler.get_scale()
|
|
|
|
| 306 |
scaler.step(optimizer)
|
| 307 |
scaler.update()
|
| 308 |
skipped = scaler.get_scale() < previous_scale
|
|
@@ -319,7 +351,8 @@ def run_training(name, make_optimizer, args):
|
|
| 319 |
writer.writerow({"step": state["step"], "train_tokens": state["position"],
|
| 320 |
"batch_tokens": count, "loss": loss_value,
|
| 321 |
"gradient_norm": state["last_gradient_norm"], "update_skipped": skipped,
|
| 322 |
-
"step_seconds": seconds
|
|
|
|
| 323 |
progress.update(count)
|
| 324 |
progress.set_postfix(loss=f"{loss_value:.4f}")
|
| 325 |
stream.flush()
|
|
|
|
| 45 |
parser.add_argument("--ns-epsilon", type=float, default=1e-7, help="Muon matrix normalization epsilon.")
|
| 46 |
parser.add_argument("--aux-learning-rate", type=float, default=3e-4,
|
| 47 |
help="Muon's custom AdamW learning rate for embeddings/head/biases.")
|
| 48 |
+
parser.add_argument("--lr-schedule", choices=("constant", "cosine"), default="constant")
|
| 49 |
+
parser.add_argument("--warmup-fraction", type=float, default=0.05,
|
| 50 |
+
help="Fraction of training tokens used for linear warmup with cosine scheduling.")
|
| 51 |
+
parser.add_argument("--min-lr-ratio", type=float, default=0.1,
|
| 52 |
+
help="Final learning rate / peak learning rate for cosine scheduling.")
|
| 53 |
parser.add_argument("--max-grad-norm", type=float, default=1.0)
|
| 54 |
parser.add_argument("--prompt-tokens", type=int, default=128)
|
| 55 |
parser.add_argument("--max-new-tokens", type=int, default=64)
|
|
|
|
| 67 |
if name == "lion":
|
| 68 |
parser.set_defaults(learning_rate=1e-4, beta2=0.99)
|
| 69 |
elif name == "muon":
|
| 70 |
+
parser.set_defaults(learning_rate=1e-3, aux_learning_rate=1e-4,
|
| 71 |
+
beta2=0.95, lr_schedule="cosine", mixed_precision=False,
|
| 72 |
+
batch_size=64,
|
| 73 |
+
output_dir=str(HERE / "part2_results" / "output" / name)
|
| 74 |
+
if (HERE / "part2_results").is_dir() else str(HERE / "output" / name))
|
| 75 |
args = parser.parse_args()
|
| 76 |
if args.evaluation_fraction is None:
|
| 77 |
args.evaluation_fraction = args.data_fraction
|
|
|
|
| 88 |
parser.error("Token caps must be nonnegative and learning rates positive.")
|
| 89 |
if args.max_grad_norm <= 0 or args.weight_decay < 0:
|
| 90 |
parser.error("Use positive max-grad-norm and nonnegative weight-decay.")
|
| 91 |
+
if not 0 <= args.warmup_fraction < 1 or not 0 < args.min_lr_ratio <= 1:
|
| 92 |
+
parser.error("Use warmup-fraction in [0, 1) and min-lr-ratio in (0, 1].")
|
| 93 |
return args
|
| 94 |
|
| 95 |
|
| 96 |
+
def set_learning_rates(optimizer, args, position, budget):
|
| 97 |
+
"""Schedule each group's peak rate by tokens, including after a checkpoint resume."""
|
| 98 |
+
if args.lr_schedule == "constant":
|
| 99 |
+
return
|
| 100 |
+
progress = min(max(position / budget, 0.0), 1.0)
|
| 101 |
+
if args.warmup_fraction and progress < args.warmup_fraction:
|
| 102 |
+
factor = progress / args.warmup_fraction
|
| 103 |
+
else:
|
| 104 |
+
decay = (progress - args.warmup_fraction) / (1 - args.warmup_fraction)
|
| 105 |
+
factor = args.min_lr_ratio + (1 - args.min_lr_ratio) * 0.5 * (1 + math.cos(math.pi * decay))
|
| 106 |
+
for group in optimizer.param_groups:
|
| 107 |
+
group.setdefault("peak_lr", group["lr"])
|
| 108 |
+
group["lr"] = group["peak_lr"] * factor
|
| 109 |
+
|
| 110 |
+
|
| 111 |
def get_device(name):
|
| 112 |
if name == "auto":
|
| 113 |
if torch.cuda.is_available():
|
|
|
|
| 252 |
raise ValueError("Resume checkpoint has a different optimizer or dataset selection.")
|
| 253 |
ignored_args = {"device", "output_dir", "resume", "prepare_only", "mixed_precision"}
|
| 254 |
current_args = {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}
|
| 255 |
+
schedule_defaults = {"lr_schedule": "constant", "warmup_fraction": 0.05, "min_lr_ratio": 0.1}
|
| 256 |
+
for key, value in current_args.items():
|
| 257 |
+
saved_value = checkpoint["args"].get(key, schedule_defaults.get(key))
|
| 258 |
+
if key not in ignored_args and saved_value != value:
|
| 259 |
raise ValueError(f"Resume argument changed: {key}.")
|
| 260 |
model.load_state_dict(checkpoint["model_state_dict"])
|
| 261 |
optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
|
|
|
|
| 270 |
})
|
| 271 |
history = checkpoint["history"]
|
| 272 |
|
| 273 |
+
set_learning_rates(optimizer, args, state["position"], budget)
|
| 274 |
tokenizer.save_pretrained(directory / "tokenizer")
|
| 275 |
groups = [{key: value for key, value in group.items() if key != "params"}
|
| 276 |
for group in optimizer.param_groups]
|
|
|
|
| 278 |
for key, value in vars(args).items()},
|
| 279 |
"model": asdict(config), "parameters": state["parameters"],
|
| 280 |
"optimizer_groups": groups, "data": prepared["metadata"],
|
| 281 |
+
"learning_rate_schedule": args.lr_schedule,
|
| 282 |
+
"evaluation_interval": "0.1 of selected training budget",
|
| 283 |
"precision_changes": state.get("precision_changes", []),
|
| 284 |
"bleu_definition": "Greedy continuation after prompt_tokens; reference is the next max_new_tokens tokens."}
|
| 285 |
(directory / "settings.json").write_text(json.dumps(settings, indent=2))
|
|
|
|
| 294 |
if args.resume and log_path.exists():
|
| 295 |
with log_path.open() as stream:
|
| 296 |
previous_rows = [row for row in csv.DictReader(stream) if int(row["step"]) <= state["step"]]
|
| 297 |
+
rate_columns = [f"learning_rate_{index}" for index in range(len(optimizer.param_groups))]
|
| 298 |
+
columns = ["step", "train_tokens", "batch_tokens", "loss", "gradient_norm", "update_skipped", "step_seconds"] + rate_columns
|
| 299 |
boundaries = sorted(set(math.ceil(budget * fraction / 10) for fraction in range(1, 11)))
|
| 300 |
progress = tqdm(total=budget, initial=state["position"], desc=name, unit="tok")
|
| 301 |
with log_path.open("w", newline="") as stream:
|
|
|
|
| 334 |
if not math.isfinite(gradient_norm) and not scaler.is_enabled():
|
| 335 |
raise RuntimeError("Gradient norm is not finite; the last evaluation checkpoint is saved.")
|
| 336 |
previous_scale = scaler.get_scale()
|
| 337 |
+
set_learning_rates(optimizer, args, state["position"] + count, budget)
|
| 338 |
scaler.step(optimizer)
|
| 339 |
scaler.update()
|
| 340 |
skipped = scaler.get_scale() < previous_scale
|
|
|
|
| 351 |
writer.writerow({"step": state["step"], "train_tokens": state["position"],
|
| 352 |
"batch_tokens": count, "loss": loss_value,
|
| 353 |
"gradient_norm": state["last_gradient_norm"], "update_skipped": skipped,
|
| 354 |
+
"step_seconds": seconds,
|
| 355 |
+
**{column: group["lr"] for column, group in zip(rate_columns, optimizer.param_groups)}})
|
| 356 |
progress.update(count)
|
| 357 |
progress.set_postfix(loss=f"{loss_value:.4f}")
|
| 358 |
stream.flush()
|
source/part2/train_muon.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""Muon
|
| 2 |
|
| 3 |
import math
|
| 4 |
import sys
|
|
@@ -27,7 +27,7 @@ def newton_schulz(matrix, steps=5, eps=1e-7):
|
|
| 27 |
|
| 28 |
for _ in range(steps):
|
| 29 |
gram = x @ x.transpose(-2, -1)
|
| 30 |
-
#
|
| 31 |
correction = -4.7750 * gram + 2.0315 * (gram @ gram)
|
| 32 |
x = 3.4445 * x + correction @ x
|
| 33 |
|
|
@@ -59,8 +59,7 @@ class Muon(Optimizer):
|
|
| 59 |
for current, saved in zip(self.param_groups, state_dict["param_groups"]):
|
| 60 |
if current["use_muon"] and any(p.ndim == 3 for p in current["params"]):
|
| 61 |
if saved.get("attention_layout") != "full_projection":
|
| 62 |
-
raise ValueError("
|
| 63 |
-
"with the corrected attention layout.")
|
| 64 |
super().load_state_dict(state_dict)
|
| 65 |
|
| 66 |
@torch.no_grad()
|
|
@@ -105,16 +104,21 @@ class Muon(Optimizer):
|
|
| 105 |
|
| 106 |
def make_optimizer(model, args):
|
| 107 |
matrix_parameters = []
|
| 108 |
-
|
|
|
|
| 109 |
for name, parameter in model.named_parameters():
|
| 110 |
is_embedding_or_head = name.startswith("embedding.") or name.startswith("unembedding.")
|
| 111 |
if parameter.ndim >= 2 and not is_embedding_or_head:
|
| 112 |
matrix_parameters.append(parameter)
|
|
|
|
|
|
|
| 113 |
else:
|
| 114 |
-
|
| 115 |
groups = [
|
| 116 |
{"params": matrix_parameters, "lr": args.learning_rate, "use_muon": True},
|
| 117 |
-
{"params":
|
|
|
|
|
|
|
| 118 |
]
|
| 119 |
return Muon(groups, momentum=args.momentum, ns_steps=args.ns_steps,
|
| 120 |
betas=(args.beta1, args.beta2), eps=args.epsilon,
|
|
|
|
| 1 |
+
"""Muon with conservative rates, full attention projections and auxiliary AdamW."""
|
| 2 |
|
| 3 |
import math
|
| 4 |
import sys
|
|
|
|
| 27 |
|
| 28 |
for _ in range(steps):
|
| 29 |
gram = x @ x.transpose(-2, -1)
|
| 30 |
+
# Muon's quintic Newton-Schulz polynomial.
|
| 31 |
correction = -4.7750 * gram + 2.0315 * (gram @ gram)
|
| 32 |
x = 3.4445 * x + correction @ x
|
| 33 |
|
|
|
|
| 59 |
for current, saved in zip(self.param_groups, state_dict["param_groups"]):
|
| 60 |
if current["use_muon"] and any(p.ndim == 3 for p in current["params"]):
|
| 61 |
if saved.get("attention_layout") != "full_projection":
|
| 62 |
+
raise ValueError("Muon checkpoints must use the full_projection attention layout.")
|
|
|
|
| 63 |
super().load_state_dict(state_dict)
|
| 64 |
|
| 65 |
@torch.no_grad()
|
|
|
|
| 104 |
|
| 105 |
def make_optimizer(model, args):
|
| 106 |
matrix_parameters = []
|
| 107 |
+
auxiliary_weights = []
|
| 108 |
+
gains_and_biases = []
|
| 109 |
for name, parameter in model.named_parameters():
|
| 110 |
is_embedding_or_head = name.startswith("embedding.") or name.startswith("unembedding.")
|
| 111 |
if parameter.ndim >= 2 and not is_embedding_or_head:
|
| 112 |
matrix_parameters.append(parameter)
|
| 113 |
+
elif parameter.ndim >= 2:
|
| 114 |
+
auxiliary_weights.append(parameter)
|
| 115 |
else:
|
| 116 |
+
gains_and_biases.append(parameter)
|
| 117 |
groups = [
|
| 118 |
{"params": matrix_parameters, "lr": args.learning_rate, "use_muon": True},
|
| 119 |
+
{"params": auxiliary_weights, "lr": args.aux_learning_rate, "use_muon": False},
|
| 120 |
+
{"params": gains_and_biases, "lr": args.aux_learning_rate, "use_muon": False,
|
| 121 |
+
"weight_decay": 0.0},
|
| 122 |
]
|
| 123 |
return Muon(groups, momentum=args.momentum, ns_steps=args.ns_steps,
|
| 124 |
betas=(args.beta1, args.beta2), eps=args.epsilon,
|
source/part2/update_report.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fill the Muon report row from a completed full-budget run."""
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import json
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
import shutil
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def main():
|
| 10 |
+
here = Path(__file__).resolve().parent
|
| 11 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 12 |
+
parser.add_argument("--output-dir", type=Path, default=here / "part2_results" / "output")
|
| 13 |
+
args = parser.parse_args()
|
| 14 |
+
directory = args.output_dir / "muon"
|
| 15 |
+
history = json.loads((directory / "metrics.json").read_text())
|
| 16 |
+
settings = json.loads((directory / "settings.json").read_text())
|
| 17 |
+
final = history[-1]
|
| 18 |
+
if (final["train_tokens"] != settings["data"]["train_target_tokens"]
|
| 19 |
+
or not (directory / "final.pt").is_file()):
|
| 20 |
+
raise ValueError("The report requires a completed pass over the selected dataset.")
|
| 21 |
+
report = here.parent / "report.md"
|
| 22 |
+
if not report.is_file():
|
| 23 |
+
print("No report.md found; recorded results are available in", directory)
|
| 24 |
+
return
|
| 25 |
+
peak = settings["arguments"]["learning_rate"]
|
| 26 |
+
auxiliary = settings["arguments"]["aux_learning_rate"]
|
| 27 |
+
checkpoint = directory.resolve().relative_to(here.parent.resolve()) / "final.pt"
|
| 28 |
+
run_link = f"[Local checkpoint]({checkpoint.as_posix()})"
|
| 29 |
+
wandb_file = directory / ".wandb_upload.json"
|
| 30 |
+
if wandb_file.is_file():
|
| 31 |
+
uploaded = json.loads(wandb_file.read_text())
|
| 32 |
+
run_link = f"[W&B]({uploaded['url']})"
|
| 33 |
+
row = (f"| Muon | {peak:g} / {auxiliary:g} auxiliary | "
|
| 34 |
+
f"{final['validation_loss']:.4f} | {final['test_perplexity']:.3f} | "
|
| 35 |
+
f"{final['test_bleu']:.4f} | {final['training_seconds']:.1f} | "
|
| 36 |
+
f"{run_link} |")
|
| 37 |
+
text = report.read_text()
|
| 38 |
+
text = "\n".join(row if line.startswith("| Muon |") else line for line in text.splitlines()) + "\n"
|
| 39 |
+
text = text.replace("Muon results are pending completion of its configured training budget.",
|
| 40 |
+
"Muon completed its configured training budget.")
|
| 41 |
+
huggingface_file = directory / ".huggingface_upload.json"
|
| 42 |
+
if huggingface_file.is_file():
|
| 43 |
+
uploaded = json.loads(huggingface_file.read_text())
|
| 44 |
+
lines = text.splitlines()
|
| 45 |
+
for index, line in enumerate(lines):
|
| 46 |
+
if line.startswith("Checkpoints:") and "[AdamW]" in line:
|
| 47 |
+
# Regenerate only Muon's checkpoint link; preserve the other optimizers.
|
| 48 |
+
line = line.split(", [Muon]")[0].split(". Each folder")[0]
|
| 49 |
+
lines[index] = line.rstrip(". ") + f", [Muon]({uploaded['url']}). Each folder has `final.pt` and `latest.pt`."
|
| 50 |
+
break
|
| 51 |
+
text = "\n".join(lines) + "\n"
|
| 52 |
+
report.write_text(text)
|
| 53 |
+
shutil.copy2(args.output_dir / "comparison_dataset_dynamics.png", here.parent / "assets/part2_dynamics.png")
|
| 54 |
+
print(f"Updated {report} from {final['train_tokens']:,} Muon training tokens.")
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
if __name__ == "__main__":
|
| 58 |
+
main()
|