diff --git a/060_sparse_mol/top2of7_best.pt b/060_sparse_mol/top2of7_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..3006f614af3397380af9c6d4778fc35e3caad3c8 --- /dev/null +++ b/060_sparse_mol/top2of7_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73b649130aad500103abf20c72887804b9f6ae4ada051bc6ebbaee0c63f2b415 +size 343197825 diff --git a/060_sparse_mol/top2of7_results.json b/060_sparse_mol/top2of7_results.json new file mode 100644 index 0000000000000000000000000000000000000000..9e7e86c83b875deec0c2ac62f9315287c0a95c98 --- /dev/null +++ b/060_sparse_mol/top2of7_results.json @@ -0,0 +1,2057 @@ +{ + "label": "top2of7", + "top_k_blocks": 2, + "num_thin_blocks": 7, + "total_params": 85676160, + "router_params": 57344, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 14.68391227722168, + 12.598575592041016, + 7.815629959106445, + 7.190178394317627, + 7.055052280426025, + 6.9140472412109375, + 6.828429222106934, + 6.668107509613037, + 6.575447082519531, + 6.337263107299805, + 6.348662853240967, + 6.243175029754639, + 6.200726509094238, + 6.031182765960693, + 6.1005048751831055, + 5.850208759307861, + 5.826308727264404, + 5.870480060577393, + 5.6683478355407715, + 5.726961135864258, + 5.549850940704346, + 5.503999710083008, + 5.4730753898620605, + 5.43950080871582, + 5.425447463989258, + 5.285499572753906, + 5.241710186004639, + 5.292726993560791, + 5.24367618560791, + 5.2555341720581055, + 5.186985015869141, + 5.204561233520508, + 5.106333255767822, + 5.008531093597412, + 5.0105509757995605, + 5.1520304679870605, + 4.994633674621582, + 5.018867015838623, + 4.996848106384277, + 4.97343635559082, + 5.010418891906738, + 5.047163963317871, + 4.998603343963623, + 4.872885227203369, + 4.954370498657227, + 4.929011344909668, + 5.008867263793945, + 4.797094345092773, + 4.885744094848633, + 4.863659858703613, + 4.905482292175293, + 4.8203911781311035, + 4.828099250793457, + 4.739256858825684, + 4.821928977966309, + 4.79324197769165, + 4.74005651473999, + 4.671626567840576, + 4.591048240661621, + 4.659725666046143, + 4.771541595458984, + 4.749525547027588, + 4.746364593505859, + 4.838879108428955, + 4.63433313369751, + 4.725112438201904, + 4.709056377410889, + 4.619563579559326, + 4.519699573516846, + 4.707879066467285, + 4.731076717376709, + 4.6175947189331055, + 4.489523410797119, + 4.737494468688965, + 4.668169975280762, + 4.6136980056762695, + 4.665773868560791, + 4.560977935791016, + 4.489831924438477, + 4.525557041168213, + 4.5385236740112305, + 4.688464641571045, + 4.509011745452881, + 4.4822492599487305, + 4.647952079772949, + 4.526414394378662, + 4.61146879196167, + 4.499902248382568, + 4.544570446014404, + 4.488522052764893, + 4.399788856506348, + 4.566093921661377, + 4.515257835388184, + 4.454895973205566, + 4.536690711975098, + 4.503605842590332, + 4.47681188583374, + 4.583615303039551, + 4.39484167098999, + 4.423252105712891, + 4.344427585601807, + 4.5844268798828125, + 4.48106575012207, + 4.40180778503418, + 4.461156368255615, + 4.499713897705078, + 4.3547468185424805, + 4.447062015533447, + 4.42684268951416, + 4.300321578979492, + 4.310922622680664, + 4.33964729309082, + 4.334997653961182, + 4.48938512802124, + 4.459761142730713, + 4.456637859344482, + 4.410120964050293, + 4.363800048828125, + 4.355828285217285, + 4.167102336883545, + 4.348094940185547, + 4.301021099090576, + 4.275911331176758, + 4.4059319496154785, + 4.339108943939209, + 4.374811172485352, + 4.3517889976501465, + 4.235577583312988, + 4.309335708618164, + 4.3768839836120605, + 4.256218433380127, + 4.331357479095459, + 4.286679744720459, + 4.29638671875, + 4.335381507873535, + 4.087395191192627, + 4.264644145965576, + 4.181642532348633, + 4.28421688079834, + 4.249885559082031, + 4.263835430145264, + 4.1105875968933105, + 4.313791275024414, + 4.254703998565674, + 4.207864284515381, + 4.243544101715088, + 4.207887649536133, + 4.119456768035889, + 4.229982852935791, + 4.225602149963379, + 4.193696022033691, + 4.216570854187012, + 4.169745445251465, + 4.266745090484619, + 4.249704360961914, + 4.159995079040527, + 4.12717866897583, + 4.25294828414917, + 4.248024940490723, + 4.213744163513184, + 4.156491756439209, + 4.205617427825928, + 4.160524845123291, + 4.142789840698242, + 4.2870378494262695, + 4.139543056488037, + 4.219180583953857, + 4.222052574157715, + 4.202698230743408, + 4.327624797821045, + 4.193352699279785, + 4.228542804718018, + 4.202315807342529, + 4.218052387237549, + 4.238234519958496, + 4.15584659576416, + 4.146408557891846, + 4.2923502922058105, + 4.150472164154053, + 4.220519065856934, + 4.30888557434082, + 4.306270122528076, + 4.201187610626221, + 4.145691871643066, + 4.195092678070068, + 4.2724809646606445, + 4.174814701080322, + 4.180525779724121, + 4.176600456237793, + 4.371306896209717, + 4.1685380935668945, + 4.056394577026367, + 4.213273048400879, + 4.048681735992432, + 4.075473785400391, + 4.177267074584961, + 4.056390285491943, + 4.136119365692139, + 4.17404317855835, + 4.1278977394104, + 4.190057277679443, + 4.09877347946167, + 4.138834476470947, + 4.091730117797852, + 4.152151584625244, + 4.148661136627197, + 4.153239727020264, + 4.1988725662231445, + 4.244430065155029, + 4.125945568084717, + 4.238224983215332, + 4.203845977783203, + 4.2612223625183105, + 4.051921844482422, + 4.244781494140625, + 4.042118549346924, + 4.183774948120117, + 4.19638729095459, + 4.187971115112305, + 4.13134241104126, + 4.036304473876953, + 4.116549491882324, + 4.128639221191406, + 4.013546943664551, + 4.122891426086426, + 4.213181495666504, + 4.224728584289551, + 4.08246374130249, + 4.081482887268066, + 4.165450572967529, + 4.142253875732422, + 4.065138816833496, + 4.104351997375488, + 4.222418308258057, + 4.119991779327393, + 4.112694263458252, + 4.141626358032227, + 4.189451217651367, + 4.052356243133545, + 4.1309051513671875, + 4.073043346405029, + 4.111188888549805, + 3.999927043914795, + 4.16348123550415, + 4.199490070343018, + 4.013392925262451, + 4.074688911437988, + 4.145191669464111, + 3.9569857120513916, + 4.064319610595703, + 4.083582401275635, + 4.166633129119873, + 4.091865062713623, + 3.876008987426758, + 3.9349372386932373, + 3.993823766708374, + 3.9391655921936035, + 3.9528486728668213, + 4.0328192710876465, + 4.138321876525879, + 4.0411248207092285, + 3.9220972061157227, + 4.03977108001709, + 3.9396984577178955, + 3.8267881870269775, + 3.9733588695526123, + 4.0713210105896, + 3.9946248531341553, + 4.04682731628418, + 3.922236680984497, + 3.9683921337127686, + 3.969268321990967, + 4.057724475860596, + 3.9889636039733887, + 3.953260898590088, + 4.009583950042725, + 3.9492194652557373, + 3.9072117805480957, + 3.8886032104492188, + 3.893204927444458, + 3.956012010574341, + 3.9294676780700684, + 4.036238193511963, + 3.9819138050079346, + 3.989941358566284, + 3.906395673751831, + 3.9456374645233154, + 4.0062665939331055, + 3.999539613723755, + 4.059412479400635, + 3.944591522216797, + 3.821061134338379, + 3.8043735027313232, + 3.7980806827545166, + 3.991305112838745, + 3.935133457183838, + 3.9999942779541016, + 3.8375771045684814, + 3.9765372276306152, + 4.113861560821533, + 3.930140972137451, + 3.923128843307495, + 3.9921250343322754, + 3.866506814956665, + 3.8397915363311768, + 4.121239185333252, + 3.984369993209839, + 3.9262657165527344, + 3.837308645248413, + 3.913419246673584, + 3.9044055938720703, + 3.8976800441741943, + 4.089669704437256, + 3.944767475128174, + 4.027477741241455, + 3.8886806964874268, + 4.019046306610107, + 3.788534641265869, + 3.9899728298187256, + 3.8677525520324707, + 3.9495272636413574, + 3.787240982055664, + 3.985431432723999, + 3.871751070022583, + 3.898937940597534, + 3.9160702228546143, + 3.8304202556610107, + 4.0052008628845215, + 3.9901516437530518, + 3.9135279655456543, + 3.8706867694854736, + 3.8840138912200928, + 3.822582483291626, + 3.9185938835144043, + 3.8887460231781006, + 3.98396897315979, + 4.045979976654053, + 3.8717586994171143, + 3.7501840591430664, + 3.951136350631714, + 3.898892879486084, + 3.7737996578216553, + 3.881831645965576, + 3.992896556854248, + 3.87898850440979, + 3.9876229763031006, + 3.9207067489624023, + 3.933159828186035, + 4.012601375579834, + 3.8554131984710693, + 3.929631471633911, + 3.930241107940674, + 3.995680332183838, + 3.896207571029663, + 4.030653476715088, + 3.9531867504119873, + 3.7954049110412598, + 3.903604030609131, + 3.904552698135376, + 3.8213014602661133, + 3.7755444049835205, + 3.8882620334625244, + 3.9958059787750244, + 3.986117124557495, + 3.8635942935943604, + 3.806579351425171, + 3.8366029262542725, + 3.826475143432617, + 3.9505770206451416, + 3.861332893371582, + 3.9311439990997314, + 4.00569486618042, + 3.9326272010803223, + 3.9211373329162598, + 4.020817279815674, + 3.8107635974884033, + 3.791317939758301, + 4.056236267089844, + 3.9263458251953125, + 3.8864693641662598, + 3.6681153774261475, + 3.778944492340088, + 3.8469603061676025, + 3.8401975631713867, + 3.897451639175415, + 3.70147442817688, + 3.8545172214508057, + 3.7771997451782227, + 3.775808572769165, + 3.745698928833008, + 3.835451126098633, + 3.7015531063079834, + 3.7537825107574463, + 3.819373846054077, + 3.8823156356811523, + 3.803964138031006, + 3.7469842433929443, + 3.8817296028137207, + 3.7851831912994385, + 3.8059070110321045, + 3.6682450771331787, + 3.846484899520874, + 3.734341859817505, + 3.711761713027954, + 3.916264057159424, + 3.72143816947937, + 3.8661787509918213, + 3.7813713550567627, + 3.7185540199279785, + 3.737809181213379, + 3.7412421703338623, + 3.8696768283843994, + 3.9029910564422607, + 3.7849442958831787, + 3.8795619010925293, + 3.850625991821289, + 3.9025161266326904, + 3.845372438430786, + 3.740061044692993, + 3.935523748397827, + 3.879446268081665, + 3.613387107849121, + 3.7966957092285156, + 3.823575019836426, + 3.825120210647583, + 3.7716519832611084, + 3.848994493484497, + 3.7904815673828125, + 3.7156362533569336, + 3.7798173427581787, + 3.888209819793701, + 3.8201520442962646, + 3.852180242538452, + 3.8095898628234863, + 3.817394733428955, + 3.8572113513946533, + 3.8196022510528564, + 3.7024853229522705, + 3.872314691543579, + 3.892430305480957, + 3.856240749359131, + 3.796151638031006, + 3.776587963104248, + 3.8391382694244385, + 3.760429620742798, + 4.007333755493164, + 3.8214550018310547, + 3.937871217727661, + 3.864959478378296, + 3.9207065105438232, + 3.8893353939056396, + 3.829385280609131, + 3.7299320697784424, + 3.76615571975708, + 3.8031845092773438, + 3.831599712371826, + 3.7570717334747314, + 4.039464950561523, + 3.874194383621216, + 3.708134412765503, + 3.7412703037261963, + 3.730808734893799, + 3.837522506713867, + 3.6712448596954346, + 3.796297311782837, + 3.7674527168273926, + 3.7581515312194824, + 3.9755325317382812, + 3.795494794845581, + 3.7127602100372314, + 3.7651126384735107, + 3.766188859939575, + 3.873418092727661, + 3.6103830337524414, + 3.7577195167541504, + 3.780710220336914, + 3.6937992572784424, + 3.8668339252471924, + 3.7653586864471436, + 3.8214282989501953, + 3.8416080474853516, + 3.787696361541748, + 3.8745455741882324, + 3.930877685546875, + 3.7658042907714844, + 3.79201078414917, + 3.8872880935668945, + 3.815173387527466, + 3.6608893871307373, + 3.7390594482421875, + 3.7214913368225098, + 3.858289957046509, + 3.820800304412842, + 3.727396011352539, + 3.6642861366271973, + 3.7281932830810547, + 3.989138126373291, + 3.751317262649536, + 3.686551332473755, + 3.9030628204345703, + 3.7267074584960938, + 3.84714937210083, + 3.8436241149902344, + 3.8023948669433594, + 3.7565665245056152, + 3.7987027168273926, + 3.771235227584839, + 3.744670867919922, + 3.552391290664673, + 3.7142088413238525, + 3.7080700397491455, + 3.5934505462646484, + 3.775585174560547, + 3.5695183277130127, + 3.548501491546631, + 3.6051828861236572, + 3.697112560272217, + 3.725862979888916, + 3.7781949043273926, + 3.747189998626709, + 3.617314338684082, + 3.5749669075012207, + 3.6393203735351562, + 3.7242624759674072, + 3.541642904281616, + 3.742842197418213, + 3.781643867492676, + 3.4256436824798584, + 3.581279993057251, + 3.7802069187164307, + 3.751925230026245, + 3.640079975128174, + 3.6082630157470703, + 3.747610092163086, + 3.693580389022827, + 3.6817548274993896, + 3.696645498275757, + 3.763031005859375, + 3.64544415473938, + 3.739029884338379, + 3.6475226879119873, + 3.637176036834717, + 3.592376232147217, + 3.65321946144104, + 3.700617551803589, + 3.7507975101470947, + 3.581547498703003, + 3.757249593734741, + 3.6986608505249023, + 3.7782933712005615, + 3.7256503105163574, + 3.7386255264282227, + 3.5588338375091553, + 3.6377153396606445, + 3.5418758392333984, + 3.678131580352783, + 3.7253565788269043, + 3.7351932525634766, + 3.6726043224334717, + 3.715561628341675, + 3.737248182296753, + 3.576096534729004, + 3.7638213634490967, + 3.5116260051727295, + 3.5707590579986572, + 3.670243740081787, + 3.619992971420288, + 3.70268177986145, + 3.6097989082336426, + 3.650533437728882, + 3.629424810409546, + 3.616062641143799, + 3.571471929550171, + 3.4976041316986084, + 3.711287021636963, + 3.637078285217285, + 3.6977782249450684, + 3.793828010559082, + 3.7779104709625244, + 3.591447591781616, + 3.6434502601623535, + 3.811859369277954, + 3.6760549545288086, + 3.846511125564575, + 3.7091000080108643, + 3.591615676879883, + 3.586031913757324, + 3.680521249771118, + 3.784165143966675, + 3.751734495162964, + 3.623234272003174, + 3.62522554397583, + 3.798555850982666, + 3.767988443374634, + 3.823410987854004, + 3.804354429244995, + 3.722308874130249, + 3.7673375606536865, + 3.547032117843628, + 3.6750776767730713, + 3.6325411796569824, + 3.653730869293213, + 3.679917097091675, + 3.664430856704712, + 3.592836380004883, + 3.583141565322876, + 3.590343952178955, + 3.7119297981262207, + 3.7216267585754395, + 3.7771365642547607, + 3.739330530166626, + 3.847104787826538, + 3.681243658065796, + 3.698143243789673, + 3.5917558670043945, + 3.590402603149414, + 3.615830659866333, + 3.5656790733337402, + 3.7008402347564697, + 3.637935161590576, + 3.622783899307251, + 3.7513813972473145, + 3.681750535964966, + 3.5906572341918945, + 3.5711376667022705, + 3.751861810684204, + 3.5784027576446533, + 3.7571380138397217, + 3.596679925918579, + 3.633815050125122, + 3.5137410163879395, + 3.674729585647583, + 3.4632060527801514, + 3.5495214462280273, + 3.673853874206543, + 3.66035532951355, + 3.4765729904174805, + 3.5624923706054688, + 3.578552007675171, + 3.6506452560424805, + 3.5380890369415283, + 3.4654109477996826, + 3.6347429752349854, + 3.480255365371704, + 3.617182493209839, + 3.483891487121582, + 3.6242871284484863, + 3.582185745239258, + 3.576536178588867, + 3.5932681560516357, + 3.538604736328125, + 3.5629987716674805, + 3.644787073135376, + 3.525937557220459, + 3.565622568130493, + 3.6223092079162598, + 3.603172779083252, + 3.613119602203369, + 3.548093557357788, + 3.491363525390625, + 3.653740167617798, + 3.55894136428833, + 3.581916332244873, + 3.5049691200256348, + 3.6071183681488037, + 3.5756702423095703, + 3.6753597259521484, + 3.6407623291015625, + 3.498297929763794, + 3.580970048904419, + 3.534104824066162, + 3.5725204944610596, + 3.586280584335327, + 3.5496068000793457, + 3.566124200820923, + 3.5515568256378174, + 3.515432834625244, + 3.5386884212493896, + 3.704859972000122, + 3.5482616424560547, + 3.5921013355255127, + 3.599461555480957, + 3.6169750690460205, + 3.715656042098999, + 3.4693846702575684, + 3.4530885219573975, + 3.6550142765045166, + 3.709343910217285, + 3.574089765548706, + 3.53956937789917, + 3.5733108520507812, + 3.638110876083374, + 3.587681770324707, + 3.5427331924438477, + 3.5486862659454346, + 3.5921037197113037, + 3.4582390785217285, + 3.518674850463867, + 3.564880132675171, + 3.57627534866333, + 3.6205661296844482, + 3.540538787841797, + 3.6434638500213623, + 3.636090040206909, + 3.5462779998779297, + 3.6243417263031006, + 3.563070774078369, + 3.5900325775146484, + 3.519386053085327, + 3.5427536964416504, + 3.6117966175079346, + 3.5014216899871826, + 3.54437518119812, + 3.595069408416748, + 3.6587696075439453, + 3.6967122554779053, + 3.589123249053955, + 3.64013934135437, + 3.635451316833496, + 3.6429800987243652, + 3.6436283588409424, + 3.526747941970825, + 3.555640935897827, + 3.507657766342163, + 3.4929604530334473, + 3.617961883544922, + 3.69112229347229, + 3.4644947052001953, + 3.539407253265381, + 3.662654399871826, + 3.4972410202026367, + 3.568061590194702, + 3.474600076675415, + 3.4684736728668213, + 3.6860220432281494, + 3.5546810626983643, + 3.560670852661133, + 3.59783935546875, + 3.586866855621338, + 3.5731167793273926, + 3.5713887214660645, + 3.4233343601226807, + 3.525423765182495, + 3.6521248817443848, + 3.5245819091796875, + 3.426947832107544, + 3.5600080490112305, + 3.5902538299560547, + 3.592259645462036, + 3.5653927326202393, + 3.6174280643463135, + 3.6100122928619385, + 3.4398202896118164, + 3.44205641746521, + 3.5714657306671143, + 3.560528516769409, + 3.5630383491516113, + 3.595984697341919, + 3.466848134994507, + 3.5269155502319336, + 3.355736017227173, + 3.4583582878112793, + 3.568662405014038, + 3.5550148487091064, + 3.5529582500457764, + 3.677356243133545, + 3.5834503173828125, + 3.468249797821045, + 3.436788320541382, + 3.504058837890625, + 3.5094497203826904, + 3.5389394760131836, + 3.532231569290161, + 3.5068347454071045, + 3.4911372661590576, + 3.5701534748077393, + 3.5388193130493164, + 3.6666345596313477, + 3.5279605388641357, + 3.4608161449432373, + 3.565582513809204, + 3.429067373275757, + 3.4630987644195557, + 3.569195508956909, + 3.694653272628784, + 3.5846610069274902, + 3.296563148498535, + 3.55104923248291, + 3.5234785079956055, + 3.46649432182312, + 3.508518695831299, + 3.5003366470336914, + 3.5260238647460938, + 3.4911179542541504, + 3.547572612762451, + 3.5134148597717285, + 3.509535551071167, + 3.561269521713257, + 3.5531113147735596, + 3.516148090362549, + 3.532546281814575, + 3.518235445022583, + 3.5646464824676514, + 3.5659334659576416, + 3.456594467163086, + 3.5046489238739014, + 3.562256336212158, + 3.4798474311828613, + 3.542059898376465, + 3.388664484024048, + 3.554516077041626, + 3.413910150527954, + 3.5425384044647217, + 3.298243522644043, + 3.6402366161346436, + 3.557034969329834, + 3.5380635261535645, + 3.528737783432007, + 3.371936798095703, + 3.469114065170288, + 3.5438144207000732, + 3.4501235485076904, + 3.477541446685791, + 3.7855491638183594, + 3.5297365188598633, + 3.438443183898926, + 3.540588617324829, + 3.5194361209869385, + 3.517292022705078, + 3.504167079925537, + 3.6397624015808105, + 3.4149937629699707, + 3.364293336868286, + 3.4390275478363037, + 3.5711190700531006, + 3.5315091609954834, + 3.396728277206421, + 3.5874531269073486, + 3.4449920654296875, + 3.480032444000244, + 3.5653696060180664, + 3.3526947498321533, + 3.561998128890991, + 3.556063652038574, + 3.5429434776306152, + 3.3599112033843994, + 3.5332870483398438, + 3.5259947776794434, + 3.569519519805908, + 3.5507259368896484, + 3.415658712387085, + 3.5309879779815674, + 3.4644062519073486, + 3.5051987171173096, + 3.410571813583374, + 3.735011339187622, + 3.406954288482666, + 3.4627349376678467, + 3.551347255706787, + 3.5311689376831055, + 3.582347869873047, + 3.583953857421875, + 3.4535984992980957, + 3.3753204345703125, + 3.3829431533813477, + 3.519702196121216, + 3.4802327156066895, + 3.3543899059295654, + 3.5105068683624268, + 3.4565489292144775, + 3.35892915725708, + 3.38623309135437, + 3.526571035385132, + 3.5790696144104004, + 3.552342414855957, + 3.450380802154541, + 3.507221221923828, + 3.4746811389923096, + 3.5721354484558105, + 3.468620538711548, + 3.4606451988220215, + 3.6759698390960693, + 3.4941465854644775, + 3.517756938934326, + 3.5576932430267334, + 3.322026252746582, + 3.479990005493164, + 3.3894522190093994, + 3.501397132873535, + 3.3210489749908447, + 3.3568291664123535, + 3.5074055194854736, + 3.293165922164917, + 3.509260654449463, + 3.3571009635925293, + 3.506178140640259, + 3.3750929832458496, + 3.5588886737823486, + 3.479710817337036, + 3.4554786682128906, + 3.506098985671997, + 3.278369188308716, + 3.573606014251709, + 3.5054192543029785, + 3.5338213443756104, + 3.4064393043518066, + 3.484380006790161, + 3.4013290405273438, + 3.4954164028167725, + 3.484964609146118, + 3.5423176288604736, + 3.504007339477539, + 3.5269389152526855, + 3.398319959640503, + 3.5363516807556152, + 3.518587827682495, + 3.220271348953247, + 3.383885383605957, + 3.494920492172241, + 3.513577699661255, + 3.487565755844116, + 3.4450843334198, + 3.4789276123046875, + 3.5006844997406006, + 3.555025815963745, + 3.5452866554260254, + 3.469714641571045, + 3.4392573833465576, + 3.3509960174560547, + 3.4295594692230225, + 3.5197091102600098, + 3.486398220062256, + 3.48854660987854, + 3.5634689331054688, + 3.5542914867401123, + 3.4633779525756836, + 3.4281632900238037, + 3.4007511138916016, + 3.4494407176971436, + 3.4763877391815186, + 3.5450310707092285, + 3.4386916160583496, + 3.37241268157959, + 3.5572705268859863, + 3.6266696453094482, + 3.3245179653167725, + 3.453747510910034, + 3.351201057434082, + 3.4343960285186768, + 3.4340052604675293, + 3.3855788707733154, + 3.519819736480713, + 3.3988397121429443, + 3.5764636993408203, + 3.4529030323028564, + 3.3667194843292236, + 3.4657278060913086, + 3.4804697036743164, + 3.517956018447876, + 3.4386305809020996, + 3.506929636001587, + 3.310760498046875, + 3.4460740089416504, + 3.4820518493652344, + 3.5266621112823486, + 3.4363677501678467, + 3.4393157958984375, + 3.475524663925171, + 3.280350685119629, + 3.440467119216919, + 3.5422770977020264, + 3.5706682205200195, + 3.4321165084838867, + 3.432629346847534, + 3.3825364112854004, + 3.4436545372009277, + 3.2991342544555664, + 3.4306252002716064, + 3.454113006591797, + 3.507072925567627, + 3.47044038772583, + 3.442910671234131, + 3.3902673721313477, + 3.485333204269409, + 3.4043309688568115, + 3.3695645332336426, + 3.379263162612915, + 3.3784122467041016, + 3.4025044441223145, + 3.2792303562164307, + 3.4461255073547363, + 3.440650463104248, + 3.5618789196014404, + 3.3229944705963135, + 3.5436336994171143, + 3.4020819664001465, + 3.4724221229553223, + 3.435138463973999, + 3.402848720550537, + 3.4252495765686035 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 113.46094308820578, + 78.24262766203455, + 65.21845723460221, + 57.84690503438683, + 53.21738300741326, + 49.994143457252335, + 47.513419471154364, + 45.63399196194061, + 43.529968211076906, + 41.762434299196755, + 40.68542956108317, + 39.49840000190628, + 38.767107809571435, + 37.72988925763354, + 36.99001843132054, + 36.80978880803848, + 36.244858207038625, + 36.06444875532147, + 35.97640463560301, + 35.95672233853928 + ], + "best_ppl": 35.95672233853928, + "train_time_s": 34971.148898124695 +} \ No newline at end of file diff --git a/060_sparse_mol/top4of14_best.pt b/060_sparse_mol/top4of14_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd38f79fa270c74f0df1b7eafd6f6f9738167679 --- /dev/null +++ b/060_sparse_mol/top4of14_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:116ff8b9e1c7abc62390ca3381edaca89f52c7d33a53e14ef17de757a2e66f3d +size 555315458 diff --git a/060_sparse_mol/top4of7_best.pt b/060_sparse_mol/top4of7_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc895035a1474737b2034b5ed76aaf5077f2383d --- /dev/null +++ b/060_sparse_mol/top4of7_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ff55aebdf82a88d9fb7ac1c0d8b75ea16372d53dee28f2762fe7db4b63f6d4d +size 343197825 diff --git a/060_sparse_mol/top4of7_results.json b/060_sparse_mol/top4of7_results.json new file mode 100644 index 0000000000000000000000000000000000000000..f333ca38d9f643804a6d7dfd23708d431d3c5fc2 --- /dev/null +++ b/060_sparse_mol/top4of7_results.json @@ -0,0 +1,2057 @@ +{ + "label": "top4of7", + "top_k_blocks": 4, + "num_thin_blocks": 7, + "total_params": 85676160, + "router_params": 57344, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 15.922952651977539, + 13.465934753417969, + 7.871944427490234, + 7.201005935668945, + 7.072938919067383, + 6.922619819641113, + 6.832179069519043, + 6.700571537017822, + 6.610640048980713, + 6.365942001342773, + 6.368068218231201, + 6.2671332359313965, + 6.228082180023193, + 6.053006172180176, + 6.120823383331299, + 5.867644786834717, + 5.841027736663818, + 5.874963760375977, + 5.690984725952148, + 5.741836071014404, + 5.56394624710083, + 5.51140022277832, + 5.481002330780029, + 5.432435989379883, + 5.427206516265869, + 5.282928943634033, + 5.229360103607178, + 5.290107250213623, + 5.227585315704346, + 5.248988151550293, + 5.166275978088379, + 5.201637268066406, + 5.085593223571777, + 4.9952616691589355, + 4.997631072998047, + 5.121885299682617, + 4.964172840118408, + 4.985504150390625, + 4.9692888259887695, + 4.9500226974487305, + 4.997861862182617, + 5.008114337921143, + 4.971133708953857, + 4.832202911376953, + 4.912046432495117, + 4.902180194854736, + 4.990547180175781, + 4.769317150115967, + 4.850738048553467, + 4.823439598083496, + 4.874029159545898, + 4.793235778808594, + 4.77675724029541, + 4.703404903411865, + 4.800662040710449, + 4.752368450164795, + 4.697490692138672, + 4.625783920288086, + 4.551953315734863, + 4.6125946044921875, + 4.71744966506958, + 4.712122440338135, + 4.714359283447266, + 4.765403747558594, + 4.5765380859375, + 4.661972999572754, + 4.677060127258301, + 4.563080310821533, + 4.4681525230407715, + 4.6493330001831055, + 4.691705703735352, + 4.5501017570495605, + 4.436013698577881, + 4.675163745880127, + 4.598043918609619, + 4.573880672454834, + 4.5976667404174805, + 4.496283531188965, + 4.434981822967529, + 4.462356090545654, + 4.473270416259766, + 4.6337127685546875, + 4.434659004211426, + 4.423381328582764, + 4.580365180969238, + 4.484320163726807, + 4.555619716644287, + 4.433425426483154, + 4.48984432220459, + 4.422302722930908, + 4.339358329772949, + 4.485730171203613, + 4.454977512359619, + 4.390565395355225, + 4.474174499511719, + 4.421513080596924, + 4.419271945953369, + 4.4922709465026855, + 4.325674533843994, + 4.353109359741211, + 4.288665294647217, + 4.502604007720947, + 4.403310775756836, + 4.335092544555664, + 4.386446475982666, + 4.430429458618164, + 4.292841911315918, + 4.356780052185059, + 4.354743003845215, + 4.2368340492248535, + 4.22773551940918, + 4.263389587402344, + 4.261373043060303, + 4.4138569831848145, + 4.395515441894531, + 4.3595170974731445, + 4.344120502471924, + 4.2821197509765625, + 4.273412227630615, + 4.098631858825684, + 4.255722999572754, + 4.21401309967041, + 4.2012104988098145, + 4.32580041885376, + 4.2490644454956055, + 4.296151638031006, + 4.257314682006836, + 4.143311977386475, + 4.235822677612305, + 4.307238578796387, + 4.201661586761475, + 4.268105506896973, + 4.204440116882324, + 4.2190165519714355, + 4.266382694244385, + 4.010256290435791, + 4.182242393493652, + 4.103518486022949, + 4.196938514709473, + 4.184638023376465, + 4.174100399017334, + 4.043487071990967, + 4.224959850311279, + 4.154592990875244, + 4.125119686126709, + 4.191795825958252, + 4.119335651397705, + 4.064598560333252, + 4.167088985443115, + 4.149066925048828, + 4.141265869140625, + 4.1492085456848145, + 4.082738876342773, + 4.181623935699463, + 4.166955471038818, + 4.076641082763672, + 4.049681186676025, + 4.167205810546875, + 4.174193382263184, + 4.144207954406738, + 4.079165458679199, + 4.129480361938477, + 4.085082054138184, + 4.0510334968566895, + 4.217842102050781, + 4.057997226715088, + 4.137166500091553, + 4.134531497955322, + 4.121287822723389, + 4.251450061798096, + 4.116115093231201, + 4.15383243560791, + 4.117125988006592, + 4.147226810455322, + 4.1607489585876465, + 4.063211917877197, + 4.064483642578125, + 4.224440574645996, + 4.067689895629883, + 4.150542259216309, + 4.250365734100342, + 4.2191853523254395, + 4.111179828643799, + 4.077404499053955, + 4.126224040985107, + 4.207942008972168, + 4.105756759643555, + 4.100567817687988, + 4.090517997741699, + 4.313811779022217, + 4.100875377655029, + 3.9876832962036133, + 4.1309590339660645, + 3.9860620498657227, + 3.9961841106414795, + 4.10012149810791, + 3.9932215213775635, + 4.044100761413574, + 4.092435836791992, + 4.059265613555908, + 4.123799800872803, + 4.033945560455322, + 4.070641040802002, + 4.0234527587890625, + 4.06150484085083, + 4.064716339111328, + 4.090487957000732, + 4.12720251083374, + 4.153529644012451, + 4.043868064880371, + 4.184878349304199, + 4.1289777755737305, + 4.186354637145996, + 3.986510992050171, + 4.161097049713135, + 3.9595179557800293, + 4.113043785095215, + 4.108085632324219, + 4.101855278015137, + 4.037428379058838, + 3.9723212718963623, + 4.062034606933594, + 4.06420373916626, + 3.936976432800293, + 4.0421600341796875, + 4.12136173248291, + 4.141195297241211, + 4.001479148864746, + 3.9867377281188965, + 4.089313983917236, + 4.063852787017822, + 3.981947183609009, + 4.035221099853516, + 4.149799346923828, + 4.057619094848633, + 4.033631324768066, + 4.0619072914123535, + 4.099677085876465, + 3.976226806640625, + 4.0440521240234375, + 3.984912633895874, + 4.032658576965332, + 3.928774833679199, + 4.101667404174805, + 4.109521389007568, + 3.9435508251190186, + 3.994757890701294, + 4.065024375915527, + 3.884148597717285, + 3.9964325428009033, + 4.000634670257568, + 4.096513271331787, + 4.0094380378723145, + 3.777885675430298, + 3.856720209121704, + 3.9234461784362793, + 3.87237286567688, + 3.868338108062744, + 3.933112859725952, + 4.057020664215088, + 3.9592862129211426, + 3.8249239921569824, + 3.9559874534606934, + 3.866690158843994, + 3.7413201332092285, + 3.894523859024048, + 3.9991042613983154, + 3.9138119220733643, + 3.9584083557128906, + 3.8433613777160645, + 3.880648374557495, + 3.897874116897583, + 3.981250047683716, + 3.903658151626587, + 3.884230852127075, + 3.922201633453369, + 3.8839945793151855, + 3.824064254760742, + 3.814394235610962, + 3.8269357681274414, + 3.898145914077759, + 3.8603928089141846, + 3.9606049060821533, + 3.9056172370910645, + 3.916383743286133, + 3.819716453552246, + 3.8517746925354004, + 3.927114725112915, + 3.9152333736419678, + 3.9624228477478027, + 3.8564703464508057, + 3.7488315105438232, + 3.742755889892578, + 3.7197041511535645, + 3.9085304737091064, + 3.8544459342956543, + 3.9238102436065674, + 3.75571346282959, + 3.8762712478637695, + 4.03447961807251, + 3.8448405265808105, + 3.850111246109009, + 3.9277520179748535, + 3.7823283672332764, + 3.7623965740203857, + 4.0450592041015625, + 3.9177074432373047, + 3.8549318313598633, + 3.760857105255127, + 3.8503761291503906, + 3.837749719619751, + 3.8278884887695312, + 4.021198749542236, + 3.866926431655884, + 3.9442801475524902, + 3.8102941513061523, + 3.948354482650757, + 3.7135889530181885, + 3.9150664806365967, + 3.7845640182495117, + 3.850590705871582, + 3.717359781265259, + 3.915534734725952, + 3.7939505577087402, + 3.819603443145752, + 3.831312894821167, + 3.744110584259033, + 3.9247965812683105, + 3.9303994178771973, + 3.8379247188568115, + 3.781942367553711, + 3.810851573944092, + 3.7623116970062256, + 3.844155788421631, + 3.81650447845459, + 3.889683485031128, + 3.9485838413238525, + 3.795792818069458, + 3.6755483150482178, + 3.884392023086548, + 3.828198194503784, + 3.6977379322052, + 3.794564962387085, + 3.9082212448120117, + 3.830853223800659, + 3.9302310943603516, + 3.8338875770568848, + 3.8583240509033203, + 3.962674140930176, + 3.79315447807312, + 3.846895217895508, + 3.8627474308013916, + 3.93953013420105, + 3.813351631164551, + 3.954561710357666, + 3.875412940979004, + 3.7383975982666016, + 3.832310199737549, + 3.807340621948242, + 3.7453536987304688, + 3.70343017578125, + 3.807751178741455, + 3.9029195308685303, + 3.9043469429016113, + 3.7991983890533447, + 3.751601219177246, + 3.7581164836883545, + 3.7441227436065674, + 3.8711235523223877, + 3.794818162918091, + 3.8501923084259033, + 3.928457260131836, + 3.8427963256835938, + 3.8313136100769043, + 3.930886745452881, + 3.7414748668670654, + 3.7191789150238037, + 3.982401132583618, + 3.8520402908325195, + 3.7987453937530518, + 3.6044247150421143, + 3.7161974906921387, + 3.775261402130127, + 3.7681312561035156, + 3.8215317726135254, + 3.6140949726104736, + 3.7714521884918213, + 3.6939773559570312, + 3.6951591968536377, + 3.6710736751556396, + 3.7685046195983887, + 3.6233720779418945, + 3.6714980602264404, + 3.744487762451172, + 3.807809352874756, + 3.7220568656921387, + 3.6847426891326904, + 3.8038439750671387, + 3.7025856971740723, + 3.747844696044922, + 3.5884084701538086, + 3.7519450187683105, + 3.6709587574005127, + 3.631096124649048, + 3.829648971557617, + 3.632723093032837, + 3.801583766937256, + 3.7191498279571533, + 3.645604133605957, + 3.662318706512451, + 3.6846911907196045, + 3.7852320671081543, + 3.8188114166259766, + 3.7110342979431152, + 3.800144910812378, + 3.772029161453247, + 3.8133063316345215, + 3.7813796997070312, + 3.6557559967041016, + 3.872413396835327, + 3.798868179321289, + 3.525352954864502, + 3.741987943649292, + 3.7527754306793213, + 3.757932186126709, + 3.714634895324707, + 3.7458746433258057, + 3.703788995742798, + 3.6469764709472656, + 3.7023584842681885, + 3.8039700984954834, + 3.7439048290252686, + 3.7704520225524902, + 3.75172758102417, + 3.744593381881714, + 3.7825260162353516, + 3.728963613510132, + 3.622819185256958, + 3.799783706665039, + 3.8189544677734375, + 3.790999412536621, + 3.737637996673584, + 3.716919422149658, + 3.7648749351501465, + 3.685525417327881, + 3.946392059326172, + 3.7289600372314453, + 3.874922037124634, + 3.7907907962799072, + 3.832573413848877, + 3.8040401935577393, + 3.748220443725586, + 3.655360460281372, + 3.6912004947662354, + 3.726041316986084, + 3.7597482204437256, + 3.679215908050537, + 3.9582183361053467, + 3.8082756996154785, + 3.6289453506469727, + 3.671556234359741, + 3.662198305130005, + 3.753237009048462, + 3.6073155403137207, + 3.7149312496185303, + 3.686650037765503, + 3.6612629890441895, + 3.884430170059204, + 3.727344036102295, + 3.6447947025299072, + 3.690995693206787, + 3.6693875789642334, + 3.808713912963867, + 3.5318551063537598, + 3.685453176498413, + 3.7012925148010254, + 3.6126859188079834, + 3.7843003273010254, + 3.6886072158813477, + 3.7282211780548096, + 3.7592878341674805, + 3.717468738555908, + 3.8142356872558594, + 3.842460870742798, + 3.7060558795928955, + 3.7157227993011475, + 3.794426441192627, + 3.7398719787597656, + 3.5847256183624268, + 3.681662082672119, + 3.657871961593628, + 3.7729272842407227, + 3.7541441917419434, + 3.6571803092956543, + 3.5966243743896484, + 3.644524097442627, + 3.9114761352539062, + 3.660428762435913, + 3.604496955871582, + 3.8317606449127197, + 3.637645959854126, + 3.777935743331909, + 3.761263370513916, + 3.718398094177246, + 3.6928844451904297, + 3.725302219390869, + 3.669722080230713, + 3.6645963191986084, + 3.4973864555358887, + 3.645500898361206, + 3.6312832832336426, + 3.536661386489868, + 3.697826862335205, + 3.4920201301574707, + 3.453394889831543, + 3.524148464202881, + 3.626694679260254, + 3.6423425674438477, + 3.7080845832824707, + 3.672335386276245, + 3.5496082305908203, + 3.505732297897339, + 3.5641727447509766, + 3.6378791332244873, + 3.458181619644165, + 3.656486988067627, + 3.7063653469085693, + 3.356494426727295, + 3.504770278930664, + 3.695274829864502, + 3.6605758666992188, + 3.5828816890716553, + 3.5275156497955322, + 3.6643035411834717, + 3.6329095363616943, + 3.6181600093841553, + 3.6245415210723877, + 3.6709439754486084, + 3.569573402404785, + 3.648699998855591, + 3.561558485031128, + 3.5706498622894287, + 3.509263515472412, + 3.5716161727905273, + 3.6238484382629395, + 3.6652512550354004, + 3.5029022693634033, + 3.6794116497039795, + 3.63423228263855, + 3.700115203857422, + 3.6572232246398926, + 3.6616287231445312, + 3.484983444213867, + 3.5680742263793945, + 3.4713547229766846, + 3.6192879676818848, + 3.654221534729004, + 3.658830165863037, + 3.608219623565674, + 3.65012788772583, + 3.6619067192077637, + 3.512110710144043, + 3.676502227783203, + 3.4473721981048584, + 3.489290237426758, + 3.61027455329895, + 3.550152540206909, + 3.625775098800659, + 3.5264320373535156, + 3.5777089595794678, + 3.565429449081421, + 3.5477232933044434, + 3.4907352924346924, + 3.4196228981018066, + 3.6463587284088135, + 3.5699665546417236, + 3.6227128505706787, + 3.7173497676849365, + 3.713568925857544, + 3.5195136070251465, + 3.577509641647339, + 3.7360010147094727, + 3.6055450439453125, + 3.775348424911499, + 3.6288516521453857, + 3.523120403289795, + 3.528899669647217, + 3.5927727222442627, + 3.703808307647705, + 3.6928136348724365, + 3.5375311374664307, + 3.5563137531280518, + 3.7336504459381104, + 3.682961940765381, + 3.763658285140991, + 3.7293624877929688, + 3.6513898372650146, + 3.6751558780670166, + 3.482144832611084, + 3.6040329933166504, + 3.568817615509033, + 3.582188606262207, + 3.6071712970733643, + 3.5822880268096924, + 3.5031590461730957, + 3.503551721572876, + 3.513866424560547, + 3.630258083343506, + 3.648151159286499, + 3.7209548950195312, + 3.672189474105835, + 3.765336275100708, + 3.6035423278808594, + 3.6374831199645996, + 3.5118935108184814, + 3.511641502380371, + 3.5402352809906006, + 3.4988842010498047, + 3.603692054748535, + 3.5855495929718018, + 3.526362419128418, + 3.680748701095581, + 3.6201043128967285, + 3.5176682472229004, + 3.5023443698883057, + 3.687394142150879, + 3.508193016052246, + 3.6844394207000732, + 3.5381686687469482, + 3.5542051792144775, + 3.4364960193634033, + 3.6058151721954346, + 3.4170210361480713, + 3.4822561740875244, + 3.594994068145752, + 3.5819332599639893, + 3.4102413654327393, + 3.4994540214538574, + 3.524475336074829, + 3.572411060333252, + 3.4655635356903076, + 3.37972354888916, + 3.5615992546081543, + 3.4132018089294434, + 3.5438971519470215, + 3.420234203338623, + 3.550854206085205, + 3.5088319778442383, + 3.4916319847106934, + 3.527190923690796, + 3.471885919570923, + 3.4904491901397705, + 3.5966057777404785, + 3.4540481567382812, + 3.4889583587646484, + 3.564166784286499, + 3.5280697345733643, + 3.5524771213531494, + 3.458493947982788, + 3.4247288703918457, + 3.6028640270233154, + 3.4800987243652344, + 3.511831760406494, + 3.4413342475891113, + 3.536757230758667, + 3.5030438899993896, + 3.607093572616577, + 3.54237699508667, + 3.406651735305786, + 3.491912841796875, + 3.465404748916626, + 3.523646593093872, + 3.526662826538086, + 3.4808595180511475, + 3.491669178009033, + 3.464437484741211, + 3.4454762935638428, + 3.44460391998291, + 3.636953592300415, + 3.4799766540527344, + 3.519195795059204, + 3.5165698528289795, + 3.5512993335723877, + 3.627781867980957, + 3.4032604694366455, + 3.405348539352417, + 3.5840978622436523, + 3.6353139877319336, + 3.5015769004821777, + 3.458890199661255, + 3.5068793296813965, + 3.5723750591278076, + 3.5162549018859863, + 3.4707236289978027, + 3.4805843830108643, + 3.522145986557007, + 3.3981142044067383, + 3.4522149562835693, + 3.497673511505127, + 3.5154361724853516, + 3.5534939765930176, + 3.4707698822021484, + 3.5799672603607178, + 3.5749051570892334, + 3.451965808868408, + 3.5377042293548584, + 3.4930315017700195, + 3.5143954753875732, + 3.453306198120117, + 3.4657530784606934, + 3.554966926574707, + 3.433997631072998, + 3.4868366718292236, + 3.5257070064544678, + 3.58069109916687, + 3.610132932662964, + 3.5140652656555176, + 3.584588050842285, + 3.5591788291931152, + 3.5802526473999023, + 3.576188087463379, + 3.4640488624572754, + 3.486509323120117, + 3.421616554260254, + 3.3968722820281982, + 3.5471489429473877, + 3.612826347351074, + 3.4048686027526855, + 3.4600014686584473, + 3.5944011211395264, + 3.441009044647217, + 3.4917211532592773, + 3.4038002490997314, + 3.3960132598876953, + 3.612839937210083, + 3.4852070808410645, + 3.4980862140655518, + 3.5292418003082275, + 3.5159029960632324, + 3.5009045600891113, + 3.502767324447632, + 3.3430306911468506, + 3.4611892700195312, + 3.596705198287964, + 3.4551234245300293, + 3.354820966720581, + 3.4927027225494385, + 3.516479969024658, + 3.527513027191162, + 3.5021564960479736, + 3.545165777206421, + 3.524855375289917, + 3.371204376220703, + 3.3599610328674316, + 3.504120349884033, + 3.502274990081787, + 3.485295534133911, + 3.5300657749176025, + 3.390794038772583, + 3.4719362258911133, + 3.296316146850586, + 3.3852458000183105, + 3.4903459548950195, + 3.4961371421813965, + 3.4793739318847656, + 3.5896058082580566, + 3.492971658706665, + 3.4108879566192627, + 3.367130994796753, + 3.441812038421631, + 3.452608823776245, + 3.465216875076294, + 3.460785150527954, + 3.432980537414551, + 3.425426483154297, + 3.499347686767578, + 3.4575788974761963, + 3.5842232704162598, + 3.461400032043457, + 3.3970019817352295, + 3.474654197692871, + 3.3484597206115723, + 3.3829615116119385, + 3.496553897857666, + 3.6167240142822266, + 3.497978925704956, + 3.2188782691955566, + 3.477658987045288, + 3.4478566646575928, + 3.3812220096588135, + 3.434662103652954, + 3.423785924911499, + 3.452618360519409, + 3.4333038330078125, + 3.4801106452941895, + 3.443646192550659, + 3.428640604019165, + 3.48775053024292, + 3.484764575958252, + 3.440858840942383, + 3.4629509449005127, + 3.4459309577941895, + 3.4902360439300537, + 3.5068423748016357, + 3.4073338508605957, + 3.4244775772094727, + 3.4945008754730225, + 3.4061295986175537, + 3.469632148742676, + 3.308159351348877, + 3.4795687198638916, + 3.3527016639709473, + 3.4692983627319336, + 3.223503589630127, + 3.5783231258392334, + 3.491564989089966, + 3.4679014682769775, + 3.464629650115967, + 3.312796115875244, + 3.4069013595581055, + 3.466031551361084, + 3.3701822757720947, + 3.391897678375244, + 3.701572895050049, + 3.467923402786255, + 3.3818163871765137, + 3.4751226902008057, + 3.4458658695220947, + 3.432312250137329, + 3.443967580795288, + 3.565676212310791, + 3.3398597240448, + 3.3056023120880127, + 3.359127998352051, + 3.4968960285186768, + 3.464775562286377, + 3.337639331817627, + 3.52020001411438, + 3.3557636737823486, + 3.4192700386047363, + 3.4803647994995117, + 3.2954511642456055, + 3.487527847290039, + 3.481107234954834, + 3.481787919998169, + 3.2764647006988525, + 3.4465606212615967, + 3.456310749053955, + 3.4901745319366455, + 3.4686336517333984, + 3.3504652976989746, + 3.458404302597046, + 3.399256706237793, + 3.4269001483917236, + 3.3228487968444824, + 3.6667098999023438, + 3.3373420238494873, + 3.3904597759246826, + 3.4807231426239014, + 3.452732801437378, + 3.5153839588165283, + 3.515288829803467, + 3.383227586746216, + 3.307990789413452, + 3.310662031173706, + 3.4473886489868164, + 3.419536590576172, + 3.2907989025115967, + 3.4480504989624023, + 3.3914694786071777, + 3.2903923988342285, + 3.320159673690796, + 3.450906276702881, + 3.503052234649658, + 3.478200674057007, + 3.3820407390594482, + 3.4174249172210693, + 3.417149782180786, + 3.4980673789978027, + 3.3967695236206055, + 3.3894803524017334, + 3.620716094970703, + 3.4175806045532227, + 3.4342825412750244, + 3.4979641437530518, + 3.2663471698760986, + 3.4115102291107178, + 3.3076717853546143, + 3.4185287952423096, + 3.25527286529541, + 3.2792043685913086, + 3.4068186283111572, + 3.2185909748077393, + 3.4354248046875, + 3.2832190990448, + 3.4352760314941406, + 3.3097620010375977, + 3.487236738204956, + 3.3962652683258057, + 3.3890857696533203, + 3.4227523803710938, + 3.2160027027130127, + 3.488295793533325, + 3.4320294857025146, + 3.460616111755371, + 3.3420491218566895, + 3.4079482555389404, + 3.339949369430542, + 3.427105188369751, + 3.4069228172302246, + 3.4642252922058105, + 3.436647653579712, + 3.4640181064605713, + 3.3194313049316406, + 3.4683568477630615, + 3.4425437450408936, + 3.154171943664551, + 3.309593915939331, + 3.437331199645996, + 3.4491288661956787, + 3.4119186401367188, + 3.3771119117736816, + 3.4054465293884277, + 3.4400737285614014, + 3.484772205352783, + 3.4789042472839355, + 3.405703544616699, + 3.3734753131866455, + 3.2756521701812744, + 3.3581976890563965, + 3.4383628368377686, + 3.4096267223358154, + 3.43623685836792, + 3.4704959392547607, + 3.4761996269226074, + 3.392638683319092, + 3.3581745624542236, + 3.3259024620056152, + 3.37501859664917, + 3.416497230529785, + 3.4726309776306152, + 3.367957592010498, + 3.3006772994995117, + 3.4831035137176514, + 3.561976909637451, + 3.2698769569396973, + 3.386141061782837, + 3.2861461639404297, + 3.35737681388855, + 3.362247943878174, + 3.324155330657959, + 3.438676357269287, + 3.3432061672210693, + 3.502131223678589, + 3.370908260345459, + 3.303349494934082, + 3.395493745803833, + 3.40907883644104, + 3.44269061088562, + 3.3691658973693848, + 3.445251941680908, + 3.230315923690796, + 3.3625574111938477, + 3.406846046447754, + 3.4570157527923584, + 3.3749122619628906, + 3.3564977645874023, + 3.3986120223999023, + 3.2193288803100586, + 3.37129545211792, + 3.4678940773010254, + 3.5033180713653564, + 3.375627040863037, + 3.3638498783111572, + 3.31640625, + 3.3711583614349365, + 3.208174705505371, + 3.366203546524048, + 3.3833179473876953, + 3.4339613914489746, + 3.409794330596924, + 3.3670363426208496, + 3.3389344215393066, + 3.4137821197509766, + 3.3404083251953125, + 3.308695077896118, + 3.301589012145996, + 3.306786298751831, + 3.3456249237060547, + 3.211641311645508, + 3.3772799968719482, + 3.374537706375122, + 3.482445478439331, + 3.25262188911438, + 3.4798712730407715, + 3.3352012634277344, + 3.3908402919769287, + 3.3594696521759033, + 3.319143056869507, + 3.363593339920044 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 109.53748148934524, + 72.92410496489119, + 60.371943994672634, + 53.350960292565844, + 48.89577266261622, + 46.216707971831944, + 43.89707091795752, + 42.38997664647952, + 40.46854040446494, + 38.857224043484685, + 38.00247245035326, + 36.839269623414566, + 36.04048183012044, + 35.127872592022584, + 34.39299291430119, + 34.22084830171259, + 33.69993142435428, + 33.502802920991094, + 33.417694528499496, + 33.40784258455367 + ], + "best_ppl": 33.40784258455367, + "train_time_s": 34735.24209737778 +} \ No newline at end of file diff --git a/060_sparse_mol/top6of7_best.pt b/060_sparse_mol/top6of7_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..86f69ec5c71f62b9db390cccd15c2a5096616ed2 --- /dev/null +++ b/060_sparse_mol/top6of7_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c0ada3b0c1bc06cd39acaa7f5f70c2f51daa56ebc609d37d709bcfa9303226e +size 343197825 diff --git a/060_sparse_mol/top6of7_results.json b/060_sparse_mol/top6of7_results.json new file mode 100644 index 0000000000000000000000000000000000000000..93cb592d6be4594be9051908de4b20c7c290316a --- /dev/null +++ b/060_sparse_mol/top6of7_results.json @@ -0,0 +1,2057 @@ +{ + "label": "top6of7", + "top_k_blocks": 6, + "num_thin_blocks": 7, + "total_params": 85676160, + "router_params": 57344, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 16.485641479492188, + 13.816513061523438, + 7.903622627258301, + 7.2147440910339355, + 7.092496395111084, + 6.948383331298828, + 6.856039524078369, + 6.709964275360107, + 6.605602741241455, + 6.369322299957275, + 6.375448703765869, + 6.265221118927002, + 6.232246398925781, + 6.053356170654297, + 6.119626045227051, + 5.862091541290283, + 5.843300819396973, + 5.88913106918335, + 5.682739734649658, + 5.747263431549072, + 5.55800724029541, + 5.512547016143799, + 5.474531173706055, + 5.436607360839844, + 5.429177761077881, + 5.285270690917969, + 5.231938362121582, + 5.273585319519043, + 5.221237659454346, + 5.2439284324646, + 5.165724754333496, + 5.1899943351745605, + 5.092450141906738, + 4.991771697998047, + 4.984202861785889, + 5.126115322113037, + 4.962502956390381, + 4.988138675689697, + 4.964372634887695, + 4.945426940917969, + 4.989257335662842, + 5.017624855041504, + 4.956503868103027, + 4.826070785522461, + 4.909543991088867, + 4.884881496429443, + 4.969825744628906, + 4.762185573577881, + 4.832625865936279, + 4.806126594543457, + 4.853095054626465, + 4.79286003112793, + 4.762192249298096, + 4.680446624755859, + 4.770298004150391, + 4.741377830505371, + 4.6832098960876465, + 4.6049323081970215, + 4.541823387145996, + 4.593840599060059, + 4.699113368988037, + 4.686642646789551, + 4.6895432472229, + 4.748103618621826, + 4.563568592071533, + 4.644852161407471, + 4.652817249298096, + 4.543781280517578, + 4.456375598907471, + 4.637475967407227, + 4.67241907119751, + 4.547382831573486, + 4.41170597076416, + 4.653614044189453, + 4.56707239151001, + 4.549140930175781, + 4.579623699188232, + 4.478034019470215, + 4.417269229888916, + 4.433919429779053, + 4.439769744873047, + 4.601526737213135, + 4.409531593322754, + 4.384232044219971, + 4.561067581176758, + 4.45630407333374, + 4.519096851348877, + 4.401474952697754, + 4.464888572692871, + 4.3998799324035645, + 4.3189263343811035, + 4.445921421051025, + 4.44039249420166, + 4.371018409729004, + 4.442080020904541, + 4.400763511657715, + 4.403384685516357, + 4.46990442276001, + 4.291653156280518, + 4.328330039978027, + 4.26122522354126, + 4.476418972015381, + 4.387432098388672, + 4.316651344299316, + 4.375579833984375, + 4.399216175079346, + 4.251662254333496, + 4.338657379150391, + 4.328047275543213, + 4.2035417556762695, + 4.212826251983643, + 4.2307047843933105, + 4.2403740882873535, + 4.380797863006592, + 4.3515191078186035, + 4.3412957191467285, + 4.308282375335693, + 4.251190662384033, + 4.242612361907959, + 4.074917793273926, + 4.22010612487793, + 4.192131996154785, + 4.168280601501465, + 4.304198741912842, + 4.224236011505127, + 4.275909900665283, + 4.247830867767334, + 4.119407653808594, + 4.212902545928955, + 4.27323055267334, + 4.1651611328125, + 4.24625825881958, + 4.176710605621338, + 4.185683250427246, + 4.243814468383789, + 3.9987905025482178, + 4.160077095031738, + 4.079373359680176, + 4.16512393951416, + 4.144629955291748, + 4.150649070739746, + 4.020811557769775, + 4.197044849395752, + 4.131960868835449, + 4.106512069702148, + 4.143116474151611, + 4.088163375854492, + 4.024701118469238, + 4.136808395385742, + 4.124164581298828, + 4.098514556884766, + 4.1210246086120605, + 4.066024303436279, + 4.166508674621582, + 4.139286518096924, + 4.065367698669434, + 4.0448713302612305, + 4.152475357055664, + 4.149377822875977, + 4.103628635406494, + 4.050685882568359, + 4.0984344482421875, + 4.0613932609558105, + 4.022505283355713, + 4.166879177093506, + 4.042094707489014, + 4.111185073852539, + 4.109675884246826, + 4.100444793701172, + 4.20803689956665, + 4.0955400466918945, + 4.133473873138428, + 4.103359699249268, + 4.115240573883057, + 4.135601043701172, + 4.0430498123168945, + 4.044389724731445, + 4.183032989501953, + 4.039207935333252, + 4.127887725830078, + 4.223071575164795, + 4.1908135414123535, + 4.101306438446045, + 4.044116497039795, + 4.08450984954834, + 4.179942607879639, + 4.073572635650635, + 4.058182239532471, + 4.0585784912109375, + 4.281918048858643, + 4.071489334106445, + 3.9596848487854004, + 4.121945381164551, + 3.9549829959869385, + 3.959620237350464, + 4.085546016693115, + 3.959376811981201, + 4.039037227630615, + 4.066827297210693, + 4.013848781585693, + 4.0856451988220215, + 3.999469518661499, + 4.031608581542969, + 3.9926764965057373, + 4.039815902709961, + 4.042984485626221, + 4.048748970031738, + 4.111429691314697, + 4.130058288574219, + 4.012789726257324, + 4.1404218673706055, + 4.0908637046813965, + 4.145348072052002, + 3.959430456161499, + 4.1371169090271, + 3.9162628650665283, + 4.095886707305908, + 4.087244033813477, + 4.0862932205200195, + 4.010039329528809, + 3.9439022541046143, + 4.030258655548096, + 4.0112833976745605, + 3.9168622493743896, + 3.9949071407318115, + 4.110103607177734, + 4.109832286834717, + 3.9814326763153076, + 3.946559190750122, + 4.051226615905762, + 4.03322696685791, + 3.9428200721740723, + 4.007949352264404, + 4.126665115356445, + 4.02733850479126, + 4.010913372039795, + 4.024245738983154, + 4.077503681182861, + 3.9424707889556885, + 4.029839992523193, + 3.960773468017578, + 3.998406171798706, + 3.898933172225952, + 4.069214344024658, + 4.080320358276367, + 3.9138894081115723, + 3.974640369415283, + 4.034690856933594, + 3.8516697883605957, + 3.963392496109009, + 3.970787763595581, + 4.072479248046875, + 3.991325855255127, + 3.7734122276306152, + 3.834458589553833, + 3.8919899463653564, + 3.843630313873291, + 3.832206964492798, + 3.9256484508514404, + 4.015781879425049, + 3.9198927879333496, + 3.7941856384277344, + 3.929063320159912, + 3.8442890644073486, + 3.7227365970611572, + 3.872596502304077, + 3.963620901107788, + 3.891381025314331, + 3.9244744777679443, + 3.826050281524658, + 3.8565540313720703, + 3.865072250366211, + 3.9422149658203125, + 3.8870677947998047, + 3.8653512001037598, + 3.9000942707061768, + 3.854240894317627, + 3.795825958251953, + 3.7905232906341553, + 3.8029446601867676, + 3.8615596294403076, + 3.82546067237854, + 3.9399254322052, + 3.877295970916748, + 3.8901941776275635, + 3.788890838623047, + 3.830850601196289, + 3.8913381099700928, + 3.868973970413208, + 3.9366307258605957, + 3.8341212272644043, + 3.720587730407715, + 3.7246358394622803, + 3.702507734298706, + 3.8931331634521484, + 3.8219265937805176, + 3.89785099029541, + 3.720416784286499, + 3.853384494781494, + 3.999049663543701, + 3.817211389541626, + 3.8162033557891846, + 3.898616075515747, + 3.771568775177002, + 3.734201669692993, + 4.0187153816223145, + 3.893197536468506, + 3.818565607070923, + 3.730680227279663, + 3.8156371116638184, + 3.8084466457366943, + 3.8000359535217285, + 4.006218910217285, + 3.8480353355407715, + 3.927824020385742, + 3.783222198486328, + 3.9126832485198975, + 3.6794850826263428, + 3.88223934173584, + 3.7626781463623047, + 3.83103346824646, + 3.68617582321167, + 3.883327007293701, + 3.7750041484832764, + 3.7857723236083984, + 3.809051513671875, + 3.72491192817688, + 3.903034210205078, + 3.88434100151062, + 3.8086695671081543, + 3.746958017349243, + 3.787236213684082, + 3.722322702407837, + 3.8231542110443115, + 3.790578842163086, + 3.866828680038452, + 3.9278597831726074, + 3.760305404663086, + 3.6489412784576416, + 3.8414306640625, + 3.7932052612304688, + 3.6777231693267822, + 3.7637078762054443, + 3.8778061866760254, + 3.783148765563965, + 3.906148672103882, + 3.8089957237243652, + 3.8197686672210693, + 3.9247822761535645, + 3.7635960578918457, + 3.8130500316619873, + 3.8377113342285156, + 3.910942316055298, + 3.7825496196746826, + 3.9260122776031494, + 3.8546829223632812, + 3.7105298042297363, + 3.802152395248413, + 3.7998404502868652, + 3.7181758880615234, + 3.680884838104248, + 3.7684831619262695, + 3.8675951957702637, + 3.880615711212158, + 3.7496256828308105, + 3.718268394470215, + 3.7218568325042725, + 3.7087879180908203, + 3.848403215408325, + 3.7493441104888916, + 3.815281867980957, + 3.895829439163208, + 3.8139188289642334, + 3.8277370929718018, + 3.9013376235961914, + 3.709305763244629, + 3.6955325603485107, + 3.9623537063598633, + 3.810459613800049, + 3.795172929763794, + 3.5754146575927734, + 3.683849334716797, + 3.7440028190612793, + 3.749375581741333, + 3.792454719543457, + 3.589272975921631, + 3.752051830291748, + 3.673677444458008, + 3.6661770343780518, + 3.657081365585327, + 3.724924325942993, + 3.6058249473571777, + 3.6540026664733887, + 3.714500904083252, + 3.783062696456909, + 3.692859411239624, + 3.6519181728363037, + 3.7794694900512695, + 3.6902828216552734, + 3.7046127319335938, + 3.5706658363342285, + 3.7263431549072266, + 3.642637014389038, + 3.59249210357666, + 3.806030511856079, + 3.6166529655456543, + 3.7533621788024902, + 3.68338942527771, + 3.6273725032806396, + 3.643373727798462, + 3.6382594108581543, + 3.7706594467163086, + 3.7887351512908936, + 3.6750781536102295, + 3.7807962894439697, + 3.743194103240967, + 3.7965171337127686, + 3.736609935760498, + 3.635796070098877, + 3.8410534858703613, + 3.7794227600097656, + 3.5002057552337646, + 3.7118518352508545, + 3.724432945251465, + 3.730510711669922, + 3.6883182525634766, + 3.726212739944458, + 3.683344841003418, + 3.609556198120117, + 3.676527261734009, + 3.8000309467315674, + 3.729355812072754, + 3.7412359714508057, + 3.708775281906128, + 3.7011148929595947, + 3.7528276443481445, + 3.6879994869232178, + 3.597609519958496, + 3.7752485275268555, + 3.8053066730499268, + 3.7606794834136963, + 3.701564073562622, + 3.684413194656372, + 3.7382702827453613, + 3.6432347297668457, + 3.906196355819702, + 3.7014198303222656, + 3.8469574451446533, + 3.782230854034424, + 3.8083884716033936, + 3.781388759613037, + 3.71077036857605, + 3.6212382316589355, + 3.6633989810943604, + 3.7033722400665283, + 3.740920066833496, + 3.645155191421509, + 3.923830270767212, + 3.7711799144744873, + 3.6111056804656982, + 3.641371488571167, + 3.632779598236084, + 3.723280191421509, + 3.5756728649139404, + 3.675485610961914, + 3.6622891426086426, + 3.645496129989624, + 3.8835065364837646, + 3.691628932952881, + 3.6261255741119385, + 3.662998676300049, + 3.657729387283325, + 3.7667222023010254, + 3.500417470932007, + 3.646080493927002, + 3.671078681945801, + 3.595557928085327, + 3.744323492050171, + 3.6580564975738525, + 3.704582691192627, + 3.728837728500366, + 3.693051338195801, + 3.7861993312835693, + 3.820272922515869, + 3.6800284385681152, + 3.696026086807251, + 3.7596757411956787, + 3.725605010986328, + 3.559396743774414, + 3.6504921913146973, + 3.634284734725952, + 3.7554831504821777, + 3.7110140323638916, + 3.637408971786499, + 3.5653562545776367, + 3.605159044265747, + 3.8742172718048096, + 3.649749755859375, + 3.5720760822296143, + 3.7874579429626465, + 3.589857339859009, + 3.749556541442871, + 3.7372491359710693, + 3.6935343742370605, + 3.64862060546875, + 3.6975419521331787, + 3.672342538833618, + 3.6333565711975098, + 3.463594436645508, + 3.6201608180999756, + 3.616792678833008, + 3.512505292892456, + 3.6606407165527344, + 3.4707093238830566, + 3.4441473484039307, + 3.5000479221343994, + 3.5958452224731445, + 3.6233432292938232, + 3.6762661933898926, + 3.647826910018921, + 3.5108859539031982, + 3.4797022342681885, + 3.5346643924713135, + 3.621243476867676, + 3.4449191093444824, + 3.6291232109069824, + 3.6746482849121094, + 3.3355298042297363, + 3.4841670989990234, + 3.668792486190796, + 3.654094934463501, + 3.546323776245117, + 3.51603364944458, + 3.638266086578369, + 3.6021628379821777, + 3.5793867111206055, + 3.59128737449646, + 3.662341833114624, + 3.5325582027435303, + 3.6138992309570312, + 3.5571889877319336, + 3.547783136367798, + 3.5022027492523193, + 3.5543510913848877, + 3.5810654163360596, + 3.639509439468384, + 3.4813361167907715, + 3.642144203186035, + 3.598933696746826, + 3.679056406021118, + 3.636864185333252, + 3.6351935863494873, + 3.4405691623687744, + 3.5333328247070312, + 3.4504618644714355, + 3.596872091293335, + 3.635420560836792, + 3.6332614421844482, + 3.573803424835205, + 3.615396738052368, + 3.630333423614502, + 3.487377643585205, + 3.655141592025757, + 3.416515827178955, + 3.4640767574310303, + 3.5821874141693115, + 3.5280468463897705, + 3.590703010559082, + 3.48728084564209, + 3.5574541091918945, + 3.541611909866333, + 3.512355089187622, + 3.4636359214782715, + 3.3849072456359863, + 3.6188220977783203, + 3.558088541030884, + 3.6021358966827393, + 3.6971147060394287, + 3.663870334625244, + 3.483180284500122, + 3.5457546710968018, + 3.696112632751465, + 3.601034164428711, + 3.729675769805908, + 3.599165678024292, + 3.508976459503174, + 3.4958531856536865, + 3.578624725341797, + 3.677683115005493, + 3.6692254543304443, + 3.5155813694000244, + 3.510786533355713, + 3.705460548400879, + 3.6529877185821533, + 3.7344202995300293, + 3.696983814239502, + 3.6262049674987793, + 3.665395736694336, + 3.4537055492401123, + 3.560804843902588, + 3.540099620819092, + 3.5529978275299072, + 3.591914653778076, + 3.5418941974639893, + 3.464367628097534, + 3.4771018028259277, + 3.5026402473449707, + 3.61377215385437, + 3.611236095428467, + 3.6833174228668213, + 3.64039945602417, + 3.7532241344451904, + 3.5897605419158936, + 3.5899932384490967, + 3.4926698207855225, + 3.4870779514312744, + 3.517601728439331, + 3.483934164047241, + 3.5824599266052246, + 3.5421142578125, + 3.5170719623565674, + 3.6504688262939453, + 3.591219425201416, + 3.500899076461792, + 3.4548585414886475, + 3.661586284637451, + 3.4831337928771973, + 3.670701503753662, + 3.508829355239868, + 3.526715040206909, + 3.4130654335021973, + 3.564000129699707, + 3.3889331817626953, + 3.4552011489868164, + 3.572126626968384, + 3.547776699066162, + 3.3779046535491943, + 3.47015643119812, + 3.4930686950683594, + 3.5395567417144775, + 3.432272434234619, + 3.3621912002563477, + 3.551396369934082, + 3.3795807361602783, + 3.532705783843994, + 3.3934481143951416, + 3.536381483078003, + 3.4690029621124268, + 3.4768991470336914, + 3.4971258640289307, + 3.435387372970581, + 3.4556939601898193, + 3.5466339588165283, + 3.417431592941284, + 3.4609827995300293, + 3.516772985458374, + 3.509674549102783, + 3.5220651626586914, + 3.4369893074035645, + 3.4074816703796387, + 3.565593957901001, + 3.464841604232788, + 3.4908089637756348, + 3.422877311706543, + 3.49346923828125, + 3.468096971511841, + 3.58087420463562, + 3.5263214111328125, + 3.3883094787597656, + 3.4772839546203613, + 3.4225587844848633, + 3.481553792953491, + 3.4881722927093506, + 3.4538216590881348, + 3.471592903137207, + 3.4446640014648438, + 3.4282901287078857, + 3.4260520935058594, + 3.60921311378479, + 3.4669904708862305, + 3.5065252780914307, + 3.4913241863250732, + 3.5186052322387695, + 3.5992188453674316, + 3.3846161365509033, + 3.3786611557006836, + 3.5617008209228516, + 3.612859010696411, + 3.4776415824890137, + 3.429717779159546, + 3.484994411468506, + 3.550825595855713, + 3.4761109352111816, + 3.4458062648773193, + 3.4556772708892822, + 3.498689651489258, + 3.369939088821411, + 3.4353535175323486, + 3.4783060550689697, + 3.470745086669922, + 3.5220530033111572, + 3.451097011566162, + 3.5413098335266113, + 3.5520431995391846, + 3.4242424964904785, + 3.514617919921875, + 3.464343786239624, + 3.4871580600738525, + 3.4230732917785645, + 3.444902181625366, + 3.5178096294403076, + 3.392486572265625, + 3.450711727142334, + 3.509495496749878, + 3.5477428436279297, + 3.588207721710205, + 3.481551170349121, + 3.561887264251709, + 3.5361883640289307, + 3.5259618759155273, + 3.5571937561035156, + 3.4429678916931152, + 3.464437961578369, + 3.4100944995880127, + 3.3806583881378174, + 3.5209081172943115, + 3.5743777751922607, + 3.368279457092285, + 3.429875612258911, + 3.571793556213379, + 3.403421401977539, + 3.4633443355560303, + 3.3769540786743164, + 3.375478506088257, + 3.579630136489868, + 3.4421703815460205, + 3.4653401374816895, + 3.5002031326293945, + 3.481450319290161, + 3.4749252796173096, + 3.4724433422088623, + 3.318389654159546, + 3.414212465286255, + 3.5611841678619385, + 3.441154718399048, + 3.330284595489502, + 3.462883949279785, + 3.4876558780670166, + 3.49766206741333, + 3.468580961227417, + 3.52042818069458, + 3.4987101554870605, + 3.355897903442383, + 3.336367607116699, + 3.4705116748809814, + 3.46832275390625, + 3.464872360229492, + 3.4916155338287354, + 3.3564507961273193, + 3.442460298538208, + 3.2679734230041504, + 3.3570446968078613, + 3.4681549072265625, + 3.468320369720459, + 3.457514524459839, + 3.5576021671295166, + 3.474806070327759, + 3.374225378036499, + 3.347001791000366, + 3.4189460277557373, + 3.419548749923706, + 3.439340591430664, + 3.4508488178253174, + 3.417494058609009, + 3.391014814376831, + 3.467481851577759, + 3.431281805038452, + 3.545490264892578, + 3.415825128555298, + 3.3724489212036133, + 3.472534418106079, + 3.34055495262146, + 3.3646938800811768, + 3.466336727142334, + 3.578944683074951, + 3.4528286457061768, + 3.1959402561187744, + 3.448507308959961, + 3.4302878379821777, + 3.343090534210205, + 3.42415189743042, + 3.4086203575134277, + 3.42848539352417, + 3.406418800354004, + 3.452970027923584, + 3.422635555267334, + 3.406503200531006, + 3.4706709384918213, + 3.448930501937866, + 3.420741081237793, + 3.4171011447906494, + 3.425187349319458, + 3.4642436504364014, + 3.473536491394043, + 3.3677072525024414, + 3.407740831375122, + 3.470449447631836, + 3.382394552230835, + 3.450855016708374, + 3.274611473083496, + 3.449477195739746, + 3.3140649795532227, + 3.4366519451141357, + 3.195204496383667, + 3.531001567840576, + 3.4773247241973877, + 3.4530608654022217, + 3.448423147201538, + 3.2905943393707275, + 3.367600917816162, + 3.441041946411133, + 3.350954294204712, + 3.3754312992095947, + 3.672234058380127, + 3.439598798751831, + 3.3439722061157227, + 3.4563751220703125, + 3.395822763442993, + 3.4039340019226074, + 3.4143834114074707, + 3.539299964904785, + 3.3182172775268555, + 3.277531862258911, + 3.3344898223876953, + 3.4707558155059814, + 3.4305319786071777, + 3.2929694652557373, + 3.4842638969421387, + 3.3350000381469727, + 3.378387212753296, + 3.4672346115112305, + 3.275881767272949, + 3.4537646770477295, + 3.4648640155792236, + 3.4585814476013184, + 3.2728216648101807, + 3.430274486541748, + 3.415858268737793, + 3.468522787094116, + 3.4352667331695557, + 3.314032793045044, + 3.432157516479492, + 3.374173402786255, + 3.4012839794158936, + 3.2937233448028564, + 3.6380138397216797, + 3.3027381896972656, + 3.363006114959717, + 3.4538183212280273, + 3.4262473583221436, + 3.4858081340789795, + 3.482412099838257, + 3.3630073070526123, + 3.283296585083008, + 3.2827155590057373, + 3.426656484603882, + 3.3910603523254395, + 3.2619283199310303, + 3.3940839767456055, + 3.355293035507202, + 3.2589004039764404, + 3.2874996662139893, + 3.430046319961548, + 3.4658172130584717, + 3.4418578147888184, + 3.366217613220215, + 3.393068313598633, + 3.368488073348999, + 3.476469039916992, + 3.3781795501708984, + 3.3583016395568848, + 3.590616226196289, + 3.4003734588623047, + 3.4078023433685303, + 3.4649407863616943, + 3.2307443618774414, + 3.3859617710113525, + 3.287936210632324, + 3.3792266845703125, + 3.2289981842041016, + 3.250739097595215, + 3.396310567855835, + 3.1993818283081055, + 3.4098761081695557, + 3.2619423866271973, + 3.4081530570983887, + 3.2856903076171875, + 3.466977596282959, + 3.3736493587493896, + 3.3562171459198, + 3.405221700668335, + 3.2138187885284424, + 3.472566843032837, + 3.406830072402954, + 3.4307215213775635, + 3.315436363220215, + 3.3717825412750244, + 3.317056894302368, + 3.3824691772460938, + 3.379692554473877, + 3.4524827003479004, + 3.410184860229492, + 3.4323532581329346, + 3.289640426635742, + 3.437995672225952, + 3.407728910446167, + 3.140305280685425, + 3.2825982570648193, + 3.404710054397583, + 3.4355976581573486, + 3.393214464187622, + 3.356555461883545, + 3.3707220554351807, + 3.4023149013519287, + 3.4570939540863037, + 3.4402990341186523, + 3.3667731285095215, + 3.3425416946411133, + 3.249764919281006, + 3.3414127826690674, + 3.413076400756836, + 3.3769798278808594, + 3.3910815715789795, + 3.453770637512207, + 3.4382829666137695, + 3.3563194274902344, + 3.3261477947235107, + 3.3137593269348145, + 3.353238821029663, + 3.3787670135498047, + 3.452059507369995, + 3.338787794113159, + 3.264967679977417, + 3.4624383449554443, + 3.5365514755249023, + 3.2393975257873535, + 3.3598947525024414, + 3.2607500553131104, + 3.338068723678589, + 3.3426997661590576, + 3.2881219387054443, + 3.4172301292419434, + 3.3044469356536865, + 3.4771225452423096, + 3.3406262397766113, + 3.267970561981201, + 3.3612561225891113, + 3.3899900913238525, + 3.413330078125, + 3.3367092609405518, + 3.4099438190460205, + 3.22224760055542, + 3.354994058609009, + 3.374211072921753, + 3.4189999103546143, + 3.344806432723999, + 3.340745210647583, + 3.3911492824554443, + 3.192996025085449, + 3.3466882705688477, + 3.4390599727630615, + 3.4766385555267334, + 3.3476195335388184, + 3.3419582843780518, + 3.2893927097320557, + 3.3271186351776123, + 3.1755082607269287, + 3.331629753112793, + 3.362999439239502, + 3.3936290740966797, + 3.3864288330078125, + 3.345912456512451, + 3.2984046936035156, + 3.3883767127990723, + 3.308375358581543, + 3.293670654296875, + 3.295557737350464, + 3.267348527908325, + 3.3230557441711426, + 3.1920363903045654, + 3.3539621829986572, + 3.3407375812530518, + 3.47037935256958, + 3.223825216293335, + 3.4570977687835693, + 3.30362868309021, + 3.3793654441833496, + 3.3504393100738525, + 3.290114402770996, + 3.3309128284454346 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 107.99717446189203, + 71.0436341687742, + 58.458383872879686, + 51.922507404235915, + 47.82383867278007, + 44.98559738309991, + 42.74613760114148, + 41.186008797286924, + 39.36934808874591, + 37.77085967469391, + 36.89671046841085, + 35.8367751364508, + 35.12330185902695, + 34.218415874574056, + 33.48772099234332, + 33.26818595931509, + 32.79618736479951, + 32.60435366127775, + 32.50602717752871, + 32.490073206311905 + ], + "best_ppl": 32.490073206311905, + "train_time_s": 32587.863648176193 +} \ No newline at end of file diff --git a/060_sparse_mol/top7of7_best.pt b/060_sparse_mol/top7of7_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2447c81ba646e06b576cf5abf91a0d95ee91353 --- /dev/null +++ b/060_sparse_mol/top7of7_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0be82b870fef83596a34795cda24100bf8f98c4374c75d8eba20095cdd204890 +size 343197825 diff --git a/060_sparse_mol/top7of7_results.json b/060_sparse_mol/top7of7_results.json new file mode 100644 index 0000000000000000000000000000000000000000..c4501db2cb75ced9805ff53ba6ddf296c346ea84 --- /dev/null +++ b/060_sparse_mol/top7of7_results.json @@ -0,0 +1,1037 @@ +{ + "label": "top7of7", + "top_k_blocks": 7, + "num_thin_blocks": 7, + "total_params": 85676160, + "router_params": 57344, + "steps": [ + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 3.6047894954681396, + 3.6993038654327393, + 3.565004348754883, + 3.6105124950408936, + 3.6123106479644775, + 3.6396982669830322, + 3.700716972351074, + 3.572094202041626, + 3.58168625831604, + 3.465416669845581, + 3.648641347885132, + 3.6161768436431885, + 3.6411585807800293, + 3.594578266143799, + 3.772310972213745, + 3.61519455909729, + 3.5753488540649414, + 3.672722816467285, + 3.5787603855133057, + 3.704827070236206, + 3.553386688232422, + 3.643362045288086, + 3.600127696990967, + 3.635768175125122, + 3.7010903358459473, + 3.479304075241089, + 3.5027849674224854, + 3.5799033641815186, + 3.7427585124969482, + 3.720857620239258, + 3.6523094177246094, + 3.6090149879455566, + 3.5740737915039062, + 3.524324893951416, + 3.568737268447876, + 3.7013790607452393, + 3.537433624267578, + 3.6166186332702637, + 3.575144052505493, + 3.5795493125915527, + 3.6436901092529297, + 3.6849544048309326, + 3.6303558349609375, + 3.5599141120910645, + 3.618220329284668, + 3.649339199066162, + 3.6719353199005127, + 3.608597755432129, + 3.6034414768218994, + 3.644973039627075, + 3.7053987979888916, + 3.5612287521362305, + 3.6221065521240234, + 3.5832297801971436, + 3.662543296813965, + 3.622390031814575, + 3.605534791946411, + 3.505629539489746, + 3.4332544803619385, + 3.457402229309082, + 3.7103679180145264, + 3.6559293270111084, + 3.6145853996276855, + 3.700187921524048, + 3.5181546211242676, + 3.6114683151245117, + 3.690737247467041, + 3.534231185913086, + 3.4626622200012207, + 3.617701768875122, + 3.675398349761963, + 3.564401388168335, + 3.408029556274414, + 3.684906482696533, + 3.6507482528686523, + 3.6336379051208496, + 3.6268320083618164, + 3.517862558364868, + 3.4799389839172363, + 3.55535888671875, + 3.5760338306427, + 3.684220790863037, + 3.519813299179077, + 3.5127058029174805, + 3.7156314849853516, + 3.565264940261841, + 3.691012382507324, + 3.485442876815796, + 3.5730552673339844, + 3.573430061340332, + 3.4699878692626953, + 3.6301281452178955, + 3.6079154014587402, + 3.5304558277130127, + 3.6139869689941406, + 3.5720579624176025, + 3.6095244884490967, + 3.637737512588501, + 3.4561421871185303, + 3.5779507160186768, + 3.4767539501190186, + 3.674819231033325, + 3.618842840194702, + 3.5455942153930664, + 3.5922892093658447, + 3.64573073387146, + 3.494380235671997, + 3.569498062133789, + 3.6034154891967773, + 3.4193761348724365, + 3.4618875980377197, + 3.514803647994995, + 3.527740716934204, + 3.662916660308838, + 3.641235113143921, + 3.6043412685394287, + 3.5839455127716064, + 3.5173227787017822, + 3.5276899337768555, + 3.3527300357818604, + 3.541177749633789, + 3.4817299842834473, + 3.4579362869262695, + 3.5749006271362305, + 3.503228187561035, + 3.546096086502075, + 3.4908573627471924, + 3.3984909057617188, + 3.4982097148895264, + 3.53537917137146, + 3.453947067260742, + 3.5463128089904785, + 3.5103604793548584, + 3.484760284423828, + 3.5450408458709717, + 3.2888200283050537, + 3.448843240737915, + 3.414591073989868, + 3.4906551837921143, + 3.4760172367095947, + 3.4445343017578125, + 3.3349273204803467, + 3.5004701614379883, + 3.454974412918091, + 3.4419121742248535, + 3.479412078857422, + 3.4613866806030273, + 3.396592378616333, + 3.5132274627685547, + 3.4490675926208496, + 3.4333717823028564, + 3.477588176727295, + 3.4272854328155518, + 3.5272538661956787, + 3.5012011528015137, + 3.415804624557495, + 3.425288200378418, + 3.506166696548462, + 3.51485013961792, + 3.4694957733154297, + 3.4009647369384766, + 3.4612927436828613, + 3.444624185562134, + 3.400623321533203, + 3.5664286613464355, + 3.4243454933166504, + 3.4725799560546875, + 3.5044188499450684, + 3.4672985076904297, + 3.6243035793304443, + 3.4972314834594727, + 3.556629180908203, + 3.5108258724212646, + 3.5340232849121094, + 3.5228261947631836, + 3.4680745601654053, + 3.4202661514282227, + 3.578819751739502, + 3.435595989227295, + 3.5742592811584473, + 3.6164169311523438, + 3.610183000564575, + 3.503108263015747, + 3.4532930850982666, + 3.4926915168762207, + 3.572758197784424, + 3.506166458129883, + 3.464571952819824, + 3.4863245487213135, + 3.698209524154663, + 3.4689033031463623, + 3.4100189208984375, + 3.565509557723999, + 3.3950130939483643, + 3.3952112197875977, + 3.519914150238037, + 3.4071037769317627, + 3.4481070041656494, + 3.5052244663238525, + 3.4488439559936523, + 3.5217840671539307, + 3.443250894546509, + 3.497906446456909, + 3.448428153991699, + 3.498349905014038, + 3.4729950428009033, + 3.5143675804138184, + 3.561389207839966, + 3.592231273651123, + 3.445976972579956, + 3.5770697593688965, + 3.5331268310546875, + 3.6093242168426514, + 3.4082958698272705, + 3.5685205459594727, + 3.3818647861480713, + 3.541846752166748, + 3.559593915939331, + 3.5337319374084473, + 3.4847753047943115, + 3.4230730533599854, + 3.4838809967041016, + 3.4979279041290283, + 3.4050822257995605, + 3.4663140773773193, + 3.567617654800415, + 3.573833465576172, + 3.4887032508850098, + 3.4106805324554443, + 3.5252954959869385, + 3.5380990505218506, + 3.4315693378448486, + 3.507091760635376, + 3.623835563659668, + 3.5062427520751953, + 3.4876930713653564, + 3.524904727935791, + 3.545431613922119, + 3.4410862922668457, + 3.519378900527954, + 3.4668703079223633, + 3.4849467277526855, + 3.3745956420898438, + 3.5620553493499756, + 3.583761692047119, + 3.422043561935425, + 3.4745452404022217, + 3.5275394916534424, + 3.3688037395477295, + 3.4852328300476074, + 3.5070548057556152, + 3.607619047164917, + 3.4563405513763428, + 3.2956109046936035, + 3.348223924636841, + 3.4391043186187744, + 3.382824659347534, + 3.3775181770324707, + 3.4430735111236572, + 3.539961099624634, + 3.469151496887207, + 3.3350963592529297, + 3.44670033454895, + 3.3906266689300537, + 3.238572359085083, + 3.3707776069641113, + 3.4808077812194824, + 3.4216692447662354, + 3.4595232009887695, + 3.3167614936828613, + 3.368450880050659, + 3.39963436126709, + 3.432493209838867, + 3.4086878299713135, + 3.391054391860962, + 3.430894374847412, + 3.3872146606445312, + 3.3261678218841553, + 3.3321011066436768, + 3.327035665512085, + 3.417994499206543, + 3.3894076347351074, + 3.4505717754364014, + 3.408547878265381, + 3.4319522380828857, + 3.299955129623413, + 3.403306484222412, + 3.4153168201446533, + 3.4087536334991455, + 3.4819583892822266, + 3.3357748985290527, + 3.2713403701782227, + 3.2500174045562744, + 3.2438433170318604, + 3.404339551925659, + 3.33583927154541, + 3.46791410446167, + 3.2472622394561768, + 3.3931264877319336, + 3.561197280883789, + 3.3598227500915527, + 3.382040500640869, + 3.427506923675537, + 3.326164722442627, + 3.2943434715270996, + 3.539869785308838, + 3.4468188285827637, + 3.377946376800537, + 3.2752416133880615, + 3.3668413162231445, + 3.33970046043396, + 3.3523337841033936, + 3.5345637798309326, + 3.3993072509765625, + 3.497535228729248, + 3.3597021102905273, + 3.4834866523742676, + 3.2411155700683594, + 3.4283692836761475, + 3.350255250930786, + 3.3779642581939697, + 3.240394353866577, + 3.433412551879883, + 3.316706657409668, + 3.3184380531311035, + 3.347144842147827, + 3.2897207736968994, + 3.4666876792907715, + 3.4547812938690186, + 3.3576807975769043, + 3.3313300609588623, + 3.3528292179107666, + 3.292353391647339, + 3.3646364212036133, + 3.340907573699951, + 3.450101375579834, + 3.4828991889953613, + 3.3438711166381836, + 3.2001771926879883, + 3.4008591175079346, + 3.3607194423675537, + 3.253312587738037, + 3.329962968826294, + 3.4637863636016846, + 3.3827223777770996, + 3.4572150707244873, + 3.370558500289917, + 3.3809893131256104, + 3.498342275619507, + 3.3121304512023926, + 3.387759208679199, + 3.4328453540802, + 3.5000054836273193, + 3.377579689025879, + 3.5173556804656982, + 3.4448435306549072, + 3.2904715538024902, + 3.3763973712921143, + 3.354003667831421, + 3.305856704711914, + 3.2607877254486084, + 3.3281524181365967, + 3.439178466796875, + 3.4529528617858887, + 3.3466031551361084, + 3.3389408588409424, + 3.314297676086426, + 3.289393901824951, + 3.4573006629943848, + 3.3677244186401367, + 3.4099502563476562, + 3.490086317062378, + 3.3977863788604736, + 3.4101717472076416, + 3.5103585720062256, + 3.2964704036712646, + 3.2536978721618652, + 3.5670158863067627, + 3.402815341949463, + 3.4501049518585205, + 3.251364231109619, + 3.3208446502685547, + 3.3974058628082275, + 3.3742613792419434, + 3.4442696571350098, + 3.2590441703796387, + 3.3659801483154297, + 3.3283016681671143, + 3.32228946685791, + 3.3110642433166504, + 3.363999605178833, + 3.2250680923461914, + 3.277282476425171, + 3.3687546253204346, + 3.417158842086792, + 3.33994197845459, + 3.3015329837799072, + 3.4113574028015137, + 3.3066041469573975, + 3.346778631210327, + 3.2174417972564697, + 3.3760454654693604, + 3.273662805557251, + 3.2519328594207764, + 3.4488680362701416, + 3.259140729904175, + 3.4088943004608154, + 3.3228347301483154, + 3.2460169792175293, + 3.2668068408966064, + 3.285118579864502, + 3.3994052410125732, + 3.427534818649292, + 3.329162359237671, + 3.405038833618164, + 3.381582260131836, + 3.42714524269104, + 3.3794009685516357, + 3.263575553894043, + 3.4468495845794678, + 3.396436929702759, + 3.1254873275756836, + 3.3398454189300537, + 3.357013702392578, + 3.382920026779175, + 3.3135108947753906, + 3.342726469039917, + 3.3173718452453613, + 3.2672183513641357, + 3.3025975227355957, + 3.4449620246887207, + 3.3733599185943604, + 3.3873345851898193, + 3.3513638973236084, + 3.3438007831573486, + 3.375730276107788, + 3.340934991836548, + 3.2501380443573, + 3.418151378631592, + 3.460921287536621, + 3.412815570831299, + 3.3304855823516846, + 3.326857328414917, + 3.3874247074127197, + 3.2846014499664307, + 3.534102439880371, + 3.3366453647613525, + 3.484722137451172, + 3.4127697944641113, + 3.445739507675171, + 3.4227073192596436, + 3.3560194969177246, + 3.276221752166748, + 3.31139874458313, + 3.3414061069488525, + 3.4011223316192627, + 3.2926652431488037, + 3.569795846939087, + 3.434107780456543, + 3.2619147300720215, + 3.2939231395721436, + 3.286386489868164, + 3.3729519844055176, + 3.235004186630249, + 3.331350803375244, + 3.340052843093872, + 3.294536828994751, + 3.5126681327819824, + 3.3326754570007324, + 3.282210111618042, + 3.3058462142944336, + 3.3040900230407715, + 3.4392476081848145, + 3.153639078140259, + 3.2735602855682373, + 3.3310508728027344, + 3.24472975730896, + 3.4218974113464355, + 3.3018155097961426, + 3.3506855964660645, + 3.398566484451294, + 3.3668341636657715, + 3.45011305809021, + 3.4781622886657715, + 3.327779769897461, + 3.3409438133239746, + 3.4115986824035645, + 3.3856163024902344, + 3.2213501930236816, + 3.319648504257202, + 3.281144857406616, + 3.409592628479004, + 3.3818511962890625, + 3.2886693477630615, + 3.226701498031616, + 3.268451452255249, + 3.532414436340332, + 3.29634428024292, + 3.2472362518310547, + 3.514526128768921 + ], + "val_steps": [ + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 37.15613980055124, + 35.97459484059456, + 35.14242321854828, + 34.38639963242825, + 33.69330804751239, + 33.31312871089495, + 32.97084290656807, + 32.804568964356584, + 32.706805005634614, + 32.690031547873566 + ], + "best_ppl": 32.690031547873566, + "train_time_s": 17406.359367132187 +} \ No newline at end of file diff --git a/061_dense_mol/dense_k10_top3_best.pt b/061_dense_mol/dense_k10_top3_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..97b4408eee36c4323e574e035003276e66175802 --- /dev/null +++ b/061_dense_mol/dense_k10_top3_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26fa906f5483895c15f487e7be3f5514663c07c35415fce1b5b7be6fa5ec6182 +size 551791864 diff --git a/061_dense_mol/dense_k5_all_best.pt b/061_dense_mol/dense_k5_all_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..54afbe44f4f40ea760c8b0a6a8977f2dbb0fb7df --- /dev/null +++ b/061_dense_mol/dense_k5_all_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec0c74ea826d047f6924bfdc90a1df4b1c3faa140d032967959e4b7bdfaa7380 +size 341270419 diff --git a/061_dense_mol/dense_k5_all_results.json b/061_dense_mol/dense_k5_all_results.json new file mode 100644 index 0000000000000000000000000000000000000000..8c2267ee249769cdf3ce291243ad4de3a7e9c6ec --- /dev/null +++ b/061_dense_mol/dense_k5_all_results.json @@ -0,0 +1,2059 @@ +{ + "label": "dense_k5_all", + "top_k_blocks": null, + "num_thin_blocks": 5, + "total_params": 85260288, + "router_params": 0, + "dense_ffn": true, + "d_ff_thin": 1024, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.070724487304688, + 13.612395286560059, + 7.997864723205566, + 7.571474075317383, + 7.389724254608154, + 7.2704243659973145, + 7.05997371673584, + 6.966979026794434, + 6.81105899810791, + 6.666530132293701, + 6.529507637023926, + 6.452744007110596, + 6.413489818572998, + 6.4176530838012695, + 6.2632222175598145, + 6.117846488952637, + 6.045698165893555, + 6.068695545196533, + 6.005083084106445, + 5.875415802001953, + 5.829569339752197, + 5.839130401611328, + 5.761042594909668, + 5.5799360275268555, + 5.590349197387695, + 5.595692157745361, + 5.5454421043396, + 5.5211615562438965, + 5.381040573120117, + 5.4563727378845215, + 5.433462619781494, + 5.364553928375244, + 5.38865327835083, + 5.396468162536621, + 5.196792125701904, + 5.319888591766357, + 5.302035808563232, + 5.190334320068359, + 5.220688819885254, + 5.1116485595703125, + 5.143195152282715, + 5.153780937194824, + 5.021317005157471, + 5.119734287261963, + 5.106103420257568, + 4.989871978759766, + 4.9881205558776855, + 4.941500663757324, + 5.120760440826416, + 5.073657512664795, + 4.953105926513672, + 4.967444896697998, + 4.9781341552734375, + 5.004116058349609, + 4.9917311668396, + 4.800647258758545, + 4.914721488952637, + 4.858126640319824, + 4.958495616912842, + 4.918061256408691, + 4.898271083831787, + 4.899853706359863, + 4.912131309509277, + 4.87312650680542, + 4.7826924324035645, + 4.842643737792969, + 4.701728343963623, + 4.748537063598633, + 4.688404560089111, + 4.801039218902588, + 4.652254104614258, + 4.709909439086914, + 4.719845771789551, + 4.770066738128662, + 4.67941427230835, + 4.811110019683838, + 4.733743190765381, + 4.747159957885742, + 4.746601104736328, + 4.760013580322266, + 4.705589771270752, + 4.762825965881348, + 4.501718997955322, + 4.633505344390869, + 4.7344136238098145, + 4.674438953399658, + 4.73960018157959, + 4.646899223327637, + 4.649718761444092, + 4.596114158630371, + 4.602783203125, + 4.569770336151123, + 4.555212497711182, + 4.536740303039551, + 4.560263156890869, + 4.414535045623779, + 4.650325775146484, + 4.502536773681641, + 4.510170936584473, + 4.581280708312988, + 4.476032257080078, + 4.451568126678467, + 4.4535064697265625, + 4.535117149353027, + 4.571606159210205, + 4.502740383148193, + 4.3769001960754395, + 4.508957386016846, + 4.404378890991211, + 4.505998134613037, + 4.40771484375, + 4.421051025390625, + 4.407856464385986, + 4.515388488769531, + 4.404118537902832, + 4.343591213226318, + 4.526661396026611, + 4.3610944747924805, + 4.470168113708496, + 4.397058486938477, + 4.409600257873535, + 4.430878639221191, + 4.354002475738525, + 4.553160190582275, + 4.298649311065674, + 4.443589210510254, + 4.4565582275390625, + 4.305356025695801, + 4.326000690460205, + 4.285740375518799, + 4.358609676361084, + 4.348220348358154, + 4.247827529907227, + 4.331943511962891, + 4.184874534606934, + 4.273941993713379, + 4.382309913635254, + 4.27902364730835, + 4.239485740661621, + 4.306357383728027, + 4.408023834228516, + 4.278514862060547, + 4.314990997314453, + 4.331496715545654, + 4.364373683929443, + 4.405332565307617, + 4.332542419433594, + 4.3115739822387695, + 4.282298564910889, + 4.341415882110596, + 4.270750999450684, + 4.255738735198975, + 4.3188581466674805, + 4.2698655128479, + 4.1652655601501465, + 4.212048053741455, + 4.313793182373047, + 4.317106246948242, + 4.246700286865234, + 4.200335502624512, + 4.141587257385254, + 4.1476149559021, + 4.345272541046143, + 4.161682605743408, + 4.367251873016357, + 4.161456108093262, + 4.133021831512451, + 4.26413631439209, + 4.290573596954346, + 4.188206672668457, + 4.163483619689941, + 4.218229293823242, + 4.156885623931885, + 4.262388706207275, + 4.213654518127441, + 4.239328861236572, + 4.222198009490967, + 4.154384613037109, + 4.160682678222656, + 4.069400310516357, + 4.144484043121338, + 4.144293308258057, + 4.198212146759033, + 4.021173000335693, + 4.258097171783447, + 4.155553340911865, + 4.1547651290893555, + 4.174774646759033, + 4.082302570343018, + 4.1276350021362305, + 4.1710944175720215, + 4.11665153503418, + 4.133130073547363, + 4.109659671783447, + 3.936318874359131, + 4.092633247375488, + 4.056926250457764, + 4.0292253494262695, + 4.079093933105469, + 4.038450717926025, + 4.0393500328063965, + 4.189126968383789, + 4.1186981201171875, + 4.061012268066406, + 4.083641529083252, + 4.179285526275635, + 4.039545059204102, + 4.179362773895264, + 4.117460250854492, + 4.133169174194336, + 4.007277488708496, + 4.190375328063965, + 4.057973861694336, + 4.16795015335083, + 4.213861465454102, + 4.066026210784912, + 4.039978504180908, + 4.1673264503479, + 4.035836219787598, + 4.1207966804504395, + 4.128304481506348, + 4.076305389404297, + 4.062729358673096, + 4.150737762451172, + 3.8788650035858154, + 4.141756534576416, + 4.005947589874268, + 4.0860819816589355, + 3.944180488586426, + 3.985255002975464, + 4.224400043487549, + 4.072048664093018, + 3.9621198177337646, + 4.054590702056885, + 4.134291648864746, + 4.043899059295654, + 4.049223899841309, + 4.03282356262207, + 4.114742755889893, + 4.023148059844971, + 4.113577365875244, + 3.9314327239990234, + 4.0121073722839355, + 4.076461315155029, + 4.136337757110596, + 3.9793877601623535, + 4.0822367668151855, + 4.063570499420166, + 4.023751258850098, + 4.091669082641602, + 3.990891933441162, + 4.0547261238098145, + 4.101351261138916, + 4.051510334014893, + 3.984335422515869, + 3.939734697341919, + 3.930536985397339, + 3.9620063304901123, + 3.9715044498443604, + 3.9384593963623047, + 3.756605625152588, + 3.884660482406616, + 3.8932900428771973, + 3.8892641067504883, + 3.7497174739837646, + 3.9992644786834717, + 3.982913017272949, + 4.078319549560547, + 3.973696231842041, + 4.054031848907471, + 3.9743213653564453, + 3.9460842609405518, + 3.878204822540283, + 3.8855645656585693, + 3.905613899230957, + 3.971713066101074, + 3.855020046234131, + 3.87463641166687, + 4.0193352699279785, + 3.955998182296753, + 3.949596643447876, + 3.8934919834136963, + 3.957324981689453, + 3.9900264739990234, + 3.932826042175293, + 3.850468397140503, + 3.985055446624756, + 3.9441354274749756, + 4.0160040855407715, + 3.963153600692749, + 3.844982385635376, + 3.9434046745300293, + 3.8950307369232178, + 3.905820608139038, + 3.854686975479126, + 3.9313180446624756, + 3.9249186515808105, + 3.8739516735076904, + 3.9263648986816406, + 3.9348506927490234, + 3.983909845352173, + 3.7697057723999023, + 3.957937717437744, + 3.9222660064697266, + 3.863558769226074, + 3.8249309062957764, + 3.8766956329345703, + 3.8598251342773438, + 3.954132556915283, + 3.900559425354004, + 3.991690158843994, + 3.837266445159912, + 3.9124929904937744, + 3.8489913940429688, + 3.9491214752197266, + 3.955535411834717, + 3.78617525100708, + 3.7697947025299072, + 3.8642897605895996, + 3.9398820400238037, + 3.9521756172180176, + 3.932849407196045, + 3.8755617141723633, + 3.888282537460327, + 3.7897067070007324, + 3.9379208087921143, + 3.789513111114502, + 3.870694637298584, + 3.907493829727173, + 3.8243768215179443, + 3.816246509552002, + 3.9177026748657227, + 3.887727975845337, + 3.94185209274292, + 3.7992324829101562, + 3.881930112838745, + 3.81786847114563, + 3.8333353996276855, + 3.8792083263397217, + 3.942002296447754, + 3.8717546463012695, + 3.838658571243286, + 3.7920992374420166, + 3.9065821170806885, + 3.9328691959381104, + 3.957388401031494, + 3.8014509677886963, + 3.868725538253784, + 3.887402057647705, + 3.944051504135132, + 3.8844387531280518, + 3.8209521770477295, + 3.896601915359497, + 3.8895673751831055, + 3.890742301940918, + 3.8384158611297607, + 3.8760554790496826, + 3.762022018432617, + 3.831303119659424, + 3.6894102096557617, + 3.7624874114990234, + 3.788594961166382, + 3.7897770404815674, + 3.753694772720337, + 3.8635973930358887, + 3.977940559387207, + 3.773113489151001, + 3.8108890056610107, + 3.815654993057251, + 3.783404588699341, + 3.812673807144165, + 3.9279067516326904, + 3.903792381286621, + 3.8204903602600098, + 3.876948595046997, + 3.866952657699585, + 3.785331964492798, + 3.9108057022094727, + 3.8097312450408936, + 3.779420852661133, + 3.542727470397949, + 3.691746950149536, + 3.855698823928833, + 3.7573323249816895, + 3.6569957733154297, + 3.74235200881958, + 3.852501630783081, + 3.778463125228882, + 3.7745959758758545, + 3.8165674209594727, + 3.849318265914917, + 3.7839279174804688, + 3.7697057723999023, + 3.5642762184143066, + 3.790881872177124, + 3.7579829692840576, + 3.7613701820373535, + 3.7844536304473877, + 3.80204701423645, + 3.8146562576293945, + 3.7454452514648438, + 3.7294867038726807, + 3.7473065853118896, + 3.9240500926971436, + 3.7041983604431152, + 3.7521281242370605, + 3.7994539737701416, + 3.724485158920288, + 3.7402656078338623, + 3.732452630996704, + 3.691592216491699, + 3.7598888874053955, + 3.681879997253418, + 3.8281328678131104, + 3.816432476043701, + 3.735805034637451, + 3.730586528778076, + 3.689647674560547, + 3.763329029083252, + 3.8260059356689453, + 3.658392906188965, + 3.8380286693573, + 3.668050765991211, + 3.705509662628174, + 3.6588070392608643, + 3.8247931003570557, + 3.77923846244812, + 3.7232255935668945, + 3.8196308612823486, + 3.8195717334747314, + 3.6940464973449707, + 3.7816388607025146, + 3.7933685779571533, + 3.735483407974243, + 3.7309725284576416, + 3.696556568145752, + 3.7230944633483887, + 3.7586519718170166, + 3.6309614181518555, + 3.6726832389831543, + 3.7820794582366943, + 3.624617576599121, + 3.7011911869049072, + 3.71907114982605, + 3.61279034614563, + 3.763723850250244, + 3.7656710147857666, + 3.727858543395996, + 3.8196635246276855, + 3.6422250270843506, + 3.8084347248077393, + 3.759601354598999, + 3.8924801349639893, + 3.724949836730957, + 3.827840805053711, + 3.7249746322631836, + 3.788473606109619, + 3.7014620304107666, + 3.79978346824646, + 3.553180456161499, + 3.6673424243927, + 3.669973850250244, + 3.5227251052856445, + 3.640338182449341, + 3.762974977493286, + 3.7804367542266846, + 3.782827615737915, + 3.819035053253174, + 3.665395736694336, + 3.791717290878296, + 3.7832629680633545, + 3.690722942352295, + 3.691589593887329, + 3.683407783508301, + 3.7458057403564453, + 3.734328269958496, + 3.601078510284424, + 3.753725290298462, + 3.7468183040618896, + 3.6993906497955322, + 3.7542011737823486, + 3.70784592628479, + 3.7471351623535156, + 3.7087018489837646, + 3.771836757659912, + 3.7173309326171875, + 3.6746602058410645, + 3.724182605743408, + 3.750166654586792, + 3.725543975830078, + 3.7024970054626465, + 3.650277614593506, + 3.71096134185791, + 3.612922191619873, + 3.7749557495117188, + 3.7647619247436523, + 3.734203577041626, + 3.633774995803833, + 3.7060320377349854, + 3.758356809616089, + 3.6766297817230225, + 3.688084602355957, + 3.65818452835083, + 3.6058349609375, + 3.6229398250579834, + 3.6595025062561035, + 3.6447954177856445, + 3.5993518829345703, + 3.675102949142456, + 3.5750269889831543, + 3.716088056564331, + 3.5027713775634766, + 3.585749626159668, + 3.629427909851074, + 3.6301729679107666, + 3.6288230419158936, + 3.568655014038086, + 3.6624691486358643, + 3.552953004837036, + 3.699594020843506, + 3.686398983001709, + 3.5496230125427246, + 3.7478373050689697, + 3.5974435806274414, + 3.5542075634002686, + 3.5745787620544434, + 3.5208234786987305, + 3.6809475421905518, + 3.5368783473968506, + 3.655672550201416, + 3.6166725158691406, + 3.6261048316955566, + 3.5130014419555664, + 3.701723098754883, + 3.621793270111084, + 3.755059003829956, + 3.659555673599243, + 3.5913124084472656, + 3.5507800579071045, + 3.573503255844116, + 3.5389292240142822, + 3.712963819503784, + 3.655848264694214, + 3.556097984313965, + 3.5836105346679688, + 3.610381603240967, + 3.617037296295166, + 3.7612996101379395, + 3.572174549102783, + 3.639450788497925, + 3.7495806217193604, + 3.60901141166687, + 3.4955179691314697, + 3.6854519844055176, + 3.589317798614502, + 3.626854181289673, + 3.7276692390441895, + 3.569125175476074, + 3.6134133338928223, + 3.6828854084014893, + 3.6527209281921387, + 3.6161069869995117, + 3.6569037437438965, + 3.6968979835510254, + 3.6169273853302, + 3.6776766777038574, + 3.5210633277893066, + 3.6105077266693115, + 3.7163784503936768, + 3.6601102352142334, + 3.4959323406219482, + 3.4797110557556152, + 3.601036548614502, + 3.573213577270508, + 3.573184013366699, + 3.543210506439209, + 3.578434705734253, + 3.529803514480591, + 3.6317434310913086, + 3.5869626998901367, + 3.631866455078125, + 3.573538303375244, + 3.623593330383301, + 3.6077611446380615, + 3.495532512664795, + 3.6965856552124023, + 3.652045726776123, + 3.4660816192626953, + 3.6602847576141357, + 3.605349063873291, + 3.707080125808716, + 3.4674930572509766, + 3.541825532913208, + 3.6277029514312744, + 3.5829083919525146, + 3.590543031692505, + 3.590012550354004, + 3.470736026763916, + 3.646986961364746, + 3.683380126953125, + 3.5751004219055176, + 3.6104040145874023, + 3.7139182090759277, + 3.633795738220215, + 3.5722973346710205, + 3.7372846603393555, + 3.721611738204956, + 3.5955893993377686, + 3.4960126876831055, + 3.520246744155884, + 3.593479633331299, + 3.6569645404815674, + 3.5208067893981934, + 3.583649158477783, + 3.72613787651062, + 3.6182610988616943, + 3.6824090480804443, + 3.5723469257354736, + 3.5576303005218506, + 3.537022590637207, + 3.6014444828033447, + 3.5537617206573486, + 3.5567688941955566, + 3.551109552383423, + 3.5079140663146973, + 3.702385425567627, + 3.6442387104034424, + 3.4976983070373535, + 3.685243844985962, + 3.5884721279144287, + 3.5308635234832764, + 3.5181095600128174, + 3.48220157623291, + 3.546992778778076, + 3.520122766494751, + 3.4547929763793945, + 3.5501112937927246, + 3.4387319087982178, + 3.563033103942871, + 3.555668592453003, + 3.509536027908325, + 3.5664894580841064, + 3.394761800765991, + 3.5937750339508057, + 3.456845760345459, + 3.4517195224761963, + 3.5863797664642334, + 3.580859422683716, + 3.4761362075805664, + 3.3272008895874023, + 3.444228410720825, + 3.411818504333496, + 3.5775868892669678, + 3.538609504699707, + 3.62446665763855, + 3.5718517303466797, + 3.4801762104034424, + 3.652991533279419, + 3.452411651611328, + 3.542450189590454, + 3.5118906497955322, + 3.5389010906219482, + 3.549992561340332, + 3.5564658641815186, + 3.5904648303985596, + 3.5082406997680664, + 3.496490716934204, + 3.523729085922241, + 3.36025071144104, + 3.461322069168091, + 3.5157175064086914, + 3.4366610050201416, + 3.436392307281494, + 3.540954828262329, + 3.4057281017303467, + 3.5440659523010254, + 3.576925754547119, + 3.517378807067871, + 3.563006639480591, + 3.583968162536621, + 3.5123801231384277, + 3.518847942352295, + 3.563751220703125, + 3.447844982147217, + 3.4854235649108887, + 3.4105637073516846, + 3.378251791000366, + 3.4804022312164307, + 3.443796157836914, + 3.731812000274658, + 3.585846424102783, + 3.4873950481414795, + 3.4607651233673096, + 3.6492292881011963, + 3.5494444370269775, + 3.512369155883789, + 3.537546396255493, + 3.5411014556884766, + 3.5957512855529785, + 3.552929639816284, + 3.47654128074646, + 3.4044270515441895, + 3.5403921604156494, + 3.516664743423462, + 3.6572840213775635, + 3.4352478981018066, + 3.4893031120300293, + 3.469472885131836, + 3.4843590259552, + 3.4818451404571533, + 3.5018978118896484, + 3.5590577125549316, + 3.514108657836914, + 3.5750579833984375, + 3.5855274200439453, + 3.4829208850860596, + 3.5173521041870117, + 3.448943614959717, + 3.4961142539978027, + 3.4660093784332275, + 3.5586094856262207, + 3.3537228107452393, + 3.5213334560394287, + 3.5246598720550537, + 3.544020891189575, + 3.4222424030303955, + 3.6104447841644287, + 3.394962787628174, + 3.462428092956543, + 3.511366844177246, + 3.3998665809631348, + 3.4269731044769287, + 3.5383102893829346, + 3.4511847496032715, + 3.673882484436035, + 3.3977696895599365, + 3.456061601638794, + 3.6178531646728516, + 3.5392634868621826, + 3.5046401023864746, + 3.4202587604522705, + 3.5695059299468994, + 3.5306456089019775, + 3.5268020629882812, + 3.4520320892333984, + 3.556858539581299, + 3.4786505699157715, + 3.5327470302581787, + 3.545715808868408, + 3.41339373588562, + 3.547559976577759, + 3.40612530708313, + 3.497126340866089, + 3.613967180252075, + 3.522033452987671, + 3.444296360015869, + 3.555626153945923, + 3.5386197566986084, + 3.6239378452301025, + 3.436732053756714, + 3.3410472869873047, + 3.369588851928711, + 3.4569180011749268, + 3.4479475021362305, + 3.4406354427337646, + 3.5290050506591797, + 3.461894989013672, + 3.4975297451019287, + 3.519129991531372, + 3.444352626800537, + 3.3725814819335938, + 3.428480625152588, + 3.4193081855773926, + 3.395434856414795, + 3.4915733337402344, + 3.4814717769622803, + 3.35117506980896, + 3.4544639587402344, + 3.499180793762207, + 3.441195487976074, + 3.4290127754211426, + 3.5004286766052246, + 3.3472535610198975, + 3.6230602264404297, + 3.4188146591186523, + 3.400947332382202, + 3.4404234886169434, + 3.4548723697662354, + 3.3631527423858643, + 3.38564133644104, + 3.3460135459899902, + 3.459371566772461, + 3.507126569747925, + 3.5010745525360107, + 3.5131654739379883, + 3.5440545082092285, + 3.5023000240325928, + 3.408576250076294, + 3.497711181640625, + 3.423832654953003, + 3.355329751968384, + 3.4671850204467773, + 3.5448787212371826, + 3.4525420665740967, + 3.525692939758301, + 3.4677977561950684, + 3.399813413619995, + 3.373281955718994, + 3.405468225479126, + 3.36848783493042, + 3.339128017425537, + 3.409723997116089, + 3.5316243171691895, + 3.6108009815216064, + 3.486647844314575, + 3.4669201374053955, + 3.5059733390808105, + 3.5195202827453613, + 3.5526907444000244, + 3.3720037937164307, + 3.450470447540283, + 3.2906553745269775, + 3.4565892219543457, + 3.413933038711548, + 3.3675286769866943, + 3.5368709564208984, + 3.5041115283966064, + 3.398303270339966, + 3.496263265609741, + 3.5335991382598877, + 3.4720699787139893, + 3.3513424396514893, + 3.4327585697174072, + 3.470864772796631, + 3.559018135070801, + 3.38210129737854, + 3.520040273666382, + 3.43298077583313, + 3.3768367767333984, + 3.4075942039489746, + 3.483185052871704, + 3.4454736709594727, + 3.4438328742980957, + 3.464306116104126, + 3.3748104572296143, + 3.5461599826812744, + 3.528944492340088, + 3.538494348526001, + 3.565156936645508, + 3.359931707382202, + 3.390775442123413, + 3.4316606521606445, + 3.5317282676696777, + 3.4181532859802246, + 3.3838536739349365, + 3.3842883110046387, + 3.425149917602539, + 3.389457941055298, + 3.3489763736724854, + 3.452540397644043, + 3.5122931003570557, + 3.4054946899414062, + 3.3455541133880615, + 3.40128755569458, + 3.4627068042755127, + 3.4926302433013916, + 3.4512882232666016, + 3.517514705657959, + 3.47413969039917, + 3.423379421234131, + 3.3969969749450684, + 3.414473056793213, + 3.3453774452209473, + 3.461190938949585, + 3.508679151535034, + 3.377396821975708, + 3.4549400806427, + 3.4084455966949463, + 3.512704849243164, + 3.377384662628174, + 3.4460480213165283, + 3.4458320140838623, + 3.4707090854644775, + 3.457240104675293, + 3.4399304389953613, + 3.3782758712768555, + 3.362060070037842, + 3.474766254425049, + 3.4013659954071045, + 3.337111234664917, + 3.4828479290008545, + 3.3709428310394287, + 3.4677205085754395, + 3.3795642852783203, + 3.445652484893799, + 3.453120231628418, + 3.3503224849700928, + 3.4403486251831055, + 3.3010547161102295, + 3.3872196674346924, + 3.4420881271362305, + 3.430666446685791, + 3.356412649154663, + 3.473104238510132, + 3.4212610721588135, + 3.4486541748046875, + 3.444728374481201, + 3.350475788116455, + 3.3525404930114746, + 3.352858304977417, + 3.478882074356079, + 3.4201951026916504, + 3.4572672843933105, + 3.466243267059326, + 3.418398857116699, + 3.3649487495422363, + 3.3058159351348877, + 3.445020914077759, + 3.4969751834869385, + 3.5759925842285156, + 3.3978452682495117, + 3.4427666664123535, + 3.292954683303833, + 3.3787155151367188, + 3.5389132499694824, + 3.4477694034576416, + 3.320580005645752, + 3.440109968185425, + 3.3467633724212646, + 3.3734304904937744, + 3.500610589981079, + 3.398869276046753, + 3.4207258224487305, + 3.441801071166992, + 3.389676332473755, + 3.4493119716644287, + 3.3617377281188965, + 3.374974012374878, + 3.3592722415924072, + 3.3256008625030518, + 3.369013786315918, + 3.310814380645752, + 3.363645076751709, + 3.375143527984619, + 3.3343851566314697, + 3.4869210720062256, + 3.4532291889190674, + 3.4396510124206543, + 3.4560012817382812, + 3.412602424621582, + 3.4206550121307373, + 3.37422776222229, + 3.4403762817382812, + 3.4107508659362793, + 3.3633694648742676, + 3.4543278217315674, + 3.3234081268310547, + 3.4965262413024902, + 3.327275276184082, + 3.3056578636169434, + 3.422389507293701, + 3.4751036167144775, + 3.4849135875701904, + 3.4007716178894043, + 3.429516315460205, + 3.3453359603881836, + 3.4232616424560547, + 3.3384180068969727, + 3.4743876457214355, + 3.349825859069824, + 3.4464924335479736, + 3.5392467975616455, + 3.3936336040496826, + 3.3107409477233887, + 3.441699981689453, + 3.4517576694488525, + 3.387538433074951, + 3.3592934608459473, + 3.505383253097534, + 3.413344621658325, + 3.4201865196228027, + 3.4395053386688232, + 3.3184409141540527, + 3.418865203857422, + 3.35628604888916, + 3.3597888946533203, + 3.616569757461548, + 3.3710596561431885, + 3.3962514400482178, + 3.387108564376831, + 3.3996741771698, + 3.378462553024292, + 3.3275246620178223, + 3.5012576580047607, + 3.387995719909668, + 3.450376510620117, + 3.434454917907715, + 3.418565273284912, + 3.324408769607544, + 3.332937002182007 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 133.77455214613195, + 83.37890963238469, + 63.63202882445569, + 54.50323672358589, + 49.44669669624289, + 45.82952835390986, + 43.19046065141615, + 41.161193257146174, + 39.06104491253887, + 37.513671912891596, + 36.33206070807312, + 35.099019536771564, + 34.378684087860634, + 33.58752060699822, + 32.805107515551974, + 32.51674946628261, + 32.100812829517025, + 31.91271558318103, + 31.830014899354712, + 31.80124442158557 + ], + "best_ppl": 31.80124442158557, + "train_time_s": 18520.108202457428 +} \ No newline at end of file diff --git a/061_dense_mol/dense_k5_all_s137_best.pt b/061_dense_mol/dense_k5_all_s137_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2c2d76f6d756d025274a37c09482378b3da30c9 --- /dev/null +++ b/061_dense_mol/dense_k5_all_s137_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa374e5cae49d206dd794738c8fd795a006d8c1629a0f021ee2c917feee6efc3 +size 341273328 diff --git a/061_dense_mol/dense_k5_all_s137_results.json b/061_dense_mol/dense_k5_all_s137_results.json new file mode 100644 index 0000000000000000000000000000000000000000..0a682a71a83e07473d61a1b9b800748a2a891631 --- /dev/null +++ b/061_dense_mol/dense_k5_all_s137_results.json @@ -0,0 +1,2059 @@ +{ + "label": "dense_k5_all_s137", + "top_k_blocks": null, + "num_thin_blocks": 5, + "total_params": 85260288, + "router_params": 0, + "dense_ffn": true, + "d_ff_thin": 1024, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.028093338012695, + 13.684922218322754, + 8.032440185546875, + 7.45949125289917, + 7.4345550537109375, + 7.225525856018066, + 7.1362199783325195, + 6.987271785736084, + 6.80167293548584, + 6.716446876525879, + 6.646402835845947, + 6.55151891708374, + 6.450263977050781, + 6.26204776763916, + 6.20962381362915, + 6.1471452713012695, + 6.073149681091309, + 5.988809108734131, + 5.927733421325684, + 5.936685562133789, + 5.753668308258057, + 5.699251174926758, + 5.754493713378906, + 5.703005313873291, + 5.648772239685059, + 5.720773696899414, + 5.5756306648254395, + 5.4926533699035645, + 5.4622063636779785, + 5.503584861755371, + 5.465065002441406, + 5.4032673835754395, + 5.340695381164551, + 5.18040132522583, + 5.312648773193359, + 5.202548027038574, + 5.323311805725098, + 5.2695207595825195, + 5.336825370788574, + 5.208956718444824, + 5.095912456512451, + 5.191556453704834, + 5.060064315795898, + 5.067648887634277, + 5.077356815338135, + 5.090632438659668, + 4.982236385345459, + 5.03317403793335, + 5.039735794067383, + 5.006917476654053, + 4.996973991394043, + 4.967208385467529, + 4.942922592163086, + 4.972583770751953, + 4.9658308029174805, + 4.742112159729004, + 4.841814994812012, + 5.014279365539551, + 4.830621719360352, + 4.883389949798584, + 4.758388519287109, + 4.795822620391846, + 4.825115203857422, + 4.679274559020996, + 4.799718856811523, + 4.748561859130859, + 4.879546642303467, + 4.818293571472168, + 4.831288814544678, + 4.7270894050598145, + 4.777462482452393, + 4.73742151260376, + 4.767153263092041, + 4.761833190917969, + 4.724284648895264, + 4.716057777404785, + 4.700389385223389, + 4.766665935516357, + 4.768581390380859, + 4.793619632720947, + 4.753695964813232, + 4.748819828033447, + 4.765684127807617, + 4.663779258728027, + 4.633289813995361, + 4.654069900512695, + 4.672582626342773, + 4.741877555847168, + 4.647899150848389, + 4.697320461273193, + 4.576617240905762, + 4.6432037353515625, + 4.669466495513916, + 4.503787517547607, + 4.6149444580078125, + 4.6748857498168945, + 4.551187038421631, + 4.648822784423828, + 4.556088924407959, + 4.541932106018066, + 4.392312049865723, + 4.583238124847412, + 4.377395153045654, + 4.518528461456299, + 4.495662689208984, + 4.522961616516113, + 4.63636589050293, + 4.549774169921875, + 4.47861909866333, + 4.479155540466309, + 4.546811103820801, + 4.477746963500977, + 4.4651031494140625, + 4.481105804443359, + 4.585752010345459, + 4.470656394958496, + 4.377625942230225, + 4.514806270599365, + 4.5923871994018555, + 4.511075019836426, + 4.485746383666992, + 4.499919414520264, + 4.297519207000732, + 4.400887966156006, + 4.425225257873535, + 4.335681438446045, + 4.367465496063232, + 4.330784320831299, + 4.2745442390441895, + 4.349457740783691, + 4.296463489532471, + 4.266404151916504, + 4.386037349700928, + 4.242969512939453, + 4.33571720123291, + 4.254011631011963, + 4.297770977020264, + 4.203963756561279, + 4.278047561645508, + 4.372576713562012, + 4.342288494110107, + 4.3872389793396, + 4.297924995422363, + 4.249495506286621, + 4.288506507873535, + 4.312887191772461, + 4.271244525909424, + 4.391453742980957, + 4.215461254119873, + 4.3242387771606445, + 4.257345676422119, + 4.1619977951049805, + 4.31427001953125, + 4.2603864669799805, + 4.349725246429443, + 4.3019328117370605, + 4.354248523712158, + 4.270906925201416, + 4.208056449890137, + 4.160401344299316, + 4.196836471557617, + 4.372086524963379, + 4.315142631530762, + 4.253114700317383, + 4.35486364364624, + 4.1917314529418945, + 4.383200168609619, + 4.204350471496582, + 4.177653789520264, + 4.212420463562012, + 4.322704792022705, + 4.280097007751465, + 4.182077884674072, + 4.208957672119141, + 4.207525253295898, + 4.092999458312988, + 4.138424396514893, + 4.247786521911621, + 4.178735733032227, + 4.177596569061279, + 4.249925136566162, + 4.252195835113525, + 4.196413993835449, + 4.098898887634277, + 4.1659836769104, + 4.1191606521606445, + 4.190781116485596, + 4.0355048179626465, + 4.145471096038818, + 4.185755252838135, + 4.206650257110596, + 4.1681132316589355, + 4.029094219207764, + 4.078282833099365, + 4.111809253692627, + 4.160662651062012, + 4.051391124725342, + 3.968036651611328, + 3.9643924236297607, + 4.117505073547363, + 4.171380043029785, + 4.0790934562683105, + 4.1606926918029785, + 4.05878210067749, + 4.100035190582275, + 4.089503288269043, + 4.062138080596924, + 4.115429401397705, + 4.15643310546875, + 4.137076377868652, + 4.050444602966309, + 4.081338882446289, + 4.095387935638428, + 4.153312683105469, + 4.003017425537109, + 4.181910991668701, + 4.120916843414307, + 4.135295867919922, + 4.094429969787598, + 4.053163051605225, + 4.155765056610107, + 4.0374603271484375, + 4.073144912719727, + 4.074639320373535, + 4.140322208404541, + 4.172617435455322, + 4.0125603675842285, + 4.056713104248047, + 4.070901393890381, + 4.059574127197266, + 3.905956268310547, + 4.060032844543457, + 4.064514636993408, + 4.0399370193481445, + 3.994990825653076, + 4.112334728240967, + 3.9891717433929443, + 4.04659366607666, + 4.052341461181641, + 4.108369827270508, + 4.01617431640625, + 4.091523170471191, + 3.9247117042541504, + 4.026711463928223, + 4.003237247467041, + 3.968355417251587, + 3.9455058574676514, + 4.02523136138916, + 3.9866859912872314, + 4.012932777404785, + 4.234533786773682, + 4.002360820770264, + 4.051571369171143, + 3.892214775085449, + 3.906719923019409, + 3.947169542312622, + 3.844216823577881, + 4.127892017364502, + 3.8592746257781982, + 3.897933006286621, + 3.8875770568847656, + 3.934359550476074, + 4.0744547843933105, + 3.967735767364502, + 3.8887553215026855, + 3.9148776531219482, + 3.9199469089508057, + 4.0422043800354, + 3.901655435562134, + 3.9301071166992188, + 3.9393227100372314, + 3.889580249786377, + 3.827789068222046, + 3.948725700378418, + 3.9640910625457764, + 3.830519676208496, + 3.9529149532318115, + 4.016435623168945, + 3.981626510620117, + 4.054501056671143, + 3.8194825649261475, + 3.9138293266296387, + 4.068084239959717, + 3.7973403930664062, + 3.86552095413208, + 3.9427595138549805, + 3.8376567363739014, + 3.8458051681518555, + 3.9613068103790283, + 4.006308555603027, + 3.9088079929351807, + 4.021663665771484, + 3.8860695362091064, + 3.7560291290283203, + 3.6794564723968506, + 3.9395058155059814, + 3.8755714893341064, + 3.9422447681427, + 3.9796721935272217, + 3.7321572303771973, + 3.76552152633667, + 3.931401252746582, + 3.9109041690826416, + 3.9200103282928467, + 3.8598737716674805, + 4.01100492477417, + 3.955324411392212, + 4.004974365234375, + 3.7858991622924805, + 3.974551200866699, + 3.928431272506714, + 3.930340051651001, + 3.9957475662231445, + 3.797481060028076, + 3.9623682498931885, + 3.8863015174865723, + 3.882798671722412, + 3.895735025405884, + 3.900365114212036, + 3.8670740127563477, + 3.8413941860198975, + 3.928640365600586, + 3.8454432487487793, + 3.8453316688537598, + 3.838096857070923, + 4.005338668823242, + 3.7563934326171875, + 3.9241292476654053, + 3.947871685028076, + 4.111273765563965, + 3.863861322402954, + 3.8086888790130615, + 3.8263351917266846, + 3.907468318939209, + 3.9646832942962646, + 3.741340398788452, + 3.9617536067962646, + 3.9127848148345947, + 3.8490357398986816, + 3.8422882556915283, + 3.8216707706451416, + 3.8546247482299805, + 3.8749942779541016, + 3.9511101245880127, + 3.8728229999542236, + 3.8316023349761963, + 3.7825682163238525, + 3.9495182037353516, + 3.9440460205078125, + 3.807988405227661, + 3.897214651107788, + 3.9857277870178223, + 3.798440933227539, + 3.9061129093170166, + 3.7440054416656494, + 3.891810178756714, + 3.881350517272949, + 3.76857328414917, + 3.7592663764953613, + 3.945164918899536, + 3.8498892784118652, + 3.8326733112335205, + 3.845578193664551, + 3.806859016418457, + 3.838277578353882, + 3.7613942623138428, + 3.763026714324951, + 3.880939483642578, + 3.6182689666748047, + 3.975243091583252, + 3.9855730533599854, + 3.9018523693084717, + 3.7811062335968018, + 3.854388475418091, + 3.8444736003875732, + 3.895930767059326, + 3.9736926555633545, + 3.821885108947754, + 3.833953619003296, + 3.7089366912841797, + 3.6827340126037598, + 3.678237199783325, + 3.7680790424346924, + 3.795138120651245, + 3.8504767417907715, + 3.7299516201019287, + 3.699667453765869, + 3.771195411682129, + 3.8170082569122314, + 3.7503323554992676, + 3.768994092941284, + 3.806286096572876, + 3.749157190322876, + 3.7671658992767334, + 3.748619556427002, + 3.7007129192352295, + 3.7234885692596436, + 3.81120228767395, + 3.8438427448272705, + 3.7104148864746094, + 3.8122644424438477, + 3.7312307357788086, + 3.896152973175049, + 3.8288846015930176, + 3.8832530975341797, + 3.5994088649749756, + 3.7781901359558105, + 3.87158203125, + 3.791855812072754, + 3.9075276851654053, + 3.7327933311462402, + 3.7690281867980957, + 3.8195419311523438, + 3.7658731937408447, + 3.7678427696228027, + 3.7712981700897217, + 3.7239902019500732, + 3.833965301513672, + 3.908889055252075, + 3.773829936981201, + 3.689065456390381, + 3.7554819583892822, + 3.7303435802459717, + 3.754345655441284, + 3.7985734939575195, + 3.7799062728881836, + 3.6648526191711426, + 3.703338623046875, + 3.72761607170105, + 3.703568935394287, + 3.800337314605713, + 3.898029327392578, + 3.831752300262451, + 3.5660741329193115, + 3.9093539714813232, + 3.8686134815216064, + 3.8100972175598145, + 3.7156224250793457, + 3.7271811962127686, + 3.6305394172668457, + 3.785874366760254, + 3.596912145614624, + 3.750979423522949, + 3.7859749794006348, + 3.8086841106414795, + 3.7208168506622314, + 3.748502016067505, + 3.754897356033325, + 3.7913827896118164, + 3.7122204303741455, + 3.7079238891601562, + 3.6598830223083496, + 3.8051536083221436, + 3.8015406131744385, + 3.714696168899536, + 3.7993156909942627, + 3.740403890609741, + 3.8238115310668945, + 3.8403258323669434, + 3.7449851036071777, + 3.7087929248809814, + 3.683908700942993, + 3.672497272491455, + 3.629009485244751, + 3.774275302886963, + 3.801020860671997, + 3.652423620223999, + 3.7597200870513916, + 3.568972587585449, + 3.635223627090454, + 3.851012706756592, + 3.691497564315796, + 3.8310625553131104, + 3.7347559928894043, + 3.6312365531921387, + 3.6991641521453857, + 3.822000026702881, + 3.719882011413574, + 3.730489492416382, + 3.7210285663604736, + 3.7339954376220703, + 3.7412726879119873, + 3.69350004196167, + 3.6441102027893066, + 3.7353248596191406, + 3.814326286315918, + 3.586939573287964, + 3.6143484115600586, + 3.865680456161499, + 3.7207682132720947, + 3.6800825595855713, + 3.726379156112671, + 3.6927595138549805, + 3.5258185863494873, + 3.72900390625, + 3.792853593826294, + 3.750584363937378, + 3.662489175796509, + 3.6940228939056396, + 3.688769578933716, + 3.595048427581787, + 3.7344963550567627, + 3.6419804096221924, + 3.6023707389831543, + 3.7427735328674316, + 3.586254835128784, + 3.6237375736236572, + 3.6451737880706787, + 3.7144572734832764, + 3.64162278175354, + 3.66095232963562, + 3.6747324466705322, + 3.727048635482788, + 3.6308436393737793, + 3.6474034786224365, + 3.635845184326172, + 3.5579581260681152, + 3.6418986320495605, + 3.672882080078125, + 3.6047327518463135, + 3.6459617614746094, + 3.5764853954315186, + 3.602457046508789, + 3.5557503700256348, + 3.631995439529419, + 3.5237269401550293, + 3.627051591873169, + 3.5426502227783203, + 3.490699291229248, + 3.743032693862915, + 3.742048501968384, + 3.6884140968322754, + 3.4572081565856934, + 3.692660093307495, + 3.6858978271484375, + 3.6433966159820557, + 3.5550882816314697, + 3.721257209777832, + 3.6605849266052246, + 3.541247844696045, + 3.4969098567962646, + 3.5754666328430176, + 3.684520721435547, + 3.653258800506592, + 3.6296138763427734, + 3.6193504333496094, + 3.7243590354919434, + 3.5150516033172607, + 3.650585412979126, + 3.4734086990356445, + 3.689842700958252, + 3.65508770942688, + 3.528186321258545, + 3.401988983154297, + 3.5822322368621826, + 3.6866164207458496, + 3.6057844161987305, + 3.520190477371216, + 3.566960096359253, + 3.5523681640625, + 3.628035306930542, + 3.544764280319214, + 3.625082492828369, + 3.638850212097168, + 3.659086227416992, + 3.571152687072754, + 3.5624313354492188, + 3.7108407020568848, + 3.581465482711792, + 3.650179862976074, + 3.6096534729003906, + 3.711620330810547, + 3.6392388343811035, + 3.563863515853882, + 3.580832004547119, + 3.749610662460327, + 3.5620386600494385, + 3.631927013397217, + 3.511129140853882, + 3.5757029056549072, + 3.5949723720550537, + 3.614797592163086, + 3.653407573699951, + 3.6723413467407227, + 3.5777149200439453, + 3.6680335998535156, + 3.6107699871063232, + 3.685497999191284, + 3.689256429672241, + 3.643989086151123, + 3.4893648624420166, + 3.741950035095215, + 3.6766395568847656, + 3.581901788711548, + 3.5938193798065186, + 3.520517110824585, + 3.6910688877105713, + 3.543342351913452, + 3.6387054920196533, + 3.7345292568206787, + 3.4394330978393555, + 3.669464111328125, + 3.4951157569885254, + 3.469961643218994, + 3.6164140701293945, + 3.561666965484619, + 3.596165895462036, + 3.581312894821167, + 3.6938769817352295, + 3.6075806617736816, + 3.5543484687805176, + 3.4858474731445312, + 3.585217237472534, + 3.7689616680145264, + 3.4692866802215576, + 3.658879518508911, + 3.65486216545105, + 3.625375270843506, + 3.5982701778411865, + 3.668144941329956, + 3.4281387329101562, + 3.643084764480591, + 3.551888942718506, + 3.684483051300049, + 3.6212985515594482, + 3.615610122680664, + 3.6253371238708496, + 3.595458984375, + 3.6200828552246094, + 3.4084632396698, + 3.5547730922698975, + 3.535914897918701, + 3.5594356060028076, + 3.507761240005493, + 3.6561429500579834, + 3.572964668273926, + 3.474578619003296, + 3.509337902069092, + 3.557199239730835, + 3.639779567718506, + 3.501758575439453, + 3.5308713912963867, + 3.5906105041503906, + 3.6402204036712646, + 3.3764500617980957, + 3.6408536434173584, + 3.44657826423645, + 3.497121810913086, + 3.567903995513916, + 3.538447856903076, + 3.5196938514709473, + 3.588233470916748, + 3.4982705116271973, + 3.3784451484680176, + 3.4786908626556396, + 3.4831655025482178, + 3.3963568210601807, + 3.5524673461914062, + 3.5114545822143555, + 3.4539713859558105, + 3.539746046066284, + 3.520015239715576, + 3.662414789199829, + 3.5752615928649902, + 3.440415382385254, + 3.4988698959350586, + 3.4923155307769775, + 3.469658374786377, + 3.583059310913086, + 3.583418846130371, + 3.4138033390045166, + 3.6052534580230713, + 3.618243455886841, + 3.5602197647094727, + 3.6398303508758545, + 3.535026788711548, + 3.5483174324035645, + 3.46112060546875, + 3.5213212966918945, + 3.4332971572875977, + 3.4197096824645996, + 3.5989136695861816, + 3.5414087772369385, + 3.5922718048095703, + 3.591231346130371, + 3.5220186710357666, + 3.4966511726379395, + 3.4501211643218994, + 3.5329625606536865, + 3.541837215423584, + 3.493016004562378, + 3.5249598026275635, + 3.555150032043457, + 3.6212682723999023, + 3.6573245525360107, + 3.513472080230713, + 3.5420281887054443, + 3.511162519454956, + 3.6407809257507324, + 3.54805064201355, + 3.6222896575927734, + 3.54557466506958, + 3.467435598373413, + 3.4680392742156982, + 3.442735433578491, + 3.4722368717193604, + 3.5756795406341553, + 3.5079565048217773, + 3.533536195755005, + 3.6109025478363037, + 3.482395887374878, + 3.5052103996276855, + 3.605797052383423, + 3.56373929977417, + 3.4715566635131836, + 3.5227904319763184, + 3.4177749156951904, + 3.542813301086426, + 3.598917007446289, + 3.55074143409729, + 3.50921368598938, + 3.4395394325256348, + 3.4223058223724365, + 3.599841833114624, + 3.51566219329834, + 3.5578627586364746, + 3.4907655715942383, + 3.5756165981292725, + 3.5918779373168945, + 3.6679608821868896, + 3.4677462577819824, + 3.324307918548584, + 3.510529041290283, + 3.4562625885009766, + 3.5976574420928955, + 3.4626262187957764, + 3.4409146308898926, + 3.4481778144836426, + 3.5615200996398926, + 3.5317375659942627, + 3.4878060817718506, + 3.3125672340393066, + 3.4390265941619873, + 3.469733715057373, + 3.363574504852295, + 3.6278269290924072, + 3.5636041164398193, + 3.5010600090026855, + 3.471792697906494, + 3.47393798828125, + 3.4729905128479004, + 3.548696756362915, + 3.5216825008392334, + 3.536590576171875, + 3.3457515239715576, + 3.393739938735962, + 3.635849952697754, + 3.4307212829589844, + 3.51753306388855, + 3.3631227016448975, + 3.420733690261841, + 3.410112142562866, + 3.453554153442383, + 3.407681703567505, + 3.5348217487335205, + 3.511990785598755, + 3.2955851554870605, + 3.2707738876342773, + 3.478983163833618, + 3.4313697814941406, + 3.5164713859558105, + 3.5329604148864746, + 3.511929512023926, + 3.509223222732544, + 3.4303715229034424, + 3.503753900527954, + 3.488985538482666, + 3.5003631114959717, + 3.466215133666992, + 3.406381368637085, + 3.5010123252868652, + 3.486340284347534, + 3.3369297981262207, + 3.343445062637329, + 3.4696264266967773, + 3.4973371028900146, + 3.39752459526062, + 3.50239634513855, + 3.3777072429656982, + 3.451258420944214, + 3.401111364364624, + 3.420836925506592, + 3.3897416591644287, + 3.4522759914398193, + 3.407162666320801, + 3.4997944831848145, + 3.449924945831299, + 3.5553503036499023, + 3.4358112812042236, + 3.4832613468170166, + 3.4737701416015625, + 3.430997610092163, + 3.512697458267212, + 3.381503105163574, + 3.4660370349884033, + 3.454735279083252, + 3.457198143005371, + 3.430778980255127, + 3.530195951461792, + 3.389242649078369, + 3.5506889820098877, + 3.464545726776123, + 3.4980711936950684, + 3.4200587272644043, + 3.4328017234802246, + 3.6040611267089844, + 3.3093912601470947, + 3.5049123764038086, + 3.4327902793884277, + 3.4572346210479736, + 3.4043917655944824, + 3.464935064315796, + 3.448859453201294, + 3.3785624504089355, + 3.4159419536590576, + 3.4137680530548096, + 3.4872562885284424, + 3.4937584400177, + 3.5376317501068115, + 3.5513994693756104, + 3.4654955863952637, + 3.516913890838623, + 3.4064438343048096, + 3.3986892700195312, + 3.5166513919830322, + 3.479609251022339, + 3.3634493350982666, + 3.387657880783081, + 3.3580055236816406, + 3.4311816692352295, + 3.4403417110443115, + 3.374281167984009, + 3.497185230255127, + 3.34203839302063, + 3.46205735206604, + 3.33465313911438, + 3.3798015117645264, + 3.488093376159668, + 3.3946428298950195, + 3.4082577228546143, + 3.4620463848114014, + 3.3783862590789795, + 3.4808194637298584, + 3.47159481048584, + 3.322141170501709, + 3.3244071006774902, + 3.4768729209899902, + 3.5347042083740234, + 3.4486124515533447, + 3.4702932834625244, + 3.398075580596924, + 3.433872699737549, + 3.567335367202759, + 3.445167064666748, + 3.3844099044799805, + 3.547544240951538, + 3.471217393875122, + 3.4983932971954346, + 3.2976953983306885, + 3.3815300464630127, + 3.4559342861175537, + 3.579653263092041, + 3.4227209091186523, + 3.4370522499084473, + 3.462606191635132, + 3.4828529357910156, + 3.5767781734466553, + 3.5879902839660645, + 3.5406625270843506, + 3.4280030727386475, + 3.349147319793701, + 3.31099009513855, + 3.4766786098480225, + 3.408782482147217, + 3.5669310092926025, + 3.4628398418426514, + 3.372941255569458, + 3.4043705463409424, + 3.32867693901062, + 3.302999258041382, + 3.371392250061035, + 3.3333680629730225, + 3.305673122406006, + 3.38494610786438, + 3.351863384246826, + 3.5051352977752686, + 3.3117828369140625, + 3.4271562099456787, + 3.4958419799804688, + 3.409806489944458, + 3.1952693462371826, + 3.4953532218933105, + 3.408747673034668, + 3.533188819885254, + 3.3722245693206787, + 3.4645578861236572, + 3.439537763595581, + 3.470520257949829, + 3.5477676391601562, + 3.387629985809326, + 3.520163059234619, + 3.507146120071411, + 3.457521915435791, + 3.497694730758667, + 3.3443899154663086, + 3.373809576034546, + 3.3144657611846924, + 3.4867000579833984, + 3.3820619583129883, + 3.451619863510132, + 3.4190149307250977, + 3.357356548309326, + 3.392120838165283, + 3.424379348754883, + 3.3356642723083496, + 3.374635934829712, + 3.3743293285369873, + 3.4962620735168457, + 3.3947038650512695, + 3.466698408126831, + 3.502298355102539, + 3.457557201385498, + 3.479308605194092, + 3.3847038745880127, + 3.332550048828125, + 3.3785054683685303, + 3.324052333831787, + 3.3851401805877686, + 3.481614828109741, + 3.403066635131836, + 3.3469624519348145, + 3.5080766677856445, + 3.4647836685180664, + 3.3836684226989746, + 3.25715970993042, + 3.374680280685425, + 3.4219677448272705, + 3.4217588901519775, + 3.3253068923950195, + 3.5104081630706787, + 3.2489078044891357, + 3.480931043624878, + 3.4021646976470947, + 3.320455312728882, + 3.412940263748169, + 3.4247689247131348, + 3.4848079681396484, + 3.4891393184661865, + 3.5000739097595215, + 3.4971089363098145, + 3.452136754989624, + 3.372368574142456, + 3.397075891494751, + 3.352924108505249, + 3.4485297203063965, + 3.360811233520508, + 3.4261839389801025, + 3.37292742729187, + 3.3466248512268066, + 3.474295139312744, + 3.4362001419067383, + 3.3724868297576904, + 3.3581438064575195, + 3.5079874992370605, + 3.3791041374206543, + 3.514643907546997, + 3.2493319511413574, + 3.430804491043091, + 3.314924716949463, + 3.3352794647216797, + 3.4488885402679443, + 3.4620351791381836, + 3.311718702316284, + 3.4147963523864746, + 3.3132071495056152, + 3.3256776332855225, + 3.5359420776367188, + 3.3721954822540283, + 3.4342963695526123, + 3.2891435623168945, + 3.333353281021118, + 3.4212045669555664, + 3.3409268856048584, + 3.4029297828674316, + 3.2872259616851807, + 3.2622241973876953, + 3.4765546321868896, + 3.4693939685821533, + 3.478090524673462, + 3.4812886714935303 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 133.9284533325189, + 83.66268539322242, + 64.66427383824858, + 55.1793731108962, + 50.14864599914215, + 46.13362451170187, + 43.43214518701639, + 41.4477956543419, + 39.53325479940061, + 37.7861286304273, + 36.69349662463218, + 35.46311630025596, + 34.63757204118767, + 33.80362967271721, + 33.052213228817784, + 32.668596972663074, + 32.29887327810002, + 32.157003141314426, + 32.04453843380292, + 32.03008678832555 + ], + "best_ppl": 32.03008678832555, + "train_time_s": 17953.78867316246 +} \ No newline at end of file diff --git a/061_dense_mol/dense_k5_all_s256_best.pt b/061_dense_mol/dense_k5_all_s256_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..452f7bf4ab7bae6b631506a79a9d73bb94ce6a53 --- /dev/null +++ b/061_dense_mol/dense_k5_all_s256_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f4a3ec71f936e5044b6bced8e598c58a1b43efa499d2701f8107b8698841a02 +size 341273328 diff --git a/061_dense_mol/dense_k5_all_s256_results.json b/061_dense_mol/dense_k5_all_s256_results.json new file mode 100644 index 0000000000000000000000000000000000000000..491f42872877c6cd0fe2110ce0d79df0bea6aba0 --- /dev/null +++ b/061_dense_mol/dense_k5_all_s256_results.json @@ -0,0 +1,2059 @@ +{ + "label": "dense_k5_all_s256", + "top_k_blocks": null, + "num_thin_blocks": 5, + "total_params": 85260288, + "router_params": 0, + "dense_ffn": true, + "d_ff_thin": 1024, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.019495010375977, + 13.558667182922363, + 8.007065773010254, + 7.493119716644287, + 7.367128849029541, + 7.240716457366943, + 7.143317222595215, + 6.995200157165527, + 6.850440979003906, + 6.762411117553711, + 6.628208160400391, + 6.497525215148926, + 6.400839805603027, + 6.318500995635986, + 6.313564777374268, + 6.092601776123047, + 6.091847896575928, + 5.98136043548584, + 5.922291278839111, + 5.83870792388916, + 5.822402000427246, + 5.831479072570801, + 5.713124752044678, + 5.591527462005615, + 5.661971092224121, + 5.604327201843262, + 5.539680004119873, + 5.448101043701172, + 5.5277509689331055, + 5.453388690948486, + 5.399202823638916, + 5.35145378112793, + 5.385952949523926, + 5.22157096862793, + 5.265182018280029, + 5.326718807220459, + 5.328843116760254, + 5.180933475494385, + 5.163289546966553, + 5.288060188293457, + 5.205002784729004, + 5.128995418548584, + 5.0735039710998535, + 5.039422035217285, + 5.130804538726807, + 5.001859664916992, + 5.051473140716553, + 5.013697624206543, + 4.962174892425537, + 5.032228946685791, + 4.928572654724121, + 5.003533840179443, + 5.014918327331543, + 4.8830790519714355, + 4.9702677726745605, + 4.94411039352417, + 4.8754706382751465, + 4.866363525390625, + 4.888963222503662, + 4.785062789916992, + 4.781894207000732, + 4.846888065338135, + 4.863199234008789, + 4.902984619140625, + 4.858797550201416, + 4.817627906799316, + 4.761096000671387, + 4.726353645324707, + 4.888665676116943, + 4.715950012207031, + 4.830822944641113, + 4.726360321044922, + 4.745944023132324, + 4.541954040527344, + 4.796708106994629, + 4.709839344024658, + 4.597582817077637, + 4.642263412475586, + 4.77793025970459, + 4.707329273223877, + 4.67863130569458, + 4.631631851196289, + 4.65395450592041, + 4.653307914733887, + 4.479835033416748, + 4.771848678588867, + 4.739274024963379, + 4.666484355926514, + 4.677361965179443, + 4.595808506011963, + 4.54573392868042, + 4.567105770111084, + 4.7419304847717285, + 4.660676956176758, + 4.4848551750183105, + 4.614228248596191, + 4.518594741821289, + 4.709370136260986, + 4.591193199157715, + 4.562418460845947, + 4.570526599884033, + 4.480901718139648, + 4.427608966827393, + 4.547603607177734, + 4.537751197814941, + 4.586178302764893, + 4.531315326690674, + 4.408409118652344, + 4.585477828979492, + 4.488409042358398, + 4.53167724609375, + 4.490981578826904, + 4.538548469543457, + 4.522124290466309, + 4.4054670333862305, + 4.418276786804199, + 4.483567237854004, + 4.328851699829102, + 4.450702667236328, + 4.348128318786621, + 4.245718479156494, + 4.4905924797058105, + 4.458269119262695, + 4.415597915649414, + 4.366002559661865, + 4.461676597595215, + 4.396079063415527, + 4.380098342895508, + 4.299656867980957, + 4.24072790145874, + 4.295907020568848, + 4.4606781005859375, + 4.293857097625732, + 4.393674373626709, + 4.486939907073975, + 4.334892749786377, + 4.292394161224365, + 4.281396389007568, + 4.295592784881592, + 4.370272159576416, + 4.319377422332764, + 4.169338703155518, + 4.298633575439453, + 4.290559768676758, + 4.127170085906982, + 4.1702446937561035, + 4.283299922943115, + 4.154012203216553, + 4.3729472160339355, + 4.375343322753906, + 4.331050395965576, + 4.19765043258667, + 4.371118545532227, + 4.333089828491211, + 4.302001953125, + 4.2389373779296875, + 4.321296691894531, + 4.299225807189941, + 4.33120584487915, + 4.293947219848633, + 4.318002700805664, + 4.23565673828125, + 4.190417289733887, + 4.293069839477539, + 4.202868461608887, + 4.233086109161377, + 4.2321672439575195, + 4.219963550567627, + 4.28984260559082, + 4.271109580993652, + 4.141883373260498, + 4.026560306549072, + 4.101181983947754, + 4.115406036376953, + 4.060061931610107, + 4.120220184326172, + 4.2788405418396, + 4.106910705566406, + 4.166386127471924, + 4.119293689727783, + 4.224097728729248, + 4.300160884857178, + 4.162077903747559, + 4.212298393249512, + 4.226622581481934, + 4.264876842498779, + 4.228888034820557, + 4.068085193634033, + 4.002753257751465, + 4.1328253746032715, + 4.229259967803955, + 4.2165679931640625, + 4.077999114990234, + 4.18774938583374, + 4.194653034210205, + 4.165799140930176, + 4.209352016448975, + 4.187953472137451, + 4.140440464019775, + 4.141944885253906, + 4.084014892578125, + 4.097827434539795, + 4.148556232452393, + 4.1014084815979, + 4.13107442855835, + 4.051477909088135, + 4.154263496398926, + 4.089181900024414, + 4.1172637939453125, + 4.128264904022217, + 4.156362533569336, + 4.171823024749756, + 4.061779499053955, + 4.049281597137451, + 4.056463241577148, + 3.9972169399261475, + 4.0824480056762695, + 4.141080379486084, + 4.023201942443848, + 4.131373882293701, + 4.111171722412109, + 4.112490177154541, + 4.035871982574463, + 4.121306419372559, + 4.0026655197143555, + 4.046724319458008, + 4.0891571044921875, + 4.102759838104248, + 4.098142147064209, + 4.188315391540527, + 4.172951698303223, + 4.031005382537842, + 4.172188758850098, + 4.019253730773926, + 3.9663219451904297, + 4.040327072143555, + 3.9682934284210205, + 4.02146577835083, + 4.150362014770508, + 3.9350128173828125, + 4.074317455291748, + 3.9277007579803467, + 4.076138019561768, + 3.9810054302215576, + 4.023192882537842, + 4.001926898956299, + 4.040445804595947, + 3.9370293617248535, + 4.092470169067383, + 4.027658462524414, + 3.9922776222229004, + 4.048092842102051, + 3.991079092025757, + 3.8460001945495605, + 3.8985681533813477, + 3.9748992919921875, + 3.9406120777130127, + 3.892887592315674, + 4.00808048248291, + 4.041107177734375, + 3.9915125370025635, + 3.8148252964019775, + 4.046282768249512, + 3.983563184738159, + 4.049823760986328, + 4.033483028411865, + 3.8259689807891846, + 3.8728911876678467, + 3.9147379398345947, + 4.0114054679870605, + 3.8454058170318604, + 3.915984630584717, + 3.8536770343780518, + 3.9841809272766113, + 3.928537368774414, + 4.013866424560547, + 3.952296018600464, + 3.897509813308716, + 3.9709856510162354, + 3.9054083824157715, + 3.8814966678619385, + 3.858962297439575, + 3.986441135406494, + 3.951659679412842, + 3.8845062255859375, + 3.823517322540283, + 3.9128551483154297, + 3.9134509563446045, + 3.8835880756378174, + 3.6996355056762695, + 3.9805381298065186, + 4.0512895584106445, + 3.7052135467529297, + 3.9026832580566406, + 3.915170907974243, + 3.8090310096740723, + 3.899012327194214, + 4.007925510406494, + 3.968637466430664, + 3.9670250415802, + 3.892470598220825, + 3.9242734909057617, + 3.8146796226501465, + 3.902459144592285, + 3.9718267917633057, + 3.975972890853882, + 3.7447288036346436, + 3.7947916984558105, + 3.798563003540039, + 3.990830898284912, + 3.9357287883758545, + 3.961684226989746, + 3.7616355419158936, + 3.8686957359313965, + 3.804537296295166, + 3.8821980953216553, + 4.02338981628418, + 3.955836534500122, + 3.968351364135742, + 3.868499517440796, + 3.997755527496338, + 3.8889408111572266, + 3.8425333499908447, + 3.8870928287506104, + 3.9481253623962402, + 3.976494312286377, + 3.8825199604034424, + 3.797614097595215, + 4.005147933959961, + 3.833184003829956, + 3.9301679134368896, + 3.9142158031463623, + 3.898435354232788, + 3.867567539215088, + 3.9281697273254395, + 4.020705699920654, + 3.7739925384521484, + 3.790783405303955, + 3.938553810119629, + 3.866231918334961, + 3.8668947219848633, + 3.783010244369507, + 3.876462697982788, + 3.9546666145324707, + 3.921091079711914, + 3.8387787342071533, + 4.01759672164917, + 3.9721763134002686, + 3.862086772918701, + 3.7397964000701904, + 3.919495105743408, + 3.878016948699951, + 3.880894899368286, + 3.8877482414245605, + 3.9466617107391357, + 3.9730334281921387, + 3.790644645690918, + 4.007716655731201, + 3.860316276550293, + 3.881340742111206, + 3.929307222366333, + 3.9998159408569336, + 3.7920591831207275, + 3.839343786239624, + 4.043306350708008, + 3.8203845024108887, + 3.9290781021118164, + 3.871309995651245, + 3.8176662921905518, + 3.719358205795288, + 3.7729759216308594, + 3.9327211380004883, + 3.9257824420928955, + 3.776959180831909, + 3.926356315612793, + 3.8171846866607666, + 3.808267831802368, + 3.828749895095825, + 3.730924606323242, + 3.826748847961426, + 3.6499433517456055, + 3.7146337032318115, + 3.7788290977478027, + 3.741673231124878, + 3.7977867126464844, + 3.708752393722534, + 3.7754921913146973, + 3.7732927799224854, + 3.810256242752075, + 3.8438403606414795, + 3.807896852493286, + 3.70965313911438, + 3.7571816444396973, + 3.7449474334716797, + 3.8864493370056152, + 3.904689073562622, + 3.8881964683532715, + 3.698371171951294, + 3.7139525413513184, + 3.856309175491333, + 3.779707431793213, + 3.7138566970825195, + 3.7563443183898926, + 3.688020706176758, + 3.8625001907348633, + 3.6756582260131836, + 3.681204319000244, + 3.8288984298706055, + 3.786566972732544, + 3.808321952819824, + 3.870598316192627, + 3.6431381702423096, + 3.7848927974700928, + 3.7564949989318848, + 3.7600460052490234, + 3.7990944385528564, + 3.8452258110046387, + 3.6367342472076416, + 3.6919260025024414, + 3.7936134338378906, + 3.755826711654663, + 3.8820722103118896, + 3.8815815448760986, + 3.8172600269317627, + 3.8323802947998047, + 3.783306121826172, + 3.7290725708007812, + 3.7704155445098877, + 3.761448383331299, + 3.802922487258911, + 3.8499903678894043, + 3.8254497051239014, + 3.63508939743042, + 3.841491222381592, + 3.7302210330963135, + 3.8089988231658936, + 3.704216480255127, + 3.658151149749756, + 3.845064640045166, + 3.641284704208374, + 3.8359525203704834, + 3.619882822036743, + 3.892226219177246, + 3.7666854858398438, + 3.6851582527160645, + 3.8655893802642822, + 3.7271623611450195, + 3.7481272220611572, + 3.679088592529297, + 3.7185637950897217, + 3.8698065280914307, + 3.761697292327881, + 3.795640230178833, + 3.6243274211883545, + 3.710585832595825, + 3.699603319168091, + 3.67350435256958, + 3.7242634296417236, + 3.6645896434783936, + 3.7531590461730957, + 3.585268020629883, + 3.7110378742218018, + 3.581939935684204, + 3.613252639770508, + 3.7453925609588623, + 3.708362102508545, + 3.7062857151031494, + 3.577601194381714, + 3.796419620513916, + 3.700956106185913, + 3.722849130630493, + 3.7444682121276855, + 3.772854804992676, + 3.741212844848633, + 3.8417117595672607, + 3.6060564517974854, + 3.7635281085968018, + 3.725552797317505, + 3.6241519451141357, + 3.6447579860687256, + 3.656068801879883, + 3.7934534549713135, + 3.6956002712249756, + 3.7121951580047607, + 3.838376045227051, + 3.6444203853607178, + 3.6288552284240723, + 3.8562610149383545, + 3.716351270675659, + 3.73915433883667, + 3.692078113555908, + 3.806191921234131, + 3.7395689487457275, + 3.766632080078125, + 3.7431325912475586, + 3.713505268096924, + 3.777395725250244, + 3.763425350189209, + 3.799222946166992, + 3.6615231037139893, + 3.571730375289917, + 3.754207134246826, + 3.744210720062256, + 3.7494957447052, + 3.7438178062438965, + 3.478647232055664, + 3.5705883502960205, + 3.589460611343384, + 3.643075466156006, + 3.4526758193969727, + 3.5682618618011475, + 3.523115634918213, + 3.578270435333252, + 3.6042628288269043, + 3.6502628326416016, + 3.6658225059509277, + 3.581775188446045, + 3.455570936203003, + 3.533430814743042, + 3.628495931625366, + 3.5877606868743896, + 3.5971930027008057, + 3.710330009460449, + 3.685359477996826, + 3.627556324005127, + 3.5788683891296387, + 3.554356098175049, + 3.6874983310699463, + 3.752591848373413, + 3.546478748321533, + 3.540759801864624, + 3.644268274307251, + 3.540673017501831, + 3.6665546894073486, + 3.6729469299316406, + 3.58046555519104, + 3.731553316116333, + 3.7447385787963867, + 3.620164632797241, + 3.6044187545776367, + 3.5956039428710938, + 3.6074068546295166, + 3.6327922344207764, + 3.614204168319702, + 3.652409791946411, + 3.5430450439453125, + 3.678814172744751, + 3.6763925552368164, + 3.5733044147491455, + 3.5564823150634766, + 3.638958692550659, + 3.6024951934814453, + 3.646071195602417, + 3.657011032104492, + 3.8232228755950928, + 3.6462976932525635, + 3.6736347675323486, + 3.6906702518463135, + 3.5955653190612793, + 3.6672780513763428, + 3.569485902786255, + 3.5610532760620117, + 3.5524494647979736, + 3.640134334564209, + 3.744155168533325, + 3.631195068359375, + 3.6104671955108643, + 3.5219669342041016, + 3.627746105194092, + 3.616933822631836, + 3.5175981521606445, + 3.5654702186584473, + 3.588677167892456, + 3.693169116973877, + 3.6412713527679443, + 3.5047390460968018, + 3.644380807876587, + 3.5158333778381348, + 3.55293869972229, + 3.6272964477539062, + 3.491868495941162, + 3.5927510261535645, + 3.6914501190185547, + 3.6403489112854004, + 3.728670358657837, + 3.5885632038116455, + 3.6922473907470703, + 3.691845178604126, + 3.592721939086914, + 3.530715227127075, + 3.614788770675659, + 3.4210433959960938, + 3.590981960296631, + 3.5859086513519287, + 3.4499967098236084, + 3.4774527549743652, + 3.520430326461792, + 3.6033735275268555, + 3.672276496887207, + 3.6360127925872803, + 3.4865198135375977, + 3.6208269596099854, + 3.5786666870117188, + 3.5132791996002197, + 3.578200578689575, + 3.5855984687805176, + 3.5282583236694336, + 3.537041187286377, + 3.5092673301696777, + 3.6548242568969727, + 3.6925864219665527, + 3.65828013420105, + 3.526766777038574, + 3.5756986141204834, + 3.5037763118743896, + 3.4909682273864746, + 3.4767069816589355, + 3.6390256881713867, + 3.669832944869995, + 3.520414113998413, + 3.729243040084839, + 3.6457762718200684, + 3.5324606895446777, + 3.5422067642211914, + 3.5953235626220703, + 3.7381386756896973, + 3.6007447242736816, + 3.649566650390625, + 3.6223061084747314, + 3.546353816986084, + 3.5792665481567383, + 3.5540692806243896, + 3.449378728866577, + 3.4170846939086914, + 3.57906436920166, + 3.53779673576355, + 3.600761651992798, + 3.59393572807312, + 3.4703283309936523, + 3.4405500888824463, + 3.4929373264312744, + 3.4645817279815674, + 3.6202523708343506, + 3.420787811279297, + 3.57079815864563, + 3.510267734527588, + 3.5278420448303223, + 3.502570629119873, + 3.560814142227173, + 3.4953598976135254, + 3.6195626258850098, + 3.403589963912964, + 3.5661416053771973, + 3.572021245956421, + 3.4899649620056152, + 3.4794180393218994, + 3.4928603172302246, + 3.436211109161377, + 3.365476131439209, + 3.585003137588501, + 3.5415918827056885, + 3.429086685180664, + 3.5510168075561523, + 3.473977565765381, + 3.602668285369873, + 3.508171319961548, + 3.5822150707244873, + 3.571887493133545, + 3.5108227729797363, + 3.5132806301116943, + 3.5015757083892822, + 3.6294212341308594, + 3.556992292404175, + 3.4963197708129883, + 3.5403811931610107, + 3.48275089263916, + 3.236269474029541, + 3.474687099456787, + 3.5062389373779297, + 3.57782244682312, + 3.4481844902038574, + 3.658733606338501, + 3.445777177810669, + 3.579282283782959, + 3.547522783279419, + 3.354818820953369, + 3.575773000717163, + 3.5317726135253906, + 3.4784858226776123, + 3.6269633769989014, + 3.4733738899230957, + 3.5045247077941895, + 3.4623334407806396, + 3.496983051300049, + 3.474395990371704, + 3.4220285415649414, + 3.6675615310668945, + 3.4503087997436523, + 3.507509469985962, + 3.5890698432922363, + 3.4866065979003906, + 3.514876365661621, + 3.5851829051971436, + 3.5123817920684814, + 3.433122158050537, + 3.5262210369110107, + 3.5475146770477295, + 3.434695243835449, + 3.609092950820923, + 3.523733139038086, + 3.6480040550231934, + 3.4173624515533447, + 3.480095624923706, + 3.4459335803985596, + 3.4457662105560303, + 3.5400454998016357, + 3.5354669094085693, + 3.5267210006713867, + 3.434474468231201, + 3.491962432861328, + 3.557934045791626, + 3.5644123554229736, + 3.497002124786377, + 3.4813857078552246, + 3.55317759513855, + 3.472219467163086, + 3.46329402923584, + 3.5840272903442383, + 3.654616355895996, + 3.537102460861206, + 3.4402456283569336, + 3.4987363815307617, + 3.6563358306884766, + 3.5124754905700684, + 3.5456812381744385, + 3.5726969242095947, + 3.587198495864868, + 3.4700369834899902, + 3.5393483638763428, + 3.5127246379852295, + 3.536125898361206, + 3.612717866897583, + 3.429800510406494, + 3.4696812629699707, + 3.4381513595581055, + 3.5195703506469727, + 3.5952670574188232, + 3.4245777130126953, + 3.5614261627197266, + 3.5561821460723877, + 3.4217355251312256, + 3.4489049911499023, + 3.5035579204559326, + 3.580087184906006, + 3.446425199508667, + 3.4484472274780273, + 3.4892029762268066, + 3.4877474308013916, + 3.4171247482299805, + 3.4606831073760986, + 3.4232900142669678, + 3.4649455547332764, + 3.3485429286956787, + 3.3371741771698, + 3.3954410552978516, + 3.370288133621216, + 3.4235849380493164, + 3.35575532913208, + 3.3921470642089844, + 3.485717296600342, + 3.4349524974823, + 3.427034854888916, + 3.336132764816284, + 3.443713903427124, + 3.518669366836548, + 3.459157943725586, + 3.4989283084869385, + 3.481133460998535, + 3.506707191467285, + 3.5277469158172607, + 3.400986671447754, + 3.4452009201049805, + 3.3787944316864014, + 3.540311574935913, + 3.4515345096588135, + 3.48172926902771, + 3.4142136573791504, + 3.5630455017089844, + 3.328829765319824, + 3.4558908939361572, + 3.370704412460327, + 3.303133010864258, + 3.4458258152008057, + 3.489267349243164, + 3.553694486618042, + 3.4475958347320557, + 3.54015851020813, + 3.4829463958740234, + 3.4256231784820557, + 3.4411544799804688, + 3.317650556564331, + 3.4064571857452393, + 3.4641361236572266, + 3.4614996910095215, + 3.4899849891662598, + 3.5969622135162354, + 3.456549882888794, + 3.318432092666626, + 3.665945053100586, + 3.4452288150787354, + 3.5584399700164795, + 3.556673049926758, + 3.5085670948028564, + 3.437192678451538, + 3.3447494506835938, + 3.605506181716919, + 3.354952573776245, + 3.436143159866333, + 3.384577989578247, + 3.520575523376465, + 3.4699971675872803, + 3.55548095703125, + 3.4344122409820557, + 3.3685760498046875, + 3.474656105041504, + 3.460489273071289, + 3.302225351333618, + 3.4417788982391357, + 3.382295608520508, + 3.464447259902954, + 3.460878372192383, + 3.4256255626678467, + 3.40998911857605, + 3.4461991786956787, + 3.521552562713623, + 3.4971306324005127, + 3.445467710494995, + 3.4433648586273193, + 3.3802008628845215, + 3.4521260261535645, + 3.383741855621338, + 3.243669271469116, + 3.5159854888916016, + 3.584282875061035, + 3.4409196376800537, + 3.430327892303467, + 3.492851972579956, + 3.430006504058838, + 3.4160079956054688, + 3.462723731994629, + 3.5212600231170654, + 3.380495548248291, + 3.2965099811553955, + 3.3238091468811035, + 3.4519386291503906, + 3.4240827560424805, + 3.4214415550231934, + 3.515002489089966, + 3.376812219619751, + 3.439600944519043, + 3.4493818283081055, + 3.364942789077759, + 3.409226655960083, + 3.4661753177642822, + 3.3011531829833984, + 3.550492525100708, + 3.4539246559143066, + 3.383395195007324, + 3.4399333000183105, + 3.3953683376312256, + 3.5315680503845215, + 3.471384286880493, + 3.350253105163574, + 3.3806369304656982, + 3.337001085281372, + 3.444955348968506, + 3.4456470012664795, + 3.462256908416748, + 3.5170483589172363, + 3.5300965309143066, + 3.588928461074829, + 3.4583184719085693, + 3.3619120121002197, + 3.431173324584961, + 3.375530481338501, + 3.4213035106658936, + 3.428251028060913, + 3.519094944000244, + 3.423607110977173, + 3.4474480152130127, + 3.4318220615386963, + 3.528811454772949, + 3.326794147491455, + 3.4469025135040283, + 3.4636495113372803, + 3.41434383392334, + 3.3173179626464844, + 3.4667208194732666, + 3.530104637145996, + 3.390273332595825, + 3.353523015975952, + 3.5316600799560547, + 3.5136232376098633, + 3.4352848529815674, + 3.429135322570801, + 3.4814138412475586, + 3.3851587772369385, + 3.4667861461639404, + 3.386258125305176, + 3.4396166801452637, + 3.354679584503174, + 3.3571436405181885, + 3.274371862411499, + 3.277773380279541, + 3.473881721496582, + 3.413961887359619, + 3.436523675918579, + 3.4863924980163574, + 3.3682570457458496, + 3.4528865814208984, + 3.31929087638855, + 3.280891180038452, + 3.42353892326355, + 3.4269888401031494, + 3.462616443634033, + 3.365414619445801, + 3.3878633975982666, + 3.2904059886932373, + 3.434213638305664, + 3.487523317337036, + 3.36797833442688, + 3.337939500808716, + 3.4121007919311523, + 3.4912140369415283, + 3.449023723602295, + 3.433847665786743, + 3.4809072017669678, + 3.4920639991760254, + 3.4120800495147705, + 3.3290300369262695, + 3.3762214183807373, + 3.3402321338653564, + 3.491748332977295, + 3.3608970642089844, + 3.5094215869903564, + 3.433467149734497, + 3.3798317909240723, + 3.3346312046051025, + 3.4571309089660645, + 3.4782066345214844, + 3.2983756065368652, + 3.416426181793213, + 3.384437084197998, + 3.3424103260040283, + 3.534454345703125, + 3.362184762954712, + 3.4094064235687256, + 3.4746832847595215, + 3.4393298625946045, + 3.467442274093628, + 3.3890788555145264, + 3.2871696949005127, + 3.284111976623535, + 3.3607141971588135, + 3.392052173614502, + 3.444640874862671, + 3.3049893379211426, + 3.430802583694458, + 3.4208223819732666, + 3.337644338607788, + 3.337085008621216, + 3.4859163761138916, + 3.3229660987854004, + 3.4605553150177, + 3.4508893489837646, + 3.4707963466644287, + 3.337070941925049, + 3.353111505508423, + 3.4626376628875732, + 3.435950994491577, + 3.274282693862915, + 3.470839262008667, + 3.4162654876708984, + 3.335642099380493, + 3.4007644653320312, + 3.288599729537964, + 3.343057870864868, + 3.3159737586975098, + 3.4802513122558594, + 3.3600001335144043, + 3.336838960647583, + 3.453965902328491, + 3.3771185874938965, + 3.487729072570801, + 3.3935277462005615, + 3.35244083404541, + 3.3766980171203613, + 3.4484097957611084, + 3.3510289192199707, + 3.3849589824676514 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 134.25645302340072, + 83.59583924651527, + 64.59044687411033, + 55.31614922921644, + 49.71289923374855, + 46.012424495289835, + 43.46566773517377, + 41.246580752793285, + 39.25371707018053, + 37.91970771185352, + 36.58068653430312, + 35.34553019181296, + 34.56204617884562, + 33.70096504565297, + 33.05444566977557, + 32.64861146293815, + 32.2560294420313, + 32.11167379052898, + 32.0087769467745, + 31.98048427242423 + ], + "best_ppl": 31.98048427242423, + "train_time_s": 17429.111253738403 +} \ No newline at end of file diff --git a/061_dense_mol/dense_k5_top3_best.pt b/061_dense_mol/dense_k5_top3_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e207e081204778f4244a19b59962a7055346843 --- /dev/null +++ b/061_dense_mol/dense_k5_top3_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6938104655f69f97dde1025e9e5b6629a5c599211fe819ace25a4e39c7a16e97 +size 341438292 diff --git a/061_dense_mol/dense_k5_top3_results.json b/061_dense_mol/dense_k5_top3_results.json new file mode 100644 index 0000000000000000000000000000000000000000..57601348ab5484e986af8ba3dd2bc9683a617913 --- /dev/null +++ b/061_dense_mol/dense_k5_top3_results.json @@ -0,0 +1,2059 @@ +{ + "label": "dense_k5_top3", + "top_k_blocks": 3, + "num_thin_blocks": 5, + "total_params": 85301248, + "router_params": 40960, + "dense_ffn": true, + "d_ff_thin": 1024, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.075403213500977, + 14.989167213439941, + 7.974791526794434, + 7.316899299621582, + 7.045978546142578, + 6.952646732330322, + 6.923568248748779, + 6.8025031089782715, + 6.650099277496338, + 6.569061279296875, + 6.473870754241943, + 6.38222599029541, + 6.361185550689697, + 6.326168537139893, + 6.199742317199707, + 6.191432952880859, + 6.043409824371338, + 6.017180919647217, + 5.866113185882568, + 5.80708122253418, + 5.822342395782471, + 5.693378448486328, + 5.685535907745361, + 5.648721694946289, + 5.651585578918457, + 5.537415027618408, + 5.53345251083374, + 5.511941909790039, + 5.347475051879883, + 5.386723041534424, + 5.268617630004883, + 5.272661209106445, + 5.279953956604004, + 5.172795295715332, + 5.134716510772705, + 5.097721099853516, + 5.141532897949219, + 5.206697940826416, + 5.16818904876709, + 5.055859565734863, + 5.0862226486206055, + 5.018344879150391, + 4.898447036743164, + 5.0112104415893555, + 4.958449840545654, + 5.000402450561523, + 4.91301965713501, + 4.992534637451172, + 4.997373104095459, + 4.915457248687744, + 4.902679920196533, + 4.766676902770996, + 4.821236610412598, + 4.8017578125, + 4.816453456878662, + 4.713117599487305, + 4.836438179016113, + 4.74616813659668, + 4.7350239753723145, + 4.747159957885742, + 4.812458515167236, + 4.774238109588623, + 4.839298725128174, + 4.700974464416504, + 4.7263665199279785, + 4.677390098571777, + 4.7095255851745605, + 4.767474174499512, + 4.870450496673584, + 4.716841220855713, + 4.582910060882568, + 4.689180374145508, + 4.525135517120361, + 4.590797424316406, + 4.55123233795166, + 4.537211894989014, + 4.708263397216797, + 4.5492844581604, + 4.620543956756592, + 4.676320552825928, + 4.539761066436768, + 4.43330192565918, + 4.536831378936768, + 4.546805381774902, + 4.667558670043945, + 4.652440547943115, + 4.366703510284424, + 4.539286136627197, + 4.549784183502197, + 4.498823642730713, + 4.6177287101745605, + 4.572345733642578, + 4.438539505004883, + 4.300197124481201, + 4.558171272277832, + 4.542840480804443, + 4.379049777984619, + 4.449305057525635, + 4.378256797790527, + 4.4633636474609375, + 4.407858371734619, + 4.419273376464844, + 4.385214805603027, + 4.351011276245117, + 4.46150016784668, + 4.413872241973877, + 4.340606212615967, + 4.5852952003479, + 4.336183547973633, + 4.451183795928955, + 4.433951377868652, + 4.303919315338135, + 4.422789096832275, + 4.371076583862305, + 4.391994476318359, + 4.416762828826904, + 4.272546291351318, + 4.421780109405518, + 4.352207660675049, + 4.292917728424072, + 4.394893646240234, + 4.3645830154418945, + 4.245812892913818, + 4.336083889007568, + 4.307858943939209, + 4.364508628845215, + 4.3674492835998535, + 4.267165184020996, + 4.263680934906006, + 4.322206974029541, + 4.232335567474365, + 4.32529878616333, + 4.239583492279053, + 4.275751113891602, + 4.166499137878418, + 4.153955459594727, + 4.204243183135986, + 4.261561393737793, + 4.285508632659912, + 4.235783100128174, + 4.16873025894165, + 4.181886196136475, + 4.174830913543701, + 4.090977191925049, + 4.270376205444336, + 4.108957290649414, + 4.180401802062988, + 4.260923862457275, + 4.250618934631348, + 4.259207725524902, + 4.298221111297607, + 4.13369607925415, + 4.182296276092529, + 4.195666790008545, + 4.025666236877441, + 4.183835029602051, + 4.153615951538086, + 4.130311965942383, + 4.13405179977417, + 4.0964460372924805, + 4.183577537536621, + 4.173470973968506, + 4.146329402923584, + 4.192317008972168, + 4.081483364105225, + 4.262228488922119, + 4.240663528442383, + 4.209317207336426, + 4.081004619598389, + 3.974327564239502, + 4.138270854949951, + 4.191417694091797, + 4.1985249519348145, + 4.073193073272705, + 4.293991565704346, + 4.213837146759033, + 4.052644729614258, + 4.126652240753174, + 4.2121171951293945, + 4.150744438171387, + 4.1465020179748535, + 4.0969977378845215, + 4.131259918212891, + 4.095874309539795, + 4.1388139724731445, + 4.072798252105713, + 3.9587440490722656, + 4.0928263664245605, + 4.022637844085693, + 4.0341901779174805, + 4.149160385131836, + 4.103133201599121, + 3.989093542098999, + 3.9620883464813232, + 4.053597450256348, + 3.9696335792541504, + 3.9809770584106445, + 4.1297407150268555, + 4.130749225616455, + 3.977855920791626, + 4.127661228179932, + 4.10686731338501, + 4.083009243011475, + 4.176492214202881, + 3.9806573390960693, + 4.132237911224365, + 4.137613773345947, + 4.041726589202881, + 4.102180480957031, + 3.9454948902130127, + 3.909492254257202, + 4.157198429107666, + 3.967888355255127, + 4.027840614318848, + 3.9946815967559814, + 3.946765184402466, + 4.121756076812744, + 4.0088276863098145, + 4.015646457672119, + 3.8600854873657227, + 4.079128265380859, + 4.105773448944092, + 3.9194722175598145, + 3.958944320678711, + 4.121541500091553, + 3.909013509750366, + 4.035359859466553, + 4.128946781158447, + 3.9589791297912598, + 4.048763751983643, + 4.079703330993652, + 4.029348850250244, + 4.003383159637451, + 3.961308240890503, + 4.125159740447998, + 3.980497121810913, + 3.9527618885040283, + 3.9532525539398193, + 3.9532501697540283, + 3.8583626747131348, + 3.9990763664245605, + 3.9199018478393555, + 3.9854767322540283, + 4.081484794616699, + 4.119716644287109, + 4.0350165367126465, + 4.0060224533081055, + 3.9356210231781006, + 4.034941673278809, + 3.985532522201538, + 3.963094472885132, + 3.8805694580078125, + 3.9308552742004395, + 3.9987776279449463, + 3.7656397819519043, + 3.867485284805298, + 3.963148832321167, + 4.018829822540283, + 3.848897933959961, + 3.9583029747009277, + 3.965313196182251, + 3.8676226139068604, + 3.9656519889831543, + 3.915721893310547, + 3.896401882171631, + 3.961527109146118, + 3.9435131549835205, + 3.967223644256592, + 3.919398784637451, + 3.898069143295288, + 3.9972610473632812, + 3.802324056625366, + 4.069353103637695, + 3.818502902984619, + 3.954054832458496, + 3.856522560119629, + 3.9452104568481445, + 3.899731159210205, + 3.7890841960906982, + 3.854680061340332, + 3.8672871589660645, + 3.949467420578003, + 3.7570841312408447, + 3.9847140312194824, + 3.9547314643859863, + 3.9229156970977783, + 3.851053476333618, + 3.7998411655426025, + 3.8512930870056152, + 3.870837688446045, + 3.806760549545288, + 4.068005084991455, + 3.9847171306610107, + 3.843285322189331, + 3.9005510807037354, + 3.870894193649292, + 3.799272298812866, + 3.804290771484375, + 3.880964517593384, + 3.751694679260254, + 3.915362596511841, + 3.870753288269043, + 3.926307201385498, + 3.8984267711639404, + 3.809575080871582, + 3.837048053741455, + 3.8469619750976562, + 3.6650915145874023, + 4.014257431030273, + 3.92427659034729, + 3.980907917022705, + 3.931708335876465, + 3.7364730834960938, + 3.821730852127075, + 3.982510566711426, + 3.8441390991210938, + 3.8749191761016846, + 3.8975822925567627, + 3.7388479709625244, + 3.8392090797424316, + 3.847221612930298, + 3.905174970626831, + 3.846639394760132, + 3.917210102081299, + 3.7535417079925537, + 3.9027676582336426, + 3.7595481872558594, + 3.8051791191101074, + 3.7575221061706543, + 3.7620601654052734, + 3.7291667461395264, + 3.874267101287842, + 3.760453462600708, + 3.915330648422241, + 3.7925899028778076, + 3.6958084106445312, + 3.804497480392456, + 3.761434555053711, + 3.8464503288269043, + 3.7715797424316406, + 3.850024938583374, + 3.6282644271850586, + 3.862894296646118, + 3.924060106277466, + 3.6487865447998047, + 3.8777949810028076, + 3.8169806003570557, + 3.8006818294525146, + 3.7507286071777344, + 3.9184744358062744, + 3.6654746532440186, + 3.891055107116699, + 3.8051013946533203, + 3.767164945602417, + 3.835822343826294, + 3.900615930557251, + 3.889615535736084, + 3.793869733810425, + 3.7766950130462646, + 3.7516684532165527, + 3.8042707443237305, + 3.8501534461975098, + 3.7790639400482178, + 3.6576669216156006, + 3.855914354324341, + 3.8263165950775146, + 3.9236581325531006, + 3.907761335372925, + 3.8678817749023438, + 3.8998935222625732, + 3.811547040939331, + 3.7694358825683594, + 3.828695058822632, + 3.67197847366333, + 3.839653491973877, + 3.8139007091522217, + 3.7314891815185547, + 3.7922441959381104, + 3.745013475418091, + 3.7600574493408203, + 3.663073778152466, + 3.6969642639160156, + 3.6871261596679688, + 3.5982584953308105, + 3.6668496131896973, + 3.6937403678894043, + 3.6499147415161133, + 3.7489566802978516, + 3.713512659072876, + 3.8096489906311035, + 3.6800906658172607, + 3.6264994144439697, + 3.7136967182159424, + 3.7111499309539795, + 3.8622121810913086, + 3.7323482036590576, + 3.712830066680908, + 3.730680465698242, + 3.8969030380249023, + 3.6546471118927, + 3.58781361579895, + 3.6958439350128174, + 3.7240569591522217, + 3.762712240219116, + 3.76888370513916, + 3.603067398071289, + 3.7851531505584717, + 3.7559878826141357, + 3.727311611175537, + 3.768942356109619, + 3.8074939250946045, + 3.7096426486968994, + 3.7443621158599854, + 3.6727802753448486, + 3.8493759632110596, + 3.6775357723236084, + 3.7971293926239014, + 3.6747233867645264, + 3.661012887954712, + 3.7596328258514404, + 3.6454975605010986, + 3.62186598777771, + 3.7299180030822754, + 3.7609381675720215, + 3.7979376316070557, + 3.760643720626831, + 3.6996676921844482, + 3.6904470920562744, + 3.7023448944091797, + 3.649442672729492, + 3.698704957962036, + 3.727490186691284, + 3.8325626850128174, + 3.7274057865142822, + 3.7987685203552246, + 3.841921806335449, + 3.7724616527557373, + 3.643721103668213, + 3.6564455032348633, + 3.725161552429199, + 3.696725606918335, + 3.623659372329712, + 3.7625248432159424, + 3.754913568496704, + 3.62156081199646, + 3.6470391750335693, + 3.769083261489868, + 3.6887011528015137, + 3.752091884613037, + 3.687147617340088, + 3.6607141494750977, + 3.6852705478668213, + 3.685969829559326, + 3.6649281978607178, + 3.704876184463501, + 3.623084783554077, + 3.7516250610351562, + 3.784299373626709, + 3.55395245552063, + 3.7768774032592773, + 3.712064027786255, + 3.7493226528167725, + 3.6631479263305664, + 3.697324275970459, + 3.6992733478546143, + 3.6750452518463135, + 3.6886892318725586, + 3.5124526023864746, + 3.748000144958496, + 3.6186678409576416, + 3.569695234298706, + 3.714470863342285, + 3.630134344100952, + 3.614320993423462, + 3.7223310470581055, + 3.724600076675415, + 3.6251237392425537, + 3.579988718032837, + 3.67254900932312, + 3.8007593154907227, + 3.6368141174316406, + 3.7231698036193848, + 3.624217987060547, + 3.656181812286377, + 3.55423903465271, + 3.5164716243743896, + 3.695706367492676, + 3.6423115730285645, + 3.7981507778167725, + 3.6303131580352783, + 3.6086602210998535, + 3.761775493621826, + 3.7705652713775635, + 3.6218020915985107, + 3.730832815170288, + 3.6205193996429443, + 3.710294485092163, + 3.7490737438201904, + 3.6782588958740234, + 3.683798313140869, + 3.7080249786376953, + 3.681156635284424, + 3.691251754760742, + 3.6459224224090576, + 3.6040921211242676, + 3.7037510871887207, + 3.3839900493621826, + 3.605354070663452, + 3.6601743698120117, + 3.596463918685913, + 3.6073312759399414, + 3.5043787956237793, + 3.5336742401123047, + 3.650502920150757, + 3.5603182315826416, + 3.6280243396759033, + 3.597315549850464, + 3.533137083053589, + 3.6575889587402344, + 3.596325397491455, + 3.6947145462036133, + 3.5177903175354004, + 3.483350992202759, + 3.4869635105133057, + 3.5625391006469727, + 3.6159732341766357, + 3.676380157470703, + 3.5903401374816895, + 3.7576069831848145, + 3.507444381713867, + 3.4443562030792236, + 3.610513210296631, + 3.6407339572906494, + 3.5865907669067383, + 3.449589490890503, + 3.5710973739624023, + 3.581885814666748, + 3.6536598205566406, + 3.548919677734375, + 3.58304762840271, + 3.5286009311676025, + 3.6051502227783203, + 3.5719287395477295, + 3.6893041133880615, + 3.564760446548462, + 3.687356472015381, + 3.517230272293091, + 3.726813554763794, + 3.5476481914520264, + 3.5473098754882812, + 3.621694564819336, + 3.603632926940918, + 3.681542158126831, + 3.5983872413635254, + 3.612187147140503, + 3.6208648681640625, + 3.4636192321777344, + 3.4866156578063965, + 3.476950168609619, + 3.641845941543579, + 3.5926029682159424, + 3.546719551086426, + 3.591057777404785, + 3.601530075073242, + 3.4676408767700195, + 3.5994107723236084, + 3.603227138519287, + 3.5827574729919434, + 3.5598690509796143, + 3.39174222946167, + 3.7077555656433105, + 3.7378809452056885, + 3.644057512283325, + 3.57853627204895, + 3.5175728797912598, + 3.6368982791900635, + 3.482612133026123, + 3.418973207473755, + 3.6217870712280273, + 3.67699933052063, + 3.572803497314453, + 3.538877487182617, + 3.5262227058410645, + 3.466926097869873, + 3.563474178314209, + 3.580306053161621, + 3.482839345932007, + 3.581829309463501, + 3.5385923385620117, + 3.558835506439209, + 3.597960948944092, + 3.5741922855377197, + 3.5151312351226807, + 3.6078732013702393, + 3.644590377807617, + 3.564253807067871, + 3.5196826457977295, + 3.488978147506714, + 3.4970741271972656, + 3.5109035968780518, + 3.516702890396118, + 3.431809663772583, + 3.5298192501068115, + 3.6439170837402344, + 3.456594705581665, + 3.629016637802124, + 3.6548709869384766, + 3.6040937900543213, + 3.6138222217559814, + 3.527315139770508, + 3.445777177810669, + 3.541776418685913, + 3.467719554901123, + 3.6361725330352783, + 3.719386339187622, + 3.4869048595428467, + 3.6296191215515137, + 3.482227325439453, + 3.556656837463379, + 3.4307799339294434, + 3.484307289123535, + 3.6170780658721924, + 3.6569488048553467, + 3.538454055786133, + 3.5412847995758057, + 3.625998020172119, + 3.6016147136688232, + 3.63763165473938, + 3.5518693923950195, + 3.577702283859253, + 3.532686710357666, + 3.476989269256592, + 3.49741792678833, + 3.3243584632873535, + 3.3508124351501465, + 3.4929006099700928, + 3.4471921920776367, + 3.4432923793792725, + 3.4127604961395264, + 3.3719680309295654, + 3.3348782062530518, + 3.4782845973968506, + 3.4178659915924072, + 3.3323287963867188, + 3.4759223461151123, + 3.4285428524017334, + 3.476097822189331, + 3.462425470352173, + 3.522606372833252, + 3.5560860633850098, + 3.5087058544158936, + 3.3994925022125244, + 3.4713211059570312, + 3.487623929977417, + 3.546311140060425, + 3.4753377437591553, + 3.4386818408966064, + 3.408770799636841, + 3.466330051422119, + 3.588491678237915, + 3.4540493488311768, + 3.392791986465454, + 3.4692130088806152, + 3.4864912033081055, + 3.541126251220703, + 3.4569523334503174, + 3.394519090652466, + 3.487194299697876, + 3.478351593017578, + 3.493631362915039, + 3.5381031036376953, + 3.427137613296509, + 3.4802145957946777, + 3.4463603496551514, + 3.363511562347412, + 3.463134527206421, + 3.3433101177215576, + 3.47402024269104, + 3.566476345062256, + 3.370523691177368, + 3.4372644424438477, + 3.343724250793457, + 3.539490222930908, + 3.4595673084259033, + 3.582427978515625, + 3.444530487060547, + 3.4229421615600586, + 3.456425428390503, + 3.515674352645874, + 3.342395782470703, + 3.3550424575805664, + 3.4560389518737793, + 3.5809576511383057, + 3.4784750938415527, + 3.4425721168518066, + 3.5073630809783936, + 3.4915719032287598, + 3.390070676803589, + 3.57466459274292, + 3.5593886375427246, + 3.328718900680542, + 3.445460081100464, + 3.4627513885498047, + 3.380526542663574, + 3.569744110107422, + 3.555279016494751, + 3.4637460708618164, + 3.4524173736572266, + 3.468564748764038, + 3.450486183166504, + 3.3803670406341553, + 3.5236148834228516, + 3.462371349334717, + 3.345639705657959, + 3.5959761142730713, + 3.5248477458953857, + 3.6041226387023926, + 3.4863924980163574, + 3.5391359329223633, + 3.4710500240325928, + 3.4373409748077393, + 3.5355350971221924, + 3.4467597007751465, + 3.4987776279449463, + 3.4395740032196045, + 3.491356611251831, + 3.345045804977417, + 3.448625326156616, + 3.256446123123169, + 3.424288749694824, + 3.4314956665039062, + 3.387920618057251, + 3.4581589698791504, + 3.633350133895874, + 3.463810920715332, + 3.525944232940674, + 3.58835768699646, + 3.317380666732788, + 3.361647605895996, + 3.5087060928344727, + 3.358696937561035, + 3.3831326961517334, + 3.3844008445739746, + 3.585672378540039, + 3.2900443077087402, + 3.4880590438842773, + 3.4551198482513428, + 3.4368300437927246, + 3.6038687229156494, + 3.3507437705993652, + 3.5456154346466064, + 3.458970069885254, + 3.382749080657959, + 3.3804094791412354, + 3.4552297592163086, + 3.4986066818237305, + 3.397372007369995, + 3.280582904815674, + 3.424971580505371, + 3.3600356578826904, + 3.3544881343841553, + 3.352627992630005, + 3.370389223098755, + 3.169954299926758, + 3.4582889080047607, + 3.3048923015594482, + 3.5857479572296143, + 3.278881549835205, + 3.245007276535034, + 3.372415781021118, + 3.329606533050537, + 3.415239095687866, + 3.4227139949798584, + 3.381308078765869, + 3.3675317764282227, + 3.425814628601074, + 3.407216787338257, + 3.4053783416748047, + 3.3075828552246094, + 3.4413089752197266, + 3.3055858612060547, + 3.3844106197357178, + 3.4412784576416016, + 3.3825595378875732, + 3.467723846435547, + 3.2887399196624756, + 3.5158851146698, + 3.3807952404022217, + 3.445361852645874, + 3.378164291381836, + 3.2888705730438232, + 3.341156482696533, + 3.192218065261841, + 3.3424298763275146, + 3.2125744819641113, + 3.3294169902801514, + 3.3962137699127197, + 3.3603010177612305, + 3.363412618637085, + 3.2473270893096924, + 3.4680628776550293, + 3.3619682788848877, + 3.4818875789642334, + 3.5045130252838135, + 3.3834948539733887, + 3.2474491596221924, + 3.2601211071014404, + 3.2975542545318604, + 3.412837266921997, + 3.3973171710968018, + 3.420327663421631, + 3.298691987991333, + 3.4625043869018555, + 3.3940889835357666, + 3.392601490020752, + 3.3804383277893066, + 3.328023672103882, + 3.429267168045044, + 3.2795796394348145, + 3.4002974033355713, + 3.494746685028076, + 3.485123634338379, + 3.5142831802368164, + 3.3962955474853516, + 3.365927219390869, + 3.5144202709198, + 3.5537030696868896, + 3.453326463699341, + 3.281676769256592, + 3.485387086868286, + 3.171565055847168, + 3.4442310333251953, + 3.433985471725464, + 3.3043391704559326, + 3.389503240585327, + 3.3409125804901123, + 3.368046283721924, + 3.3636646270751953, + 3.4463047981262207, + 3.3511383533477783, + 3.35494327545166, + 3.4949839115142822, + 3.372856855392456, + 3.444216251373291, + 3.34566068649292, + 3.40592360496521, + 3.4154751300811768, + 3.3854966163635254, + 3.3266232013702393, + 3.1951422691345215, + 3.323286533355713, + 3.429849147796631, + 3.3733861446380615, + 3.303253412246704, + 3.416175127029419, + 3.4429054260253906, + 3.3210389614105225, + 3.3754093647003174, + 3.410459518432617, + 3.408219575881958, + 3.3178486824035645, + 3.5295822620391846, + 3.365846633911133, + 3.467912197113037, + 3.3188188076019287, + 3.375826358795166, + 3.4163856506347656, + 3.4525187015533447, + 3.3236231803894043, + 3.3350086212158203, + 3.437835693359375, + 3.341303586959839, + 3.3282759189605713, + 3.3167059421539307, + 3.2653377056121826, + 3.509014368057251, + 3.443267822265625, + 3.415100336074829, + 3.3209829330444336, + 3.335846185684204, + 3.487595319747925, + 3.3608150482177734, + 3.4554195404052734, + 3.3429830074310303, + 3.3777506351470947, + 3.369926929473877, + 3.3138985633850098, + 3.453381299972534, + 3.320465326309204, + 3.3377819061279297, + 3.3151042461395264, + 3.3077030181884766, + 3.4046247005462646, + 3.4102890491485596, + 3.363600969314575, + 3.257310390472412, + 3.373995304107666, + 3.470612049102783, + 3.473951816558838, + 3.262308359146118, + 3.31168532371521, + 3.3935258388519287, + 3.4533252716064453, + 3.194746255874634, + 3.392507314682007, + 3.4241597652435303, + 3.1953115463256836, + 3.3655776977539062, + 3.324328660964966, + 3.2652482986450195, + 3.381948709487915, + 3.3738558292388916, + 3.400282382965088, + 3.360177993774414, + 3.3973073959350586, + 3.381505012512207, + 3.3565309047698975, + 3.2850935459136963, + 3.318324565887451, + 3.3239214420318604, + 3.3613784313201904, + 3.370995283126831, + 3.378924608230591, + 3.3890671730041504, + 3.2881364822387695, + 3.403815984725952, + 3.3699862957000732, + 3.377501964569092, + 3.3402323722839355, + 3.4415810108184814, + 3.3485381603240967, + 3.3111307621002197, + 3.226907968521118, + 3.395601987838745, + 3.2962300777435303, + 3.2596771717071533, + 3.328366756439209, + 3.3719120025634766, + 3.4649577140808105, + 3.275653600692749, + 3.3282980918884277, + 3.4135024547576904, + 3.322807550430298, + 3.4011306762695312, + 3.257162094116211, + 3.351717948913574, + 3.317030668258667, + 3.3573105335235596, + 3.5096335411071777, + 3.4306185245513916, + 3.3938162326812744, + 3.353100061416626, + 3.2687013149261475, + 3.3160293102264404, + 3.3257458209991455, + 3.3502485752105713, + 3.3530521392822266, + 3.2810544967651367, + 3.3678321838378906, + 3.2910897731781006, + 3.352325677871704, + 3.3194310665130615, + 3.4183380603790283, + 3.4606590270996094, + 3.3975327014923096, + 3.3069961071014404, + 3.462376356124878, + 3.4779934883117676, + 3.3224146366119385, + 3.393801212310791, + 3.237741231918335, + 3.2660715579986572, + 3.387761354446411, + 3.310059070587158, + 3.4044189453125, + 3.3289432525634766, + 3.3865907192230225, + 3.3601627349853516, + 3.396707534790039, + 3.48393177986145, + 3.459846019744873, + 3.274170160293579, + 3.4041950702667236, + 3.3994827270507812, + 3.232515573501587, + 3.3710336685180664, + 3.3859055042266846, + 3.4429502487182617, + 3.2308173179626465, + 3.343024969100952, + 3.2563726902008057, + 3.413862466812134, + 3.285007953643799, + 3.2905352115631104, + 3.351003885269165, + 3.376110792160034, + 3.3374381065368652, + 3.3229429721832275, + 3.347733974456787, + 3.2658538818359375 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 121.1663761273476, + 73.89270509267668, + 59.4983634929169, + 51.71083945178506, + 47.356036688461046, + 44.299601870464166, + 41.4501709135491, + 39.54816061138143, + 37.951422401853165, + 36.34114729465708, + 35.3351387825715, + 34.137413094980126, + 33.308963713709446, + 32.56641838180231, + 31.850621546082934, + 31.46219717719894, + 31.180287791043902, + 30.97125623552679, + 30.877214285862838, + 30.852343859419506 + ], + "best_ppl": 30.852343859419506, + "train_time_s": 20325.323418855667 +} \ No newline at end of file diff --git a/061_dense_mol/dense_k5_top3_s137_best.pt b/061_dense_mol/dense_k5_top3_s137_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..4172d29e2f0f4d9685b243b067cf9124d73b7b34 --- /dev/null +++ b/061_dense_mol/dense_k5_top3_s137_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7897467215e23a62390bb5e0980d0d1a2179a1b95f1a1b1c53805e91f75a082 +size 341441241 diff --git a/061_dense_mol/dense_k5_top3_s137_results.json b/061_dense_mol/dense_k5_top3_s137_results.json new file mode 100644 index 0000000000000000000000000000000000000000..119f2d6f4fbe852a51113e4a45fc0c2e58e778b9 --- /dev/null +++ b/061_dense_mol/dense_k5_top3_s137_results.json @@ -0,0 +1,2059 @@ +{ + "label": "dense_k5_top3_s137", + "top_k_blocks": 3, + "num_thin_blocks": 5, + "total_params": 85301248, + "router_params": 40960, + "dense_ffn": true, + "d_ff_thin": 1024, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.004966735839844, + 14.779706001281738, + 7.931752681732178, + 7.278409004211426, + 7.102393627166748, + 6.9604997634887695, + 6.856429576873779, + 6.814227104187012, + 6.784242153167725, + 6.564796447753906, + 6.432928562164307, + 6.426309108734131, + 6.368070125579834, + 6.243048667907715, + 6.134979248046875, + 6.076487064361572, + 6.127120018005371, + 5.909240245819092, + 5.953002452850342, + 5.9080657958984375, + 5.825891971588135, + 5.804837703704834, + 5.5461649894714355, + 5.708247184753418, + 5.6143975257873535, + 5.451046466827393, + 5.477572917938232, + 5.245761394500732, + 5.33964204788208, + 5.390132904052734, + 5.322740077972412, + 5.402585506439209, + 5.222692489624023, + 5.34494161605835, + 5.135756015777588, + 5.135149955749512, + 5.188284873962402, + 5.062577247619629, + 5.136959552764893, + 4.98410177230835, + 5.081079006195068, + 5.014165878295898, + 5.0218048095703125, + 4.9213056564331055, + 5.100401878356934, + 4.993360996246338, + 4.864107131958008, + 4.8052287101745605, + 5.0377607345581055, + 4.953628063201904, + 4.9419684410095215, + 4.807288646697998, + 4.836999416351318, + 4.984342098236084, + 4.884766578674316, + 4.83286190032959, + 4.925181865692139, + 4.823574066162109, + 4.792526721954346, + 4.78813362121582, + 4.804581165313721, + 4.641773700714111, + 4.763977527618408, + 4.754238605499268, + 4.767972469329834, + 4.82776403427124, + 4.662824630737305, + 4.7278523445129395, + 4.670964241027832, + 4.696649074554443, + 4.700046062469482, + 4.682258129119873, + 4.6354570388793945, + 4.6488542556762695, + 4.59798526763916, + 4.696416854858398, + 4.536737442016602, + 4.52003812789917, + 4.726071357727051, + 4.690214157104492, + 4.5114545822143555, + 4.691585063934326, + 4.62167501449585, + 4.529074668884277, + 4.614591598510742, + 4.5201616287231445, + 4.474466800689697, + 4.574146747589111, + 4.465106010437012, + 4.494256019592285, + 4.518946647644043, + 4.559754371643066, + 4.416994571685791, + 4.467711448669434, + 4.521196365356445, + 4.505422592163086, + 4.488849639892578, + 4.352687358856201, + 4.432631969451904, + 4.507841110229492, + 4.4379096031188965, + 4.52241849899292, + 4.405303001403809, + 4.531144618988037, + 4.454674243927002, + 4.373249053955078, + 4.331098556518555, + 4.5830912590026855, + 4.493315696716309, + 4.279618740081787, + 4.337103366851807, + 4.243735313415527, + 4.387043476104736, + 4.374361515045166, + 4.293359756469727, + 4.331480503082275, + 4.325092315673828, + 4.361403465270996, + 4.466622829437256, + 4.343751907348633, + 4.433427810668945, + 4.312266826629639, + 4.292255878448486, + 4.312690734863281, + 4.433143615722656, + 4.369068145751953, + 4.267335414886475, + 4.349096298217773, + 4.327047824859619, + 4.252540111541748, + 4.3219685554504395, + 4.2303242683410645, + 4.178328037261963, + 4.18655252456665, + 4.08413028717041, + 4.275152683258057, + 4.214677333831787, + 4.118525981903076, + 4.202068328857422, + 4.314276695251465, + 4.249807357788086, + 4.20706844329834, + 4.111940860748291, + 4.206997394561768, + 4.141233921051025, + 4.320606708526611, + 4.097645282745361, + 4.225974082946777, + 4.213339328765869, + 4.214665412902832, + 4.118075847625732, + 4.063041687011719, + 4.15817403793335, + 4.19727087020874, + 4.23914098739624, + 4.186395168304443, + 4.197947025299072, + 4.209135055541992, + 4.197885990142822, + 4.162403106689453, + 4.106482982635498, + 4.074570655822754, + 4.1727142333984375, + 4.168442726135254, + 4.150406360626221, + 4.125866413116455, + 4.210746765136719, + 4.089227676391602, + 4.082394599914551, + 4.143898963928223, + 4.051882266998291, + 4.241250991821289, + 4.149052143096924, + 3.9872241020202637, + 4.085210800170898, + 4.077707290649414, + 4.140270233154297, + 3.968468189239502, + 4.050779342651367, + 4.079315662384033, + 4.1276140213012695, + 4.184043884277344, + 4.193765163421631, + 4.164812088012695, + 3.9141247272491455, + 4.166587829589844, + 4.0810933113098145, + 4.056585788726807, + 4.161518573760986, + 4.093563079833984, + 4.093836784362793, + 4.032834529876709, + 4.045803070068359, + 3.959988832473755, + 3.9896602630615234, + 4.0387396812438965, + 3.977311611175537, + 4.22334623336792, + 4.120009422302246, + 4.172422409057617, + 4.21269416809082, + 3.9776766300201416, + 4.057858943939209, + 4.173552989959717, + 3.9726407527923584, + 4.0963358879089355, + 4.046297073364258, + 3.9774954319000244, + 3.910416603088379, + 4.119848728179932, + 4.096826076507568, + 4.068907737731934, + 3.9425976276397705, + 4.057265281677246, + 4.135598659515381, + 3.9840784072875977, + 4.064267635345459, + 3.9576637744903564, + 4.044075012207031, + 4.047996997833252, + 3.848689556121826, + 3.9532344341278076, + 4.025957107543945, + 4.024180889129639, + 3.953726053237915, + 4.032247543334961, + 3.9871249198913574, + 4.058465003967285, + 3.986022710800171, + 4.112728595733643, + 3.9533324241638184, + 4.021052837371826, + 4.022307395935059, + 4.063476085662842, + 3.887284517288208, + 4.0855512619018555, + 3.8616766929626465, + 3.9155826568603516, + 3.946333885192871, + 4.037856578826904, + 3.9363224506378174, + 3.974374771118164, + 3.975463390350342, + 3.9650917053222656, + 3.9789302349090576, + 3.8323166370391846, + 4.032078742980957, + 3.9237406253814697, + 3.9180667400360107, + 3.909322738647461, + 3.9884753227233887, + 3.885692596435547, + 3.934894323348999, + 3.8813209533691406, + 3.78767728805542, + 3.819061040878296, + 3.773625612258911, + 3.8807008266448975, + 3.9539389610290527, + 3.7766430377960205, + 3.90724778175354, + 4.026951313018799, + 3.8629610538482666, + 3.9585959911346436, + 3.8103652000427246, + 4.003259181976318, + 3.898110866546631, + 4.001630783081055, + 3.9403023719787598, + 3.949864149093628, + 3.8021082878112793, + 3.931750774383545, + 3.8249456882476807, + 3.983501434326172, + 3.8031692504882812, + 3.9358129501342773, + 3.9149982929229736, + 3.9625492095947266, + 3.919795274734497, + 3.906244993209839, + 3.999760866165161, + 3.8245153427124023, + 3.901249647140503, + 3.9597766399383545, + 3.8663501739501953, + 3.9827866554260254, + 3.898930788040161, + 3.9483206272125244, + 3.8270583152770996, + 3.9352002143859863, + 3.804321050643921, + 3.878936290740967, + 3.8733577728271484, + 3.903331995010376, + 3.9174318313598633, + 3.9641618728637695, + 3.8803906440734863, + 3.8322556018829346, + 3.7962863445281982, + 3.942307233810425, + 3.9331510066986084, + 3.8817427158355713, + 3.8529505729675293, + 3.9710865020751953, + 3.895474672317505, + 3.8711228370666504, + 3.9327242374420166, + 3.8018438816070557, + 4.027503967285156, + 3.781465530395508, + 3.8100829124450684, + 3.6529786586761475, + 3.855236768722534, + 3.969817876815796, + 3.895261526107788, + 3.924872636795044, + 3.9949889183044434, + 3.847860097885132, + 3.7495553493499756, + 3.949002265930176, + 3.7530033588409424, + 3.841320037841797, + 3.8680403232574463, + 3.662567138671875, + 3.867743968963623, + 3.829840660095215, + 3.9657363891601562, + 3.716596841812134, + 3.8271093368530273, + 3.7685015201568604, + 3.9424731731414795, + 3.85929536819458, + 3.8033456802368164, + 3.8630130290985107, + 3.904083728790283, + 3.7579879760742188, + 3.833858013153076, + 3.9004619121551514, + 3.7212700843811035, + 3.795027732849121, + 3.8070154190063477, + 3.7317090034484863, + 3.8475751876831055, + 3.8063559532165527, + 3.9507908821105957, + 3.7618651390075684, + 3.925356864929199, + 3.884723663330078, + 4.003852367401123, + 3.956202983856201, + 3.8647329807281494, + 3.8771276473999023, + 3.754173994064331, + 3.8102176189422607, + 3.882951021194458, + 3.8070762157440186, + 3.948913812637329, + 3.797231435775757, + 3.876478433609009, + 3.8475327491760254, + 3.715294122695923, + 3.7886815071105957, + 3.8053336143493652, + 3.820730447769165, + 3.8364646434783936, + 3.884808301925659, + 3.889751434326172, + 3.7406277656555176, + 3.857381820678711, + 3.720916271209717, + 3.9038002490997314, + 3.8649260997772217, + 3.7325000762939453, + 3.7636802196502686, + 3.839998960494995, + 3.8115601539611816, + 3.859764814376831, + 3.916623115539551, + 3.869926691055298, + 3.8014490604400635, + 3.677706480026245, + 3.6241748332977295, + 3.5742650032043457, + 3.7538235187530518, + 3.6525392532348633, + 3.7154994010925293, + 3.763652801513672, + 3.614717721939087, + 3.714599847793579, + 3.772329092025757, + 3.7126517295837402, + 3.6639044284820557, + 3.835130214691162, + 3.6820263862609863, + 3.76035737991333, + 3.8169069290161133, + 3.6768956184387207, + 3.6971492767333984, + 3.698488235473633, + 3.7042624950408936, + 3.6379072666168213, + 3.751884937286377, + 3.7528703212738037, + 3.624081611633301, + 3.715176820755005, + 3.6668734550476074, + 3.6858034133911133, + 3.8692028522491455, + 3.7712631225585938, + 3.7363009452819824, + 3.6987264156341553, + 3.583923816680908, + 3.7281854152679443, + 3.672445774078369, + 3.7298035621643066, + 3.713669776916504, + 3.662767171859741, + 3.8043243885040283, + 3.840134859085083, + 3.765092611312866, + 3.6528632640838623, + 3.670969247817993, + 3.71626353263855, + 3.8062658309936523, + 3.6688072681427, + 3.778249502182007, + 3.7710909843444824, + 3.5356972217559814, + 3.660931348800659, + 3.629612445831299, + 3.71353816986084, + 3.7228033542633057, + 3.6944944858551025, + 3.64823842048645, + 3.660468816757202, + 3.7832348346710205, + 3.780136823654175, + 3.7310564517974854, + 3.7438085079193115, + 3.6331899166107178, + 3.7341737747192383, + 3.7153820991516113, + 3.7120208740234375, + 3.6382856369018555, + 3.5759243965148926, + 3.7362115383148193, + 3.638465404510498, + 3.7134299278259277, + 3.529510736465454, + 3.537646532058716, + 3.690544366836548, + 3.6685357093811035, + 3.7804620265960693, + 3.757988452911377, + 3.6548962593078613, + 3.761908531188965, + 3.7054476737976074, + 3.775580406188965, + 3.6198856830596924, + 3.477898120880127, + 3.752122163772583, + 3.568312644958496, + 3.7555594444274902, + 3.715057134628296, + 3.7671821117401123, + 3.5808169841766357, + 3.6166632175445557, + 3.5505173206329346, + 3.720191478729248, + 3.825197696685791, + 3.744825839996338, + 3.676206350326538, + 3.631563186645508, + 3.6180737018585205, + 3.7513740062713623, + 3.7623507976531982, + 3.61519193649292, + 3.632408380508423, + 3.675567150115967, + 3.5935425758361816, + 3.6195571422576904, + 3.6485512256622314, + 3.652447462081909, + 3.60626220703125, + 3.580857515335083, + 3.6982221603393555, + 3.6308157444000244, + 3.7496583461761475, + 3.6897287368774414, + 3.6767566204071045, + 3.691263437271118, + 3.6230623722076416, + 3.758948564529419, + 3.5935161113739014, + 3.483452081680298, + 3.7751784324645996, + 3.708885669708252, + 3.466789722442627, + 3.727250576019287, + 3.728604316711426, + 3.7629013061523438, + 3.7216455936431885, + 3.6650617122650146, + 3.6736106872558594, + 3.689333915710449, + 3.730245590209961, + 3.5203747749328613, + 3.5200283527374268, + 3.6146960258483887, + 3.6187429428100586, + 3.477532386779785, + 3.627856731414795, + 3.49360990524292, + 3.6365089416503906, + 3.524390935897827, + 3.590394973754883, + 3.5701658725738525, + 3.4651005268096924, + 3.6349453926086426, + 3.5638182163238525, + 3.647130250930786, + 3.66264271736145, + 3.636139392852783, + 3.5688326358795166, + 3.6799771785736084, + 3.735466241836548, + 3.561750888824463, + 3.687464714050293, + 3.696037530899048, + 3.6460909843444824, + 3.6190500259399414, + 3.4805660247802734, + 3.5635769367218018, + 3.692369222640991, + 3.5281167030334473, + 3.6946277618408203, + 3.4892983436584473, + 3.5934841632843018, + 3.632179021835327, + 3.55735445022583, + 3.748136043548584, + 3.5974340438842773, + 3.511389970779419, + 3.6275479793548584, + 3.5104291439056396, + 3.5688228607177734, + 3.4640004634857178, + 3.5779597759246826, + 3.723177194595337, + 3.671156883239746, + 3.6191470623016357, + 3.5526678562164307, + 3.513627052307129, + 3.6196248531341553, + 3.6912031173706055, + 3.5997378826141357, + 3.6831817626953125, + 3.6282455921173096, + 3.553309679031372, + 3.615480422973633, + 3.603487730026245, + 3.528256416320801, + 3.5972955226898193, + 3.685828447341919, + 3.603729248046875, + 3.4699888229370117, + 3.588155508041382, + 3.6298272609710693, + 3.654127597808838, + 3.5698447227478027, + 3.594585418701172, + 3.5850675106048584, + 3.6324708461761475, + 3.6093759536743164, + 3.632727861404419, + 3.5542068481445312, + 3.6634795665740967, + 3.5932817459106445, + 3.6235191822052, + 3.4440560340881348, + 3.560358762741089, + 3.622589588165283, + 3.6020777225494385, + 3.5589659214019775, + 3.528092384338379, + 3.5163657665252686, + 3.691481828689575, + 3.5411899089813232, + 3.65338134765625, + 3.602961540222168, + 3.544487237930298, + 3.521399974822998, + 3.3843815326690674, + 3.5171849727630615, + 3.51175594329834, + 3.554253578186035, + 3.550273895263672, + 3.5966176986694336, + 3.432861328125, + 3.539142608642578, + 3.665687084197998, + 3.563035726547241, + 3.5877718925476074, + 3.5019330978393555, + 3.4523072242736816, + 3.592162609100342, + 3.428629159927368, + 3.4796667098999023, + 3.4596664905548096, + 3.530343770980835, + 3.504119634628296, + 3.63041090965271, + 3.4320571422576904, + 3.502473831176758, + 3.398108720779419, + 3.627734661102295, + 3.549806594848633, + 3.5629994869232178, + 3.5062310695648193, + 3.551164150238037, + 3.656049966812134, + 3.5744142532348633, + 3.4338316917419434, + 3.464146614074707, + 3.656313896179199, + 3.6456079483032227, + 3.4254982471466064, + 3.4771764278411865, + 3.607781410217285, + 3.581303358078003, + 3.420257568359375, + 3.540249824523926, + 3.357583999633789, + 3.5155439376831055, + 3.552316904067993, + 3.4401962757110596, + 3.5867581367492676, + 3.4521217346191406, + 3.550691843032837, + 3.3664183616638184, + 3.5488507747650146, + 3.3898699283599854, + 3.5390756130218506, + 3.5028727054595947, + 3.3597238063812256, + 3.612229824066162, + 3.4303667545318604, + 3.3806982040405273, + 3.4922447204589844, + 3.4876701831817627, + 3.3512885570526123, + 3.5521411895751953, + 3.4438858032226562, + 3.2747769355773926, + 3.505765199661255, + 3.4856150150299072, + 3.472320318222046, + 3.473553419113159, + 3.538447856903076, + 3.5307116508483887, + 3.5647056102752686, + 3.4753623008728027, + 3.55275821685791, + 3.3631410598754883, + 3.614987373352051, + 3.612790822982788, + 3.4524738788604736, + 3.568488836288452, + 3.464946985244751, + 3.4196088314056396, + 3.4847793579101562, + 3.448075771331787, + 3.3421378135681152, + 3.4502525329589844, + 3.532456874847412, + 3.5334436893463135, + 3.5647659301757812, + 3.4671547412872314, + 3.447807788848877, + 3.5161352157592773, + 3.5453720092773438, + 3.4710731506347656, + 3.4152252674102783, + 3.460881471633911, + 3.3946664333343506, + 3.402862787246704, + 3.4054789543151855, + 3.4219706058502197, + 3.5201292037963867, + 3.414017915725708, + 3.555208921432495, + 3.3629770278930664, + 3.489692211151123, + 3.6271512508392334, + 3.523444414138794, + 3.516538381576538, + 3.5275394916534424, + 3.5317628383636475, + 3.3647828102111816, + 3.436704158782959, + 3.4220235347747803, + 3.3956849575042725, + 3.4743154048919678, + 3.511126756668091, + 3.4529480934143066, + 3.475951910018921, + 3.5864975452423096, + 3.5598363876342773, + 3.4617342948913574, + 3.5719997882843018, + 3.488506317138672, + 3.437821626663208, + 3.3555908203125, + 3.4014406204223633, + 3.504068374633789, + 3.5282557010650635, + 3.371483325958252, + 3.544935941696167, + 3.3524694442749023, + 3.5006263256073, + 3.5089290142059326, + 3.513275146484375, + 3.4610393047332764, + 3.5223615169525146, + 3.4341893196105957, + 3.4990198612213135, + 3.4794888496398926, + 3.526216983795166, + 3.4982755184173584, + 3.5001938343048096, + 3.516399621963501, + 3.3736424446105957, + 3.4789774417877197, + 3.431527614593506, + 3.436447858810425, + 3.4153661727905273, + 3.4318463802337646, + 3.430678367614746, + 3.462094783782959, + 3.476243257522583, + 3.438746690750122, + 3.4147074222564697, + 3.4581494331359863, + 3.472555160522461, + 3.4546384811401367, + 3.481282949447632, + 3.4826767444610596, + 3.441028356552124, + 3.4931912422180176, + 3.369542121887207, + 3.5137076377868652, + 3.523738145828247, + 3.4156816005706787, + 3.501858949661255, + 3.2968978881835938, + 3.374926805496216, + 3.5400965213775635, + 3.3699541091918945, + 3.3910763263702393, + 3.3640801906585693, + 3.371047258377075, + 3.3883962631225586, + 3.393609046936035, + 3.357919931411743, + 3.3541038036346436, + 3.330650568008423, + 3.363178253173828, + 3.4314537048339844, + 3.3494138717651367, + 3.3428428173065186, + 3.4109067916870117, + 3.472160577774048, + 3.3877170085906982, + 3.463533878326416, + 3.386202812194824, + 3.333367109298706, + 3.4945645332336426, + 3.199932813644409, + 3.3444466590881348, + 3.4986276626586914, + 3.432421922683716, + 3.341514825820923, + 3.409656524658203, + 3.3990564346313477, + 3.4929256439208984, + 3.4676241874694824, + 3.5416338443756104, + 3.370635986328125, + 3.345367431640625, + 3.4818668365478516, + 3.432940721511841, + 3.4249956607818604, + 3.493441104888916, + 3.3851819038391113, + 3.280090570449829, + 3.3835856914520264, + 3.33709716796875, + 3.318682909011841, + 3.3861215114593506, + 3.4406681060791016, + 3.534621000289917, + 3.3080883026123047, + 3.4006195068359375, + 3.4515082836151123, + 3.3871729373931885, + 3.509993553161621, + 3.434499502182007, + 3.3711800575256348, + 3.383702516555786, + 3.41304087638855, + 3.3301148414611816, + 3.3111722469329834, + 3.3484115600585938, + 3.379020929336548, + 3.4485721588134766, + 3.4327292442321777, + 3.5148472785949707, + 3.5215446949005127, + 3.453892469406128, + 3.4436795711517334, + 3.358757972717285, + 3.3977859020233154, + 3.398094415664673, + 3.4252235889434814, + 3.3299994468688965, + 3.406874418258667, + 3.369323492050171, + 3.354947090148926, + 3.4563379287719727, + 3.430443048477173, + 3.4586310386657715, + 3.403423309326172, + 3.438432455062866, + 3.1918365955352783, + 3.452678918838501, + 3.413001537322998, + 3.4348535537719727, + 3.4415860176086426, + 3.3702046871185303, + 3.3848838806152344, + 3.3497419357299805, + 3.436375141143799, + 3.3536181449890137, + 3.4161932468414307, + 3.3614840507507324, + 3.4325942993164062, + 3.4270174503326416, + 3.4610438346862793, + 3.337545871734619, + 3.369971752166748, + 3.5271692276000977, + 3.399902820587158, + 3.496513605117798, + 3.4768877029418945, + 3.469081401824951, + 3.3193607330322266, + 3.506030559539795, + 3.6055908203125, + 3.4497575759887695, + 3.3310940265655518, + 3.3318936824798584, + 3.4822049140930176, + 3.386073112487793, + 3.5603952407836914, + 3.4884772300720215, + 3.399548292160034, + 3.327817440032959, + 3.3807904720306396, + 3.2971532344818115, + 3.3620729446411133, + 3.3701882362365723, + 3.346700668334961, + 3.2994229793548584, + 3.3582565784454346, + 3.4709889888763428, + 3.476186513900757, + 3.5259408950805664, + 3.3385066986083984, + 3.436171531677246, + 3.536195993423462, + 3.299680471420288, + 3.403820276260376, + 3.4625144004821777, + 3.3619544506073, + 3.43912410736084, + 3.352318525314331, + 3.3830924034118652, + 3.3658323287963867, + 3.3593385219573975, + 3.3753981590270996, + 3.2635765075683594, + 3.219606399536133, + 3.3191542625427246, + 3.3667759895324707, + 3.206491708755493, + 3.3443474769592285, + 3.3749051094055176, + 3.223466157913208, + 3.3539435863494873, + 3.295799970626831, + 3.4054770469665527, + 3.39593768119812, + 3.4546079635620117, + 3.4374005794525146, + 3.4104137420654297, + 3.274334669113159, + 3.390509605407715, + 3.3681724071502686, + 3.410604476928711, + 3.3942158222198486, + 3.325042724609375, + 3.3611061573028564, + 3.3539962768554688, + 3.4082517623901367, + 3.3199260234832764, + 3.1072659492492676, + 3.301704168319702, + 3.302396297454834, + 3.355186939239502, + 3.4088528156280518, + 3.37951922416687, + 3.268432140350342, + 3.3402886390686035, + 3.3168747425079346, + 3.336074113845825, + 3.383915662765503, + 3.392387628555298, + 3.2360754013061523, + 3.4066579341888428, + 3.4564125537872314, + 3.327465295791626, + 3.3861682415008545, + 3.289271593093872, + 3.407858371734619, + 3.37960147857666, + 3.365441083908081, + 3.280608892440796, + 3.3984689712524414, + 3.2549095153808594, + 3.3179070949554443, + 3.15162992477417, + 3.3537917137145996, + 3.2917778491973877, + 3.331005573272705, + 3.3545281887054443, + 3.3139829635620117, + 3.3541412353515625, + 3.1933634281158447, + 3.2323105335235596, + 3.4735913276672363, + 3.2095282077789307, + 3.3473641872406006, + 3.3518309593200684, + 3.330308437347412, + 3.2019588947296143, + 3.1604111194610596, + 3.253016233444214, + 3.41271710395813, + 3.4066882133483887, + 3.4290831089019775, + 3.3508365154266357, + 3.2649378776550293, + 3.2045536041259766, + 3.3223509788513184, + 3.3143908977508545, + 3.2654521465301514, + 3.204941749572754, + 3.530871629714966, + 3.361673355102539, + 3.332707166671753, + 3.3842482566833496, + 3.278385877609253, + 3.3373782634735107, + 3.2251527309417725, + 3.30106258392334, + 3.3249282836914062, + 3.5201635360717773, + 3.336775779724121, + 3.3073086738586426, + 3.2735776901245117, + 3.432265281677246, + 3.3553433418273926, + 3.4792377948760986, + 3.473254680633545, + 3.4274890422821045, + 3.4499635696411133, + 3.279576539993286, + 3.319110155105591, + 3.242549180984497, + 3.427581787109375, + 3.3629226684570312, + 3.368922233581543, + 3.2743330001831055, + 3.3026771545410156, + 3.3153672218322754, + 3.3498144149780273, + 3.3865108489990234, + 3.3497238159179688, + 3.358499050140381, + 3.478928327560425 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 120.99618676373063, + 74.58144069625821, + 59.44965012096789, + 51.749321458236615, + 47.714042225093166, + 43.940023694801965, + 41.48289698823005, + 39.7680691444008, + 37.96050348148646, + 36.50914235731277, + 35.478404541873964, + 34.376619213052614, + 33.50015111264249, + 32.69643153181352, + 31.987754745246523, + 31.61802807823494, + 31.213043612938627, + 31.04229213984429, + 30.958558675145422, + 30.925849243672374 + ], + "best_ppl": 30.925849243672374, + "train_time_s": 20268.600777626038 +} \ No newline at end of file diff --git a/061_dense_mol/dense_k5_top3_s256_best.pt b/061_dense_mol/dense_k5_top3_s256_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0ea2cdae2b4c07a189bd6ceef06dbac7262f034 --- /dev/null +++ b/061_dense_mol/dense_k5_top3_s256_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e892f2d9a6e9d8b44837d4ce1f676bb7577d3adb96252dd7615aab1292f713f1 +size 341441241 diff --git a/061_dense_mol/dense_k5_top3_s256_results.json b/061_dense_mol/dense_k5_top3_s256_results.json new file mode 100644 index 0000000000000000000000000000000000000000..7ff5ef9748cfeff3fffdfbaada63b1985d59fdf7 --- /dev/null +++ b/061_dense_mol/dense_k5_top3_s256_results.json @@ -0,0 +1,2059 @@ +{ + "label": "dense_k5_top3_s256", + "top_k_blocks": 3, + "num_thin_blocks": 5, + "total_params": 85301248, + "router_params": 40960, + "dense_ffn": true, + "d_ff_thin": 1024, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 18.97342872619629, + 14.906354904174805, + 8.021178245544434, + 7.283962249755859, + 7.161350727081299, + 7.021988868713379, + 6.906792640686035, + 6.753620147705078, + 6.697153091430664, + 6.64033842086792, + 6.594293117523193, + 6.536739349365234, + 6.504807472229004, + 6.313175678253174, + 6.157115459442139, + 6.071370601654053, + 5.970909118652344, + 5.9188151359558105, + 5.930933475494385, + 5.7496256828308105, + 5.757286548614502, + 5.756556987762451, + 5.768063545227051, + 5.6431355476379395, + 5.591801643371582, + 5.560659885406494, + 5.456416130065918, + 5.414168834686279, + 5.480034828186035, + 5.315408229827881, + 5.288273811340332, + 5.258056163787842, + 5.207484245300293, + 5.325143337249756, + 5.34085750579834, + 5.164910793304443, + 5.158397674560547, + 5.056722640991211, + 5.100512504577637, + 5.131002426147461, + 5.007577419281006, + 5.101255893707275, + 4.976328372955322, + 5.084232330322266, + 4.923256874084473, + 4.994924545288086, + 5.011125564575195, + 4.943521499633789, + 4.9100470542907715, + 4.90866231918335, + 4.915041446685791, + 4.97756814956665, + 4.817824840545654, + 4.834584712982178, + 4.664411544799805, + 4.759128570556641, + 4.825785160064697, + 4.7758378982543945, + 4.824936866760254, + 4.802114486694336, + 4.727035999298096, + 4.7559051513671875, + 4.692360877990723, + 4.689220428466797, + 4.757246017456055, + 4.623000144958496, + 4.612479209899902, + 4.866946220397949, + 4.772555351257324, + 4.6069793701171875, + 4.752096176147461, + 4.738618850708008, + 4.610152244567871, + 4.706366539001465, + 4.703904151916504, + 4.6547160148620605, + 4.73092794418335, + 4.604288101196289, + 4.65355920791626, + 4.620968341827393, + 4.503274917602539, + 4.675694942474365, + 4.636703968048096, + 4.526118278503418, + 4.678907871246338, + 4.558104991912842, + 4.520620822906494, + 4.631813049316406, + 4.487152576446533, + 4.472474098205566, + 4.474822998046875, + 4.494061470031738, + 4.474262714385986, + 4.511330604553223, + 4.4741973876953125, + 4.38565731048584, + 4.58143424987793, + 4.417166233062744, + 4.550436019897461, + 4.486642360687256, + 4.456887722015381, + 4.488248348236084, + 4.540942668914795, + 4.517482280731201, + 4.539632320404053, + 4.210821628570557, + 4.3636016845703125, + 4.302844524383545, + 4.3580427169799805, + 4.245254993438721, + 4.384115219116211, + 4.355147361755371, + 4.372706890106201, + 4.456300258636475, + 4.507813930511475, + 4.285449981689453, + 4.387470245361328, + 4.459213733673096, + 4.392759799957275, + 4.46234655380249, + 4.310135841369629, + 4.41911506652832, + 4.318014621734619, + 4.253571510314941, + 4.352919578552246, + 4.255926132202148, + 4.244750499725342, + 4.345240116119385, + 4.245192050933838, + 4.268895626068115, + 4.289828777313232, + 4.118363380432129, + 4.290746688842773, + 4.22635555267334, + 4.16235876083374, + 4.230666160583496, + 4.203542709350586, + 4.21410608291626, + 4.277383804321289, + 4.262073993682861, + 4.2301836013793945, + 4.24317741394043, + 4.251890659332275, + 4.16795539855957, + 4.287208557128906, + 4.372817039489746, + 4.1511735916137695, + 4.245898246765137, + 4.235012531280518, + 4.260419845581055, + 4.100149154663086, + 4.206762313842773, + 4.1119303703308105, + 4.0456647872924805, + 4.162528038024902, + 4.218498229980469, + 4.121614933013916, + 4.105728626251221, + 4.206512451171875, + 4.1125264167785645, + 4.023443698883057, + 4.240143775939941, + 4.087972164154053, + 4.146010398864746, + 4.1753058433532715, + 4.199799060821533, + 4.172563552856445, + 4.187655925750732, + 4.217329025268555, + 4.105491638183594, + 4.218637466430664, + 4.242140293121338, + 4.2114996910095215, + 3.990412473678589, + 4.256192684173584, + 4.166346549987793, + 4.215542793273926, + 4.180777549743652, + 4.048194408416748, + 4.188710689544678, + 4.181105136871338, + 4.13823127746582, + 4.052390098571777, + 4.058709144592285, + 4.190768241882324, + 3.9742183685302734, + 4.209475994110107, + 4.151422023773193, + 4.099226474761963, + 4.093959808349609, + 4.0124311447143555, + 4.134918212890625, + 3.963203191757202, + 4.092091083526611, + 4.050511360168457, + 4.020603179931641, + 4.139001846313477, + 4.041295051574707, + 4.079819202423096, + 4.154494762420654, + 4.124274253845215, + 4.056436061859131, + 4.028772830963135, + 4.06688117980957, + 3.967022180557251, + 3.945383071899414, + 3.9106669425964355, + 4.024994373321533, + 4.226110935211182, + 3.9259746074676514, + 3.9430954456329346, + 3.9544625282287598, + 4.090631484985352, + 4.0265655517578125, + 3.9688825607299805, + 4.068981170654297, + 4.0916619300842285, + 4.0005316734313965, + 4.1610493659973145, + 4.053474426269531, + 3.967163562774658, + 4.027307510375977, + 4.035909175872803, + 4.156297206878662, + 4.1405839920043945, + 4.042011260986328, + 4.058520793914795, + 3.983100652694702, + 4.171107769012451, + 4.003477573394775, + 4.134170055389404, + 4.1825151443481445, + 3.925405740737915, + 4.072460174560547, + 4.072793960571289, + 4.056964874267578, + 4.086078643798828, + 3.995260000228882, + 4.05032205581665, + 3.993168592453003, + 3.9539735317230225, + 4.03278923034668, + 4.069851398468018, + 3.9343063831329346, + 4.000266075134277, + 3.984822988510132, + 3.9219040870666504, + 3.939192771911621, + 3.993825674057007, + 4.090471267700195, + 3.9899144172668457, + 4.038898944854736, + 4.012937545776367, + 3.9554226398468018, + 3.9546782970428467, + 3.878859519958496, + 3.8329694271087646, + 3.8853299617767334, + 3.8096165657043457, + 3.812455415725708, + 3.863337755203247, + 3.8654866218566895, + 3.8890624046325684, + 3.885385751724243, + 3.828176975250244, + 3.7727084159851074, + 3.882094383239746, + 3.972452402114868, + 3.913536548614502, + 3.8567893505096436, + 3.8396589756011963, + 3.876638889312744, + 4.020305633544922, + 3.946396827697754, + 3.8183650970458984, + 3.9868152141571045, + 3.9080650806427, + 3.7748382091522217, + 3.900794267654419, + 3.9654626846313477, + 3.9065449237823486, + 3.9288785457611084, + 3.817868232727051, + 3.9010369777679443, + 3.880089282989502, + 3.997912645339966, + 3.891964912414551, + 3.9105396270751953, + 3.7845396995544434, + 3.9737296104431152, + 3.9342570304870605, + 3.9328675270080566, + 3.8293943405151367, + 3.840303421020508, + 3.817788600921631, + 4.0038957595825195, + 3.836749792098999, + 3.9944796562194824, + 3.844205617904663, + 3.906104326248169, + 3.796919345855713, + 3.807957887649536, + 3.9040632247924805, + 3.787153482437134, + 3.856163263320923, + 3.878154993057251, + 3.9400250911712646, + 3.7726247310638428, + 3.868696928024292, + 3.8667008876800537, + 3.9095823764801025, + 3.7410502433776855, + 3.9546146392822266, + 3.8416061401367188, + 3.8378262519836426, + 3.8828468322753906, + 3.845909833908081, + 3.8903307914733887, + 3.805943012237549, + 3.8037374019622803, + 3.8203704357147217, + 3.711704730987549, + 3.818567991256714, + 3.854748010635376, + 3.808102607727051, + 3.813236713409424, + 3.8700003623962402, + 3.9563100337982178, + 3.8575587272644043, + 3.882786273956299, + 3.8978991508483887, + 3.7466840744018555, + 3.878119945526123, + 3.748462677001953, + 3.8695478439331055, + 3.7641053199768066, + 3.7467596530914307, + 3.802919864654541, + 3.8106110095977783, + 3.882667064666748, + 3.836325168609619, + 3.9567055702209473, + 3.8460943698883057, + 3.8123505115509033, + 3.8057467937469482, + 3.904660701751709, + 3.8963205814361572, + 3.7705323696136475, + 3.7821598052978516, + 3.816375255584717, + 3.7070460319519043, + 3.8950603008270264, + 3.78415584564209, + 3.917858362197876, + 3.8033642768859863, + 3.721829414367676, + 3.7580175399780273, + 3.8271350860595703, + 3.75276780128479, + 3.832850694656372, + 3.812878131866455, + 3.8181424140930176, + 3.809206962585449, + 3.85355806350708, + 3.786159038543701, + 3.7919514179229736, + 3.8798258304595947, + 3.850287675857544, + 3.8288090229034424, + 3.880199432373047, + 3.7975547313690186, + 3.720780372619629, + 3.8634932041168213, + 3.8731484413146973, + 3.841506242752075, + 3.9260456562042236, + 3.8400838375091553, + 3.692737102508545, + 3.7853469848632812, + 3.6188392639160156, + 3.760925054550171, + 3.813004732131958, + 3.6383397579193115, + 3.6219921112060547, + 3.7042362689971924, + 3.6627111434936523, + 3.5963637828826904, + 3.815328359603882, + 3.724569320678711, + 3.6547369956970215, + 3.74202299118042, + 3.797731637954712, + 3.8021903038024902, + 3.6913740634918213, + 3.7103312015533447, + 3.78507399559021, + 3.6796483993530273, + 3.660715103149414, + 3.633125066757202, + 3.644272804260254, + 3.888939619064331, + 3.6888225078582764, + 3.7873876094818115, + 3.840299367904663, + 3.7536513805389404, + 3.674534559249878, + 3.7286477088928223, + 3.718348264694214, + 3.8212177753448486, + 3.7688963413238525, + 3.7544710636138916, + 3.7438957691192627, + 3.7438361644744873, + 3.750850200653076, + 3.742917060852051, + 3.7166354656219482, + 3.775393486022949, + 3.63493275642395, + 3.740950107574463, + 3.7714099884033203, + 3.7015459537506104, + 3.6236190795898438, + 3.780780076980591, + 3.7206082344055176, + 3.603825807571411, + 3.69590425491333, + 3.8114562034606934, + 3.813959836959839, + 3.7000198364257812, + 3.6565895080566406, + 3.6546812057495117, + 3.745699644088745, + 3.7355053424835205, + 3.7060606479644775, + 3.734388589859009, + 3.7654521465301514, + 3.6171875, + 3.723083734512329, + 3.79430890083313, + 3.696995496749878, + 3.5323121547698975, + 3.7829084396362305, + 3.6813361644744873, + 3.7681195735931396, + 3.7275917530059814, + 3.7006900310516357, + 3.681926727294922, + 3.6913139820098877, + 3.6913700103759766, + 3.6635921001434326, + 3.6510708332061768, + 3.695120096206665, + 3.632181167602539, + 3.7192511558532715, + 3.7239692211151123, + 3.77154541015625, + 3.7095580101013184, + 3.6708970069885254, + 3.654904842376709, + 3.8375251293182373, + 3.589674711227417, + 3.728294849395752, + 3.673138380050659, + 3.6579885482788086, + 3.7557826042175293, + 3.6913747787475586, + 3.8537673950195312, + 3.7211406230926514, + 3.7345728874206543, + 3.8230059146881104, + 3.8488826751708984, + 3.6398332118988037, + 3.790771245956421, + 3.655224561691284, + 3.752291202545166, + 3.6374921798706055, + 3.6651153564453125, + 3.686079740524292, + 3.6556854248046875, + 3.6242177486419678, + 3.653428077697754, + 3.6053946018218994, + 3.7469513416290283, + 3.6674628257751465, + 3.786864757537842, + 3.719073534011841, + 3.7109901905059814, + 3.5983810424804688, + 3.619169235229492, + 3.7031168937683105, + 3.6790573596954346, + 3.6459925174713135, + 3.6616780757904053, + 3.696908712387085, + 3.5373237133026123, + 3.537079095840454, + 3.6291680335998535, + 3.709486484527588, + 3.5653436183929443, + 3.7292308807373047, + 3.5905306339263916, + 3.6555609703063965, + 3.601332426071167, + 3.689383029937744, + 3.6859331130981445, + 3.7360353469848633, + 3.661985158920288, + 3.5655314922332764, + 3.6794464588165283, + 3.5898330211639404, + 3.5618388652801514, + 3.7266385555267334, + 3.4307491779327393, + 3.556494951248169, + 3.607234001159668, + 3.579479694366455, + 3.5311388969421387, + 3.6141703128814697, + 3.593937873840332, + 3.6878597736358643, + 3.5995280742645264, + 3.437532901763916, + 3.63696551322937, + 3.5197622776031494, + 3.49826979637146, + 3.6918227672576904, + 3.6175456047058105, + 3.637164831161499, + 3.614553451538086, + 3.54323148727417, + 3.6242833137512207, + 3.5217669010162354, + 3.5522561073303223, + 3.5698397159576416, + 3.575038194656372, + 3.547822952270508, + 3.77097225189209, + 3.5787301063537598, + 3.4414560794830322, + 3.581000328063965, + 3.647718906402588, + 3.738748073577881, + 3.607433795928955, + 3.5146195888519287, + 3.5575273036956787, + 3.5150835514068604, + 3.5157856941223145, + 3.5621981620788574, + 3.643381118774414, + 3.5278027057647705, + 3.596564531326294, + 3.5919198989868164, + 3.728403091430664, + 3.6577601432800293, + 3.611140012741089, + 3.5961408615112305, + 3.5252270698547363, + 3.6079957485198975, + 3.4966814517974854, + 3.6798884868621826, + 3.6526424884796143, + 3.6063380241394043, + 3.5294058322906494, + 3.5787365436553955, + 3.6122143268585205, + 3.6006596088409424, + 3.4714267253875732, + 3.4112842082977295, + 3.639312505722046, + 3.569498300552368, + 3.5402567386627197, + 3.6430022716522217, + 3.599442720413208, + 3.6066157817840576, + 3.55204176902771, + 3.6865179538726807, + 3.6046102046966553, + 3.4131534099578857, + 3.6461100578308105, + 3.565002679824829, + 3.537738084793091, + 3.6189284324645996, + 3.621561050415039, + 3.596923589706421, + 3.672060966491699, + 3.511233329772949, + 3.655513048171997, + 3.5557749271392822, + 3.5818369388580322, + 3.5630030632019043, + 3.5406010150909424, + 3.6304495334625244, + 3.5834953784942627, + 3.5450658798217773, + 3.5057966709136963, + 3.534135580062866, + 3.726644515991211, + 3.685396432876587, + 3.609863758087158, + 3.4853434562683105, + 3.489691734313965, + 3.5514519214630127, + 3.6630406379699707, + 3.5031161308288574, + 3.6718757152557373, + 3.6023287773132324, + 3.4822351932525635, + 3.6001381874084473, + 3.628809928894043, + 3.6547605991363525, + 3.5687224864959717, + 3.6213433742523193, + 3.5440592765808105, + 3.5773558616638184, + 3.5567944049835205, + 3.6024763584136963, + 3.7523789405822754, + 3.604532241821289, + 3.6069507598876953, + 3.4204275608062744, + 3.6292190551757812, + 3.482473850250244, + 3.5635743141174316, + 3.5718891620635986, + 3.5631179809570312, + 3.5027830600738525, + 3.5557825565338135, + 3.495375633239746, + 3.5626349449157715, + 3.6293234825134277, + 3.662123680114746, + 3.5477583408355713, + 3.5120291709899902, + 3.6222317218780518, + 3.5442514419555664, + 3.414246082305908, + 3.458564519882202, + 3.4377949237823486, + 3.4142813682556152, + 3.6234519481658936, + 3.4742796421051025, + 3.43046236038208, + 3.4960336685180664, + 3.5099844932556152, + 3.3292136192321777, + 3.4863240718841553, + 3.4193732738494873, + 3.4662208557128906, + 3.481680393218994, + 3.4293301105499268, + 3.3365869522094727, + 3.456746816635132, + 3.533008098602295, + 3.4414000511169434, + 3.5908963680267334, + 3.4891796112060547, + 3.4175631999969482, + 3.515869140625, + 3.3756251335144043, + 3.5457708835601807, + 3.5106091499328613, + 3.5175368785858154, + 3.4657692909240723, + 3.5671017169952393, + 3.5196971893310547, + 3.5097131729125977, + 3.5511972904205322, + 3.4632034301757812, + 3.4508273601531982, + 3.4901106357574463, + 3.461156129837036, + 3.518233060836792, + 3.43064022064209, + 3.509673833847046, + 3.5232276916503906, + 3.530492067337036, + 3.4110682010650635, + 3.489199638366699, + 3.438985586166382, + 3.4456183910369873, + 3.46402645111084, + 3.401067018508911, + 3.532806396484375, + 3.5457353591918945, + 3.553189754486084, + 3.5300722122192383, + 3.5616531372070312, + 3.5260589122772217, + 3.6379647254943848, + 3.395735502243042, + 3.5762414932250977, + 3.4579970836639404, + 3.5840554237365723, + 3.4804437160491943, + 3.444591999053955, + 3.47109055519104, + 3.5258638858795166, + 3.4659578800201416, + 3.521700620651245, + 3.4547982215881348, + 3.516343832015991, + 3.506153106689453, + 3.4708855152130127, + 3.4683678150177, + 3.4296655654907227, + 3.379387140274048, + 3.517957925796509, + 3.511913299560547, + 3.4660274982452393, + 3.4690096378326416, + 3.5132088661193848, + 3.454272747039795, + 3.369965076446533, + 3.4228625297546387, + 3.390761137008667, + 3.4927773475646973, + 3.5391008853912354, + 3.418851852416992, + 3.4067764282226562, + 3.3196048736572266, + 3.5013487339019775, + 3.424201726913452, + 3.5844247341156006, + 3.3900742530822754, + 3.573664426803589, + 3.414525270462036, + 3.592045307159424, + 3.5344321727752686, + 3.515165090560913, + 3.5027167797088623, + 3.5280380249023438, + 3.443362236022949, + 3.5558218955993652, + 3.4846317768096924, + 3.438232183456421, + 3.493558645248413, + 3.427781820297241, + 3.3574256896972656, + 3.337533950805664, + 3.529487371444702, + 3.4800946712493896, + 3.5090670585632324, + 3.488981008529663, + 3.5217344760894775, + 3.5097813606262207, + 3.293266534805298, + 3.5985755920410156, + 3.513984203338623, + 3.459879159927368, + 3.544740676879883, + 3.4841034412384033, + 3.4525399208068848, + 3.427121162414551, + 3.4359095096588135, + 3.397655725479126, + 3.4428467750549316, + 3.3668625354766846, + 3.552839994430542, + 3.481499195098877, + 3.3489649295806885, + 3.4211654663085938, + 3.281489372253418, + 3.398724317550659, + 3.4278512001037598, + 3.4096405506134033, + 3.4259190559387207, + 3.4418556690216064, + 3.304570198059082, + 3.453110694885254, + 3.2365338802337646, + 3.4910850524902344, + 3.409641742706299, + 3.3541955947875977, + 3.5031850337982178, + 3.3661553859710693, + 3.404109001159668, + 3.302133560180664, + 3.4754059314727783, + 3.3500587940216064, + 3.297654151916504, + 3.4414424896240234, + 3.258822202682495, + 3.480365514755249, + 3.372448444366455, + 3.434117317199707, + 3.454848051071167, + 3.340458631515503, + 3.441638231277466, + 3.3168671131134033, + 3.345529079437256, + 3.5319576263427734, + 3.457866907119751, + 3.3633410930633545, + 3.4710092544555664, + 3.5293385982513428, + 3.588256359100342, + 3.447096586227417, + 3.3743796348571777, + 3.353262186050415, + 3.4073708057403564, + 3.365626335144043, + 3.450218677520752, + 3.3323590755462646, + 3.456535577774048, + 3.5167248249053955, + 3.307739734649658, + 3.4201138019561768, + 3.491041898727417, + 3.5566813945770264, + 3.433293342590332, + 3.4771931171417236, + 3.426567554473877, + 3.449225425720215, + 3.440605878829956, + 3.4320218563079834, + 3.4796154499053955, + 3.5351362228393555, + 3.4414615631103516, + 3.385200023651123, + 3.5498313903808594, + 3.2914047241210938, + 3.4729604721069336, + 3.4578325748443604, + 3.504171371459961, + 3.5595035552978516, + 3.4869160652160645, + 3.4339568614959717, + 3.4498090744018555, + 3.4088308811187744, + 3.424506664276123, + 3.3580243587493896, + 3.3860652446746826, + 3.4319007396698, + 3.3806538581848145, + 3.370398998260498, + 3.4183547496795654, + 3.3689346313476562, + 3.3086562156677246, + 3.4997200965881348, + 3.4316365718841553, + 3.3880462646484375, + 3.276658058166504, + 3.4187793731689453, + 3.355062961578369, + 3.4389777183532715, + 3.4980297088623047, + 3.4353973865509033, + 3.538741111755371, + 3.5057756900787354, + 3.393702507019043, + 3.313474655151367, + 3.332376480102539, + 3.3636674880981445, + 3.406998872756958, + 3.4789516925811768, + 3.3448023796081543, + 3.421679973602295, + 3.202646017074585, + 3.380483627319336, + 3.460346221923828, + 3.414592981338501, + 3.48744535446167, + 3.3897197246551514, + 3.318709373474121, + 3.299246311187744, + 3.3448429107666016, + 3.3821284770965576, + 3.3757669925689697, + 3.360649347305298, + 3.440220832824707, + 3.332451581954956, + 3.3752524852752686, + 3.4699344635009766, + 3.374547243118286, + 3.3308498859405518, + 3.299992084503174, + 3.3999123573303223, + 3.385620355606079, + 3.311835289001465, + 3.357323169708252, + 3.237474203109741, + 3.381730079650879, + 3.4172203540802, + 3.413357734680176, + 3.390596389770508, + 3.3475708961486816, + 3.2320635318756104, + 3.4450111389160156, + 3.317983627319336, + 3.383714437484741, + 3.4558417797088623, + 3.5646677017211914, + 3.3620784282684326, + 3.458914041519165, + 3.301619529724121, + 3.337897777557373, + 3.4392833709716797, + 3.2432210445404053, + 3.3017122745513916, + 3.2841084003448486, + 3.3678524494171143, + 3.388277292251587, + 3.4445581436157227, + 3.2465364933013916, + 3.3872451782226562, + 3.3132951259613037, + 3.338841199874878, + 3.388659954071045, + 3.3028416633605957, + 3.2305073738098145, + 3.3962152004241943, + 3.2318124771118164, + 3.388434886932373, + 3.374910593032837, + 3.546527862548828, + 3.3082847595214844, + 3.426072120666504, + 3.396456003189087, + 3.3380215167999268, + 3.3984951972961426, + 3.331144094467163, + 3.372429132461548, + 3.358484983444214, + 3.426481246948242, + 3.369544506072998, + 3.3749427795410156, + 3.36073637008667, + 3.366866111755371, + 3.296937942504883, + 3.4278621673583984, + 3.446014165878296, + 3.369342565536499, + 3.399484872817993, + 3.2648234367370605, + 3.307569742202759, + 3.460066556930542, + 3.267448663711548, + 3.3580780029296875, + 3.3810465335845947, + 3.1134963035583496, + 3.31499981880188, + 3.3318240642547607, + 3.269895315170288, + 3.3864099979400635, + 3.4612958431243896, + 3.3297019004821777, + 3.455136775970459, + 3.2165393829345703, + 3.424271821975708, + 3.351928234100342, + 3.4156007766723633, + 3.4057905673980713, + 3.359741687774658, + 3.4513416290283203, + 3.324327230453491, + 3.2607052326202393, + 3.327411413192749, + 3.31528639793396, + 3.36650013923645, + 3.3549535274505615, + 3.3419809341430664, + 3.353332042694092, + 3.4241135120391846, + 3.348114013671875, + 3.465318441390991, + 3.3869223594665527, + 3.47221040725708, + 3.373500347137451, + 3.392470121383667, + 3.2371175289154053, + 3.3500664234161377, + 3.2476460933685303, + 3.362748861312866, + 3.489758253097534, + 3.352191686630249, + 3.3199760913848877, + 3.328768253326416, + 3.379762887954712, + 3.3730971813201904, + 3.1273748874664307, + 3.3280608654022217, + 3.4790539741516113, + 3.226951837539673, + 3.3589885234832764, + 3.4058260917663574, + 3.3458805084228516, + 3.4732794761657715, + 3.320084571838379, + 3.3861396312713623, + 3.3414244651794434, + 3.391439914703369, + 3.275644063949585, + 3.365600109100342, + 3.2433340549468994, + 3.377016067504883, + 3.365216016769409, + 3.389707565307617, + 3.3710694313049316, + 3.3517189025878906, + 3.3125603199005127 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 121.9966808449226, + 75.40799773613496, + 60.13939248603045, + 52.32060787095101, + 47.92930931702772, + 44.72643885301413, + 42.019577657039676, + 39.92549425615109, + 38.081724936789236, + 36.71371102079597, + 35.63842066985369, + 34.46991800063904, + 33.650397844825015, + 32.81426144460416, + 32.10857817352344, + 31.770811816449356, + 31.393475872545245, + 31.195464726691306, + 31.085301443814746, + 31.061574713545696 + ], + "best_ppl": 31.061574713545696, + "train_time_s": 20264.06775522232 +} \ No newline at end of file diff --git a/062_attention_modes/cross_block_best.pt b/062_attention_modes/cross_block_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..11dfbb0f8b8be5f3b62504d2b7cc1ebf5d228a4e --- /dev/null +++ b/062_attention_modes/cross_block_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2693d1c139c1faacb4344c359c1be24b0bb8866a488623c2f2f9a0a753cbb6c5 +size 366703346 diff --git a/062_attention_modes/cross_block_results.json b/062_attention_modes/cross_block_results.json new file mode 100644 index 0000000000000000000000000000000000000000..d673a10ce029fa41350f320cde083bf5cd4c19b2 --- /dev/null +++ b/062_attention_modes/cross_block_results.json @@ -0,0 +1,2060 @@ +{ + "label": "cross_block", + "attention_mode": "cross_block", + "top_k_blocks": 3, + "num_thin_blocks": 5, + "total_params": 91609096, + "router_params": 40960, + "dense_ffn": true, + "d_ff_thin": 1024, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 18.97953987121582, + 14.415542602539062, + 7.891612529754639, + 7.244755744934082, + 7.049878120422363, + 6.95348596572876, + 6.8265485763549805, + 6.789417266845703, + 6.772299289703369, + 6.639427185058594, + 6.4850544929504395, + 6.42399787902832, + 6.340445518493652, + 6.341583251953125, + 6.043397903442383, + 6.089882850646973, + 6.033083438873291, + 5.858682155609131, + 5.86990213394165, + 5.978054046630859, + 5.8074564933776855, + 5.770657539367676, + 5.701529502868652, + 5.655237197875977, + 5.671500205993652, + 5.507955074310303, + 5.459601879119873, + 5.555243492126465, + 5.393081188201904, + 5.3875956535339355, + 5.347723007202148, + 5.408999443054199, + 5.222255229949951, + 5.30177640914917, + 5.394623756408691, + 5.279612064361572, + 5.25151252746582, + 5.215157508850098, + 5.058250904083252, + 5.253174781799316, + 5.196754455566406, + 5.087745666503906, + 5.1571149826049805, + 5.038326740264893, + 5.046059608459473, + 5.100889205932617, + 5.055974006652832, + 4.966579437255859, + 4.974931240081787, + 4.900268077850342, + 4.934849262237549, + 5.062710285186768, + 4.839667320251465, + 4.915714263916016, + 4.7767133712768555, + 4.852398872375488, + 4.938355445861816, + 4.763462543487549, + 4.817878723144531, + 4.847119331359863, + 4.87313175201416, + 4.696292400360107, + 4.809845447540283, + 4.807758331298828, + 4.717763900756836, + 4.6364922523498535, + 4.703780174255371, + 4.711910724639893, + 4.835500240325928, + 4.726639747619629, + 4.600361347198486, + 4.646218299865723, + 4.630770683288574, + 4.598890781402588, + 4.783161640167236, + 4.642529010772705, + 4.590871810913086, + 4.652253150939941, + 4.610143184661865, + 4.7335896492004395, + 4.613868713378906, + 4.603638172149658, + 4.57382869720459, + 4.6108174324035645, + 4.507104396820068, + 4.558809280395508, + 4.573288440704346, + 4.445677757263184, + 4.398543357849121, + 4.671299457550049, + 4.431951522827148, + 4.436777114868164, + 4.477780342102051, + 4.374772548675537, + 4.474393367767334, + 4.554654598236084, + 4.272226810455322, + 4.470791339874268, + 4.441201686859131, + 4.443889141082764, + 4.501237392425537, + 4.517204761505127, + 4.599703311920166, + 4.499124526977539, + 4.396586894989014, + 4.35153865814209, + 4.445723533630371, + 4.423483371734619, + 4.364197731018066, + 4.401976108551025, + 4.402295112609863, + 4.417320728302002, + 4.369556427001953, + 4.384324073791504, + 4.440581798553467, + 4.350742340087891, + 4.405972480773926, + 4.382232189178467, + 4.449082374572754, + 4.252155303955078, + 4.320405006408691, + 4.370519161224365, + 4.298983097076416, + 4.449313640594482, + 4.363638401031494, + 4.343100547790527, + 4.195129871368408, + 4.238239765167236, + 4.162874698638916, + 4.286713600158691, + 4.31205940246582, + 4.302095413208008, + 4.307323932647705, + 4.221314907073975, + 4.252561092376709, + 4.3231000900268555, + 4.3226799964904785, + 4.25563907623291, + 4.146069526672363, + 4.268816947937012, + 4.335666179656982, + 4.110960483551025, + 4.184981346130371, + 4.2379608154296875, + 4.168637275695801, + 4.228084087371826, + 4.139738082885742, + 4.149343967437744, + 4.212456703186035, + 4.180863380432129, + 4.299314498901367, + 4.119079113006592, + 4.225326061248779, + 4.227766990661621, + 4.218835353851318, + 4.15146017074585, + 4.272085189819336, + 4.190891265869141, + 4.090306282043457, + 4.175451755523682, + 4.048940658569336, + 4.103598117828369, + 4.0981645584106445, + 4.188573360443115, + 4.169779300689697, + 4.174205780029297, + 4.272192478179932, + 4.052433490753174, + 4.215298175811768, + 4.182474613189697, + 4.109391212463379, + 4.216856956481934, + 4.206242561340332, + 4.1554460525512695, + 4.070624828338623, + 4.218302249908447, + 4.07592248916626, + 4.1664533615112305, + 3.9777491092681885, + 4.014913558959961, + 4.0385003089904785, + 4.181593418121338, + 4.0338921546936035, + 4.20990514755249, + 4.159890651702881, + 4.0228657722473145, + 4.092063903808594, + 4.146130084991455, + 4.101378917694092, + 3.841965675354004, + 4.158720970153809, + 4.094783306121826, + 4.2005791664123535, + 4.1224684715271, + 4.048914909362793, + 4.054530143737793, + 4.0457444190979, + 4.169707775115967, + 4.094030380249023, + 4.052055358886719, + 4.0281291007995605, + 4.10755729675293, + 4.132375717163086, + 4.02440881729126, + 4.003161430358887, + 3.9623749256134033, + 4.01381254196167, + 4.112834930419922, + 4.1148881912231445, + 4.057910919189453, + 4.065483093261719, + 3.9491257667541504, + 4.047029972076416, + 4.015894412994385, + 4.068253993988037, + 4.048692226409912, + 3.9978418350219727, + 3.9921586513519287, + 3.999002695083618, + 4.006711483001709, + 4.076356887817383, + 4.076888084411621, + 4.096999168395996, + 3.9634363651275635, + 4.1378493309021, + 3.9949417114257812, + 4.056518077850342, + 3.995375156402588, + 4.01043701171875, + 4.14479923248291, + 4.046180248260498, + 4.075727462768555, + 4.094258785247803, + 3.9701356887817383, + 4.006816387176514, + 3.986661195755005, + 4.066793441772461, + 3.9735822677612305, + 4.146185874938965, + 3.97624135017395, + 3.905588150024414, + 3.9171547889709473, + 3.8810622692108154, + 4.004192352294922, + 4.061233997344971, + 4.02665901184082, + 4.079660892486572, + 3.901827335357666, + 4.0589470863342285, + 3.9924328327178955, + 3.9543166160583496, + 3.9100961685180664, + 4.011200904846191, + 3.992323875427246, + 3.7731287479400635, + 4.027574062347412, + 3.871797800064087, + 3.9494309425354004, + 3.889655113220215, + 3.9516727924346924, + 3.8226184844970703, + 3.8441882133483887, + 3.949270486831665, + 3.9529988765716553, + 3.8469889163970947, + 3.8134751319885254, + 3.746743679046631, + 3.880445718765259, + 4.044988632202148, + 3.9860544204711914, + 3.9854238033294678, + 3.968745708465576, + 3.9036507606506348, + 3.8848154544830322, + 3.8460893630981445, + 3.824059247970581, + 3.978410482406616, + 3.9983901977539062, + 3.8403751850128174, + 3.9546327590942383, + 3.862309217453003, + 3.9134390354156494, + 3.889016628265381, + 3.8489246368408203, + 4.0363264083862305, + 4.024743556976318, + 3.857743263244629, + 3.8801262378692627, + 3.823245048522949, + 3.920748710632324, + 3.847721815109253, + 4.024672985076904, + 3.949030876159668, + 3.9047653675079346, + 3.9330878257751465, + 3.888244867324829, + 3.9379029273986816, + 3.9335737228393555, + 3.8195512294769287, + 3.7100327014923096, + 3.7809970378875732, + 3.818242073059082, + 3.968867540359497, + 3.805543899536133, + 3.919008731842041, + 3.8945841789245605, + 3.9036524295806885, + 3.8236372470855713, + 3.8104758262634277, + 3.8309719562530518, + 3.9410135746002197, + 3.770432233810425, + 3.816880941390991, + 3.918835401535034, + 3.905503273010254, + 3.901292324066162, + 3.9291906356811523, + 3.974515914916992, + 3.8669850826263428, + 3.8068225383758545, + 3.959672451019287, + 3.9674971103668213, + 3.921721935272217, + 3.7929067611694336, + 3.894463539123535, + 3.9369750022888184, + 3.860532283782959, + 3.8532397747039795, + 3.805959939956665, + 3.783020257949829, + 3.9914321899414062, + 3.881582021713257, + 3.8967909812927246, + 3.952700138092041, + 3.8215086460113525, + 3.796034097671509, + 3.740638017654419, + 3.8543925285339355, + 3.8058528900146484, + 3.8754124641418457, + 3.7698628902435303, + 3.800095796585083, + 3.825827121734619, + 3.7909810543060303, + 3.85732102394104, + 4.043106555938721, + 3.9210755825042725, + 3.8226938247680664, + 3.7065036296844482, + 3.8413949012756348, + 3.7439844608306885, + 3.9123995304107666, + 3.8342716693878174, + 3.8885140419006348, + 3.840358257293701, + 3.9429373741149902, + 3.9480855464935303, + 3.816795587539673, + 3.8453707695007324, + 3.8438522815704346, + 3.844691038131714, + 3.7052183151245117, + 3.8954827785491943, + 3.8490514755249023, + 3.7620372772216797, + 3.823331594467163, + 3.873725414276123, + 3.8026282787323, + 3.6922264099121094, + 3.883591413497925, + 3.573906660079956, + 3.6742424964904785, + 3.839317560195923, + 3.900510787963867, + 3.833113670349121, + 3.791186571121216, + 3.939908742904663, + 3.8201346397399902, + 3.705726146697998, + 3.8401618003845215, + 3.674959421157837, + 3.6889400482177734, + 3.6190950870513916, + 3.7609341144561768, + 3.8252077102661133, + 3.570425510406494, + 3.699690103530884, + 3.717857837677002, + 3.810920476913452, + 3.753126621246338, + 3.5614686012268066, + 3.790309429168701, + 3.697455883026123, + 3.7969250679016113, + 3.7163617610931396, + 3.6849610805511475, + 3.761503219604492, + 3.858532667160034, + 3.802395820617676, + 3.741316556930542, + 3.706238269805908, + 3.7916922569274902, + 3.793835163116455, + 3.8224034309387207, + 3.656935691833496, + 3.79069185256958, + 3.788999319076538, + 3.7485811710357666, + 3.771549701690674, + 3.704521417617798, + 3.603391647338867, + 3.7757935523986816, + 3.697193145751953, + 3.7493128776550293, + 3.7943413257598877, + 3.711526393890381, + 3.782388687133789, + 3.835585355758667, + 3.726229429244995, + 3.7170069217681885, + 3.768678665161133, + 3.57806134223938, + 3.733670711517334, + 3.804778575897217, + 3.882578134536743, + 3.6782283782958984, + 3.7912659645080566, + 3.602743625640869, + 3.7762093544006348, + 3.7905900478363037, + 3.8861443996429443, + 3.759138345718384, + 3.702763319015503, + 3.6544837951660156, + 3.7616541385650635, + 3.678800344467163, + 3.645181179046631, + 3.799696207046509, + 3.633817434310913, + 3.744128942489624, + 3.6725738048553467, + 3.4999899864196777, + 3.6648213863372803, + 3.6955068111419678, + 3.814737319946289, + 3.793778896331787, + 3.6544711589813232, + 3.736846446990967, + 3.7117581367492676, + 3.7629730701446533, + 3.7527406215667725, + 3.702155828475952, + 3.7459428310394287, + 3.6197991371154785, + 3.7131309509277344, + 3.626319408416748, + 3.705713987350464, + 3.773818254470825, + 3.758894920349121, + 3.7289211750030518, + 3.5597105026245117, + 3.682922840118408, + 3.7370402812957764, + 3.7852561473846436, + 3.472184181213379, + 3.7335574626922607, + 3.738539695739746, + 3.6916275024414062, + 3.6152946949005127, + 3.5499656200408936, + 3.660616397857666, + 3.6543097496032715, + 3.5901761054992676, + 3.7004575729370117, + 3.6665825843811035, + 3.6880295276641846, + 3.610455274581909, + 3.732215404510498, + 3.812965154647827, + 3.5708725452423096, + 3.562051296234131, + 3.7177343368530273, + 3.650723695755005, + 3.680145263671875, + 3.612882375717163, + 3.729548692703247, + 3.65443754196167, + 3.7661805152893066, + 3.662259340286255, + 3.603317975997925, + 3.6816213130950928, + 3.680312156677246, + 3.7292752265930176, + 3.622954845428467, + 3.798468589782715, + 3.7980828285217285, + 3.6926095485687256, + 3.850142002105713, + 3.723363161087036, + 3.6371450424194336, + 3.613717555999756, + 3.7332799434661865, + 3.7325637340545654, + 3.686556816101074, + 3.7544476985931396, + 3.814343214035034, + 3.600764751434326, + 3.5897016525268555, + 3.665776252746582, + 3.640498638153076, + 3.549675226211548, + 3.583552598953247, + 3.5344014167785645, + 3.417475700378418, + 3.4949681758880615, + 3.4764528274536133, + 3.604522705078125, + 3.623267650604248, + 3.6185195446014404, + 3.636017084121704, + 3.613476037979126, + 3.6033427715301514, + 3.644528388977051, + 3.6345126628875732, + 3.610891342163086, + 3.564399003982544, + 3.599785804748535, + 3.6455886363983154, + 3.6448917388916016, + 3.484412670135498, + 3.65073823928833, + 3.633018970489502, + 3.6093406677246094, + 3.5673186779022217, + 3.5789475440979004, + 3.6402359008789062, + 3.603158712387085, + 3.6026623249053955, + 3.5112102031707764, + 3.641098737716675, + 3.491635799407959, + 3.7096190452575684, + 3.6022043228149414, + 3.4866256713867188, + 3.6680610179901123, + 3.7642462253570557, + 3.6872611045837402, + 3.5333189964294434, + 3.635512113571167, + 3.647679567337036, + 3.721795082092285, + 3.5539259910583496, + 3.592435121536255, + 3.6387522220611572, + 3.6225790977478027, + 3.5250895023345947, + 3.458956718444824, + 3.569121837615967, + 3.6241602897644043, + 3.7485649585723877, + 3.666799306869507, + 3.477644205093384, + 3.7420501708984375, + 3.5268635749816895, + 3.622124195098877, + 3.583908796310425, + 3.571672201156616, + 3.5905261039733887, + 3.6296257972717285, + 3.616445779800415, + 3.557891607284546, + 3.525698661804199, + 3.453185796737671, + 3.5362069606781006, + 3.587972640991211, + 3.6426031589508057, + 3.634368658065796, + 3.6179864406585693, + 3.481703996658325, + 3.560930013656616, + 3.480654001235962, + 3.5695292949676514, + 3.572441816329956, + 3.5235109329223633, + 3.5792369842529297, + 3.6153383255004883, + 3.6929779052734375, + 3.6262943744659424, + 3.6427347660064697, + 3.5116984844207764, + 3.5579001903533936, + 3.6958353519439697, + 3.593843460083008, + 3.566854238510132, + 3.6196951866149902, + 3.541171073913574, + 3.6496896743774414, + 3.584813117980957, + 3.6531052589416504, + 3.5461277961730957, + 3.5884931087493896, + 3.721653699874878, + 3.632166624069214, + 3.649141311645508, + 3.5801451206207275, + 3.670193910598755, + 3.5274081230163574, + 3.577296733856201, + 3.571927547454834, + 3.578665018081665, + 3.7042040824890137, + 3.5339090824127197, + 3.475395441055298, + 3.5896246433258057, + 3.543846845626831, + 3.5294272899627686, + 3.615243673324585, + 3.5393226146698, + 3.6097445487976074, + 3.607759714126587, + 3.65932297706604, + 3.661882162094116, + 3.457880973815918, + 3.629316568374634, + 3.6324336528778076, + 3.5946662425994873, + 3.6057446002960205, + 3.4990689754486084, + 3.5881848335266113, + 3.5302822589874268, + 3.6231822967529297, + 3.613046884536743, + 3.539332389831543, + 3.6191565990448, + 3.463595151901245, + 3.4200875759124756, + 3.312417984008789, + 3.563810110092163, + 3.4601781368255615, + 3.430610179901123, + 3.366912364959717, + 3.3962883949279785, + 3.4877119064331055, + 3.583091974258423, + 3.4461610317230225, + 3.311521053314209, + 3.441527843475342, + 3.5334174633026123, + 3.4913012981414795, + 3.4548585414886475, + 3.4269397258758545, + 3.6881866455078125, + 3.3856399059295654, + 3.471531629562378, + 3.457878351211548, + 3.465489149093628, + 3.547548770904541, + 3.5829758644104004, + 3.5164241790771484, + 3.5706305503845215, + 3.3957152366638184, + 3.5903420448303223, + 3.571770429611206, + 3.4868388175964355, + 3.5041658878326416, + 3.345557451248169, + 3.505270481109619, + 3.5184240341186523, + 3.626474380493164, + 3.6197171211242676, + 3.609659433364868, + 3.590569257736206, + 3.5317840576171875, + 3.5047264099121094, + 3.4509313106536865, + 3.442387819290161, + 3.491013288497925, + 3.38655424118042, + 3.348194122314453, + 3.398505926132202, + 3.480621814727783, + 3.4380335807800293, + 3.538038730621338, + 3.488377332687378, + 3.464092969894409, + 3.6004772186279297, + 3.5556371212005615, + 3.504232406616211, + 3.4930405616760254, + 3.5164971351623535, + 3.539936065673828, + 3.471060276031494, + 3.530194044113159, + 3.497767686843872, + 3.5341501235961914, + 3.6325440406799316, + 3.376755714416504, + 3.435486316680908, + 3.593982458114624, + 3.570699453353882, + 3.546198606491089, + 3.464466094970703, + 3.4877588748931885, + 3.440142869949341, + 3.569988965988159, + 3.523489236831665, + 3.452744960784912, + 3.4358835220336914, + 3.4746193885803223, + 3.4920713901519775, + 3.488189697265625, + 3.525238275527954, + 3.5478200912475586, + 3.5285658836364746, + 3.596484899520874, + 3.3623762130737305, + 3.4215939044952393, + 3.438596487045288, + 3.5517630577087402, + 3.5949783325195312, + 3.5054566860198975, + 3.473219394683838, + 3.4375410079956055, + 3.4310617446899414, + 3.5124497413635254, + 3.446469306945801, + 3.4353718757629395, + 3.524301290512085, + 3.503054141998291, + 3.488851547241211, + 3.5654995441436768, + 3.533007860183716, + 3.5283854007720947, + 3.5752322673797607, + 3.5797581672668457, + 3.4208474159240723, + 3.554598093032837, + 3.5223207473754883, + 3.4884138107299805, + 3.487464427947998, + 3.4674882888793945, + 3.378002882003784, + 3.5721423625946045, + 3.430939197540283, + 3.5654890537261963, + 3.3318803310394287, + 3.353437900543213, + 3.4407315254211426, + 3.5377790927886963, + 3.42160701751709, + 3.473428249359131, + 3.5251779556274414, + 3.5585556030273438, + 3.410799741744995, + 3.4675371646881104, + 3.541485548019409, + 3.39182710647583, + 3.5243372917175293, + 3.4842519760131836, + 3.4263453483581543, + 3.3745648860931396, + 3.4177823066711426, + 3.4644124507904053, + 3.5351758003234863, + 3.255647897720337, + 3.4488754272460938, + 3.4739649295806885, + 3.5134878158569336, + 3.463470697402954, + 3.3248000144958496, + 3.560521125793457, + 3.332873582839966, + 3.365851402282715, + 3.4314112663269043, + 3.469794988632202, + 3.488878011703491, + 3.5029642581939697, + 3.5189619064331055, + 3.3592641353607178, + 3.4263176918029785, + 3.310826301574707, + 3.389213800430298, + 3.289947748184204, + 3.406578302383423, + 3.2723212242126465, + 3.4453423023223877, + 3.509204149246216, + 3.462379217147827, + 3.4059646129608154, + 3.501689910888672, + 3.2483768463134766, + 3.4313111305236816, + 3.421837568283081, + 3.354240655899048, + 3.454615354537964, + 3.42341947555542, + 3.2566323280334473, + 3.3640658855438232, + 3.3294804096221924, + 3.418318510055542, + 3.3096184730529785, + 3.44028377532959, + 3.4951419830322266, + 3.502328634262085, + 3.3977768421173096, + 3.310824155807495, + 3.3357577323913574, + 3.28821063041687, + 3.305285930633545, + 3.45745849609375, + 3.5315377712249756, + 3.470158576965332, + 3.3582677841186523, + 3.390821695327759, + 3.4559271335601807, + 3.3953704833984375, + 3.348632335662842, + 3.3555986881256104, + 3.489762544631958, + 3.4813427925109863, + 3.4372708797454834, + 3.4248268604278564, + 3.4078168869018555, + 3.4173426628112793, + 3.4025063514709473, + 3.4174704551696777, + 3.4353678226470947, + 3.478290557861328, + 3.394871234893799, + 3.3971009254455566, + 3.371495008468628, + 3.4584431648254395, + 3.340479850769043, + 3.494901418685913, + 3.3037991523742676, + 3.4072299003601074, + 3.3690030574798584, + 3.551665782928467, + 3.587146520614624, + 3.362774610519409, + 3.475062847137451, + 3.5205838680267334, + 3.3998897075653076, + 3.3812437057495117, + 3.454296827316284, + 3.3932950496673584, + 3.4223251342773438, + 3.434382915496826, + 3.477303981781006, + 3.440884590148926, + 3.3582112789154053, + 3.345170497894287, + 3.48717999458313, + 3.4781417846679688, + 3.4002065658569336, + 3.3999476432800293, + 3.331306219100952, + 3.450618267059326, + 3.5295603275299072, + 3.38529109954834, + 3.377493381500244, + 3.483652353286743, + 3.4911060333251953, + 3.4619083404541016, + 3.380493640899658, + 3.4106013774871826, + 3.349123477935791, + 3.2199933528900146, + 3.4586386680603027, + 3.311798095703125, + 3.4276907444000244, + 3.4687743186950684, + 3.4861061573028564, + 3.4329047203063965, + 3.4064667224884033, + 3.2755112648010254, + 3.523571729660034, + 3.350355386734009, + 3.3426167964935303, + 3.4793572425842285, + 3.4662554264068604, + 3.4112279415130615, + 3.488755226135254, + 3.5240375995635986, + 3.4157323837280273, + 3.5731594562530518, + 3.4253273010253906, + 3.440035581588745, + 3.37572979927063, + 3.3542513847351074, + 3.322943687438965, + 3.419133186340332, + 3.306135892868042, + 3.424227714538574, + 3.4001450538635254, + 3.4098873138427734, + 3.237905979156494, + 3.361445426940918, + 3.294588327407837, + 3.5028185844421387, + 3.516068935394287, + 3.4495701789855957, + 3.3750157356262207, + 3.37153959274292, + 3.3980724811553955, + 3.225006103515625, + 3.433206558227539, + 3.2839438915252686, + 3.3672287464141846, + 3.3713767528533936, + 3.3936007022857666, + 3.4620280265808105, + 3.4581480026245117, + 3.2249505519866943, + 3.501387119293213, + 3.3607494831085205, + 3.3025052547454834, + 3.414036750793457, + 3.3858094215393066, + 3.3402421474456787, + 3.3491547107696533, + 3.4552574157714844, + 3.4358770847320557, + 3.411890745162964, + 3.3698601722717285, + 3.367117404937744, + 3.3597512245178223, + 3.430037260055542, + 3.3503620624542236, + 3.4182322025299072, + 3.4008500576019287, + 3.3459627628326416, + 3.226235866546631, + 3.3670411109924316, + 3.350267171859741, + 3.315990447998047, + 3.303490161895752, + 3.2334065437316895, + 3.4591219425201416, + 3.4392035007476807, + 3.3058197498321533, + 3.4708690643310547, + 3.359363317489624, + 3.2612712383270264, + 3.2621028423309326, + 3.396735429763794, + 3.3988518714904785, + 3.3294131755828857, + 3.361360549926758, + 3.438218116760254, + 3.370763063430786, + 3.4267423152923584, + 3.3442842960357666, + 3.3367693424224854, + 3.4869894981384277, + 3.422250986099243, + 3.462123394012451, + 3.4183990955352783, + 3.4370229244232178, + 3.2698757648468018, + 3.330665111541748, + 3.3765854835510254, + 3.254934787750244, + 3.3138339519500732, + 3.320761203765869, + 3.396378517150879, + 3.410712718963623, + 3.384523868560791, + 3.3054659366607666, + 3.358384609222412, + 3.2572567462921143, + 3.562272310256958, + 3.440343141555786, + 3.4408440589904785, + 3.4672043323516846, + 3.332672595977783, + 3.3570406436920166, + 3.2069051265716553, + 3.232891798019409, + 3.3882699012756348, + 3.384843349456787, + 3.324143171310425, + 3.4484665393829346, + 3.2050673961639404, + 3.3799211978912354, + 3.5219314098358154, + 3.5162782669067383, + 3.431492805480957, + 3.4372382164001465, + 3.326167345046997, + 3.3637726306915283, + 3.2877771854400635, + 3.2430450916290283, + 3.435537099838257, + 3.344485282897949, + 3.457425355911255, + 3.244418144226074, + 3.3558993339538574, + 3.528143882751465, + 3.2939839363098145 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 127.01320277018169, + 77.71921154648855, + 61.02136020296158, + 53.23473578721172, + 48.53911963500999, + 44.9824396290839, + 42.441197283293114, + 40.53816004540091, + 38.685047428517194, + 36.99264346885452, + 36.0891758854044, + 34.71110822105002, + 34.11591083522134, + 33.11242367070357, + 32.430196365471296, + 32.01981779662449, + 31.67854652190319, + 31.45930132215406, + 31.35795820654025, + 31.33542200008362 + ], + "best_ppl": 31.33542200008362, + "train_time_s": 24743.53878712654 +} \ No newline at end of file diff --git a/062_attention_modes/downsampled_kv_best.pt b/062_attention_modes/downsampled_kv_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4d5597d6a8a76cf1eb2361deb34db386cbe9dd1 --- /dev/null +++ b/062_attention_modes/downsampled_kv_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9082da11e3c044db0638f1f55b146bc7ed69b5dfb41b70f708a103102470e017 +size 341438869 diff --git a/062_attention_modes/downsampled_kv_results.json b/062_attention_modes/downsampled_kv_results.json new file mode 100644 index 0000000000000000000000000000000000000000..dd1637b33288c57f1fe9ea4c37668842b7a6f2e9 --- /dev/null +++ b/062_attention_modes/downsampled_kv_results.json @@ -0,0 +1,1550 @@ +{ + "label": "downsampled_kv", + "attention_mode": "downsampled_kv", + "top_k_blocks": 3, + "num_thin_blocks": 5, + "total_params": 85301248, + "router_params": 40960, + "dense_ffn": true, + "d_ff_thin": 1024, + "steps": [ + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 3.8848817348480225, + 4.021523475646973, + 4.074449062347412, + 4.046614170074463, + 3.91280460357666, + 3.885073184967041, + 4.1015753746032715, + 3.9334588050842285, + 4.010954856872559, + 4.000702381134033, + 3.9100120067596436, + 3.890477180480957, + 4.078331470489502, + 4.009613037109375, + 4.014769077301025, + 4.039592266082764, + 3.925844192504883, + 3.9920451641082764, + 3.829911708831787, + 3.9816572666168213, + 3.951627492904663, + 3.9337666034698486, + 4.009873867034912, + 4.037670135498047, + 4.0948591232299805, + 3.9925379753112793, + 4.008405685424805, + 3.999882459640503, + 3.9371819496154785, + 4.032991886138916, + 3.8878798484802246, + 3.9006667137145996, + 3.937696933746338, + 3.920555591583252, + 3.872012138366699, + 3.8590047359466553, + 3.933526039123535, + 4.026650428771973, + 3.947321653366089, + 3.896862506866455, + 3.8835506439208984, + 3.8330132961273193, + 3.756740093231201, + 3.922866106033325, + 3.878173828125, + 3.9621663093566895, + 3.8627941608428955, + 3.9177818298339844, + 3.920814037322998, + 3.9333314895629883, + 3.884890079498291, + 3.79636287689209, + 3.8835229873657227, + 3.8885531425476074, + 3.9272241592407227, + 3.8290631771087646, + 3.9515914916992188, + 3.882495403289795, + 3.826274871826172, + 3.9042491912841797, + 3.9201035499572754, + 3.9500441551208496, + 3.9900062084198, + 3.8553829193115234, + 3.913430690765381, + 3.882671356201172, + 3.9296886920928955, + 3.9561405181884766, + 4.046018600463867, + 3.928711175918579, + 3.8246383666992188, + 3.930504322052002, + 3.7311830520629883, + 3.873023509979248, + 3.8073441982269287, + 3.7843925952911377, + 4.02789306640625, + 3.8395330905914307, + 3.9424479007720947, + 3.9894943237304688, + 3.840292453765869, + 3.721914768218994, + 3.8749194145202637, + 3.915954828262329, + 4.064558506011963, + 4.023256778717041, + 3.732254981994629, + 3.8969085216522217, + 3.913456916809082, + 3.892604351043701, + 4.010744094848633, + 3.9616026878356934, + 3.816781759262085, + 3.6886115074157715, + 3.963768720626831, + 3.9528095722198486, + 3.7950336933135986, + 3.8671844005584717, + 3.7889463901519775, + 3.9118692874908447, + 3.842707395553589, + 3.8434393405914307, + 3.8473048210144043, + 3.771021842956543, + 3.9222371578216553, + 3.872462034225464, + 3.7986044883728027, + 4.065333843231201, + 3.814393997192383, + 3.950608491897583, + 3.901104688644409, + 3.8199081420898438, + 3.8852784633636475, + 3.8766236305236816, + 3.8974978923797607, + 3.8892993927001953, + 3.7913405895233154, + 3.9610183238983154, + 3.8709781169891357, + 3.826204299926758, + 3.9197587966918945, + 3.882622480392456, + 3.762305974960327, + 3.8729074001312256, + 3.8549773693084717, + 3.8884687423706055, + 3.877261161804199, + 3.7761709690093994, + 3.7883148193359375, + 3.8478846549987793, + 3.7739627361297607, + 3.859255313873291, + 3.7886698246002197, + 3.810203790664673, + 3.718352794647217, + 3.7069125175476074, + 3.745274543762207, + 3.7962539196014404, + 3.8392815589904785, + 3.780010461807251, + 3.7364883422851562, + 3.7703349590301514, + 3.754791736602783, + 3.643310308456421, + 3.8422250747680664, + 3.6656551361083984, + 3.7625813484191895, + 3.835340976715088, + 3.794900417327881, + 3.818814277648926, + 3.870731830596924, + 3.7257606983184814, + 3.786264181137085, + 3.7884650230407715, + 3.6111714839935303, + 3.783095121383667, + 3.7550225257873535, + 3.753635883331299, + 3.7387313842773438, + 3.693411111831665, + 3.8076181411743164, + 3.8074333667755127, + 3.774775743484497, + 3.8147244453430176, + 3.689741373062134, + 3.8834638595581055, + 3.8556978702545166, + 3.822868824005127, + 3.7044646739959717, + 3.5907881259918213, + 3.764730215072632, + 3.8142662048339844, + 3.830744743347168, + 3.720741033554077, + 3.919297456741333, + 3.8226068019866943, + 3.6935529708862305, + 3.784111738204956, + 3.8312301635742188, + 3.8082237243652344, + 3.756265878677368, + 3.7475085258483887, + 3.7987067699432373, + 3.746478319168091, + 3.7808644771575928, + 3.7266228199005127, + 3.6243724822998047, + 3.748189687728882, + 3.666278123855591, + 3.682462692260742, + 3.8270998001098633, + 3.7548158168792725, + 3.6370604038238525, + 3.6403801441192627, + 3.706955671310425, + 3.621412754058838, + 3.6306934356689453, + 3.799514055252075, + 3.793271780014038, + 3.6469979286193848, + 3.788421869277954, + 3.769157886505127, + 3.768688678741455, + 3.8404572010040283, + 3.6592609882354736, + 3.8000268936157227, + 3.81113600730896, + 3.7294821739196777, + 3.811206340789795, + 3.6176605224609375, + 3.605804920196533, + 3.8399085998535156, + 3.665419578552246, + 3.737224578857422, + 3.701111316680908, + 3.6453959941864014, + 3.821207046508789, + 3.704871892929077, + 3.687717914581299, + 3.5661263465881348, + 3.7786362171173096, + 3.8163063526153564, + 3.60029673576355, + 3.6746091842651367, + 3.817899227142334, + 3.6111390590667725, + 3.747803211212158, + 3.8340961933135986, + 3.666668176651001, + 3.753211736679077, + 3.7825777530670166, + 3.726423978805542, + 3.706137180328369, + 3.6569132804870605, + 3.818195104598999, + 3.6819727420806885, + 3.6665773391723633, + 3.6744086742401123, + 3.666808605194092, + 3.5742106437683105, + 3.7271077632904053, + 3.634146213531494, + 3.7100448608398438, + 3.8193790912628174, + 3.8397998809814453, + 3.7449300289154053, + 3.7378077507019043, + 3.6471190452575684, + 3.7678678035736084, + 3.770132064819336, + 3.7382760047912598, + 3.6508028507232666, + 3.696925401687622, + 3.7215170860290527, + 3.4911348819732666, + 3.604121446609497, + 3.703404188156128, + 3.7559611797332764, + 3.5870203971862793, + 3.714113235473633, + 3.7172763347625732, + 3.5902960300445557, + 3.6982383728027344, + 3.659623146057129, + 3.6569366455078125, + 3.707552433013916, + 3.680725574493408, + 3.7075414657592773, + 3.6769165992736816, + 3.6404640674591064, + 3.7529263496398926, + 3.5497303009033203, + 3.8133039474487305, + 3.5744497776031494, + 3.6928141117095947, + 3.6020092964172363, + 3.7053465843200684, + 3.631864309310913, + 3.5145111083984375, + 3.59955096244812, + 3.626981019973755, + 3.6993744373321533, + 3.520561695098877, + 3.7460289001464844, + 3.719761610031128, + 3.674830198287964, + 3.629563093185425, + 3.577897310256958, + 3.6166937351226807, + 3.634315252304077, + 3.5788073539733887, + 3.830327033996582, + 3.7418181896209717, + 3.6100146770477295, + 3.671304225921631, + 3.651839256286621, + 3.5669541358947754, + 3.5718302726745605, + 3.6422479152679443, + 3.513601303100586, + 3.6682772636413574, + 3.624084949493408, + 3.7107040882110596, + 3.6827659606933594, + 3.586167335510254, + 3.6209261417388916, + 3.6271917819976807, + 3.4467203617095947, + 3.779381513595581, + 3.684786558151245, + 3.7515480518341064, + 3.708353042602539, + 3.5040531158447266, + 3.5778560638427734, + 3.739561080932617, + 3.6313328742980957, + 3.6446688175201416, + 3.672611951828003, + 3.515911340713501, + 3.5992095470428467, + 3.6365854740142822, + 3.670862913131714, + 3.633383274078369, + 3.7087061405181885, + 3.539964437484741, + 3.6948132514953613, + 3.5525474548339844, + 3.579293727874756, + 3.5674989223480225, + 3.555982828140259, + 3.496635675430298, + 3.6653130054473877, + 3.5567800998687744, + 3.702554941177368, + 3.5702528953552246, + 3.484980344772339, + 3.596355438232422, + 3.548086643218994, + 3.644273281097412, + 3.5478782653808594, + 3.624784469604492, + 3.4039485454559326, + 3.6670289039611816, + 3.7169675827026367, + 3.455479383468628, + 3.6867318153381348, + 3.6047680377960205, + 3.613534450531006, + 3.5438601970672607, + 3.719243288040161, + 3.485121488571167, + 3.676720142364502, + 3.619014024734497, + 3.5660126209259033, + 3.635528326034546, + 3.6897857189178467, + 3.6775426864624023, + 3.5825986862182617, + 3.584428071975708, + 3.545811414718628, + 3.6197938919067383, + 3.6517019271850586, + 3.581299066543579, + 3.4613897800445557, + 3.6698172092437744, + 3.6183645725250244, + 3.7390694618225098, + 3.708191394805908, + 3.676478147506714, + 3.707857131958008, + 3.6221373081207275, + 3.5783963203430176, + 3.6227149963378906, + 3.4826791286468506, + 3.6603124141693115, + 3.6167078018188477, + 3.547727346420288, + 3.616914987564087, + 3.5515692234039307, + 3.5749661922454834, + 3.483163356781006, + 3.500720500946045, + 3.5035078525543213, + 3.4177567958831787, + 3.480686664581299, + 3.5093772411346436, + 3.4675440788269043, + 3.5556812286376953, + 3.5379886627197266, + 3.631387948989868, + 3.4960811138153076, + 3.43340802192688, + 3.5312745571136475, + 3.534553050994873, + 3.6735966205596924, + 3.545158863067627, + 3.5288913249969482, + 3.5579588413238525, + 3.700979232788086, + 3.464677333831787, + 3.4072368144989014, + 3.518772840499878, + 3.5351204872131348, + 3.574733018875122, + 3.598513603210449, + 3.4352736473083496, + 3.5945446491241455, + 3.5595972537994385, + 3.543924331665039, + 3.583972215652466, + 3.6249067783355713, + 3.5262322425842285, + 3.5729262828826904, + 3.4789857864379883, + 3.6762797832489014, + 3.4996821880340576, + 3.6199142932891846, + 3.493825912475586, + 3.4926679134368896, + 3.582057237625122, + 3.4693472385406494, + 3.435389280319214, + 3.5760655403137207, + 3.5794837474823, + 3.6108832359313965, + 3.576432466506958, + 3.5252468585968018, + 3.519918203353882, + 3.5407345294952393, + 3.466479539871216, + 3.522942066192627, + 3.5495781898498535, + 3.6769094467163086, + 3.5601911544799805, + 3.637967824935913, + 3.675950288772583, + 3.6012630462646484, + 3.489438772201538, + 3.506053924560547, + 3.567070484161377, + 3.5272176265716553, + 3.4603919982910156, + 3.586617946624756, + 3.592226266860962, + 3.44352126121521, + 3.475398540496826, + 3.608381509780884, + 3.5158700942993164, + 3.5907328128814697, + 3.5252134799957275, + 3.5053598880767822, + 3.532379150390625, + 3.52833890914917, + 3.5079081058502197, + 3.549957036972046, + 3.465336322784424, + 3.5805675983428955, + 3.6276254653930664, + 3.402804136276245, + 3.594282627105713, + 3.5598926544189453, + 3.590010643005371, + 3.495124578475952, + 3.524543285369873, + 3.536733627319336, + 3.51284122467041, + 3.553182601928711, + 3.3733317852020264, + 3.582331418991089, + 3.4590299129486084, + 3.412616729736328, + 3.5570120811462402, + 3.4761104583740234, + 3.443211317062378, + 3.5586109161376953, + 3.5647451877593994, + 3.465144395828247, + 3.4209165573120117, + 3.514608144760132, + 3.616128921508789, + 3.4625566005706787, + 3.569918632507324, + 3.473389148712158, + 3.506279706954956, + 3.401129722595215, + 3.3655941486358643, + 3.545745611190796, + 3.4891040325164795, + 3.6224074363708496, + 3.4784021377563477, + 3.4535903930664062, + 3.620389461517334, + 3.620047092437744, + 3.4656434059143066, + 3.571519613265991, + 3.481278657913208, + 3.5632290840148926, + 3.6008822917938232, + 3.539658546447754, + 3.528576135635376, + 3.5662453174591064, + 3.5376696586608887, + 3.529771566390991, + 3.490910768508911, + 3.4759111404418945, + 3.5563368797302246, + 3.2734341621398926, + 3.4907877445220947, + 3.5274007320404053, + 3.470088481903076, + 3.5033769607543945, + 3.3490712642669678, + 3.418034553527832, + 3.524183988571167, + 3.437422037124634, + 3.5064003467559814, + 3.460117816925049, + 3.4012997150421143, + 3.525424003601074, + 3.468226671218872, + 3.556269407272339, + 3.394192695617676, + 3.353874921798706, + 3.370917797088623, + 3.4546823501586914, + 3.491964817047119, + 3.5503852367401123, + 3.4650020599365234, + 3.6215362548828125, + 3.397165298461914, + 3.3227345943450928, + 3.4954793453216553, + 3.5017597675323486, + 3.4629549980163574, + 3.3440308570861816, + 3.454686403274536, + 3.453134298324585, + 3.5344018936157227, + 3.4429378509521484, + 3.439141035079956, + 3.3978796005249023, + 3.488842010498047, + 3.4326283931732178, + 3.5706868171691895, + 3.4489946365356445, + 3.524279832839966, + 3.3789963722229004, + 3.5931761264801025, + 3.4055192470550537, + 3.4142842292785645, + 3.4761130809783936, + 3.4678213596343994, + 3.5425496101379395, + 3.464942455291748, + 3.4870107173919678, + 3.511279821395874, + 3.331195831298828, + 3.374157667160034, + 3.370298147201538, + 3.507565975189209, + 3.461149215698242, + 3.4307100772857666, + 3.4696762561798096, + 3.4695959091186523, + 3.349250078201294, + 3.460798740386963, + 3.4864110946655273, + 3.4742159843444824, + 3.4201695919036865, + 3.2724156379699707, + 3.5994338989257812, + 3.6095266342163086, + 3.514213800430298, + 3.4452052116394043, + 3.372098922729492, + 3.50361967086792, + 3.3479793071746826, + 3.3101532459259033, + 3.510037899017334, + 3.540367364883423, + 3.449968099594116, + 3.416198253631592, + 3.4074923992156982, + 3.3662991523742676, + 3.4320662021636963, + 3.464683771133423, + 3.350064992904663, + 3.4780545234680176, + 3.4139418601989746, + 3.432741403579712, + 3.484325408935547, + 3.476679563522339, + 3.405885696411133, + 3.4816558361053467, + 3.5181045532226562, + 3.467155694961548, + 3.410595178604126, + 3.370135545730591, + 3.3822097778320312, + 3.3795742988586426, + 3.398200035095215, + 3.316312313079834, + 3.4244465827941895, + 3.518305778503418, + 3.3319060802459717, + 3.513817548751831, + 3.5173962116241455, + 3.4674274921417236, + 3.4971964359283447, + 3.432558536529541, + 3.3370418548583984, + 3.4262661933898926, + 3.3482401371002197, + 3.518409252166748, + 3.618523120880127, + 3.3731095790863037, + 3.525193214416504, + 3.3522064685821533, + 3.4436957836151123, + 3.322415351867676, + 3.3873164653778076, + 3.510528326034546, + 3.5417566299438477, + 3.421762704849243, + 3.4269614219665527, + 3.515085458755493, + 3.473344326019287, + 3.531728744506836, + 3.4349653720855713, + 3.4676671028137207, + 3.43729829788208, + 3.3853657245635986, + 3.4373252391815186, + 3.2815728187561035, + 3.2879016399383545, + 3.437324047088623, + 3.399991989135742, + 3.3647682666778564, + 3.344477891921997, + 3.2958333492279053, + 3.2672808170318604, + 3.4040021896362305, + 3.362978935241699, + 3.279392719268799, + 3.410447835922241, + 3.371476888656616, + 3.39996075630188, + 3.4007532596588135, + 3.4668917655944824, + 3.4733471870422363, + 3.430804967880249, + 3.317939043045044, + 3.3757171630859375, + 3.4029555320739746, + 3.474281072616577, + 3.4015088081359863, + 3.392915964126587, + 3.341564178466797, + 3.3972785472869873, + 3.52701735496521, + 3.389413595199585, + 3.3147029876708984, + 3.4004600048065186, + 3.41007399559021, + 3.4940876960754395, + 3.389326572418213, + 3.317682981491089, + 3.4309580326080322, + 3.4211087226867676, + 3.4256932735443115, + 3.4604742527008057, + 3.354069709777832, + 3.4270105361938477, + 3.383387565612793, + 3.2972917556762695, + 3.3955695629119873, + 3.292276620864868, + 3.383683443069458, + 3.4937076568603516, + 3.3087966442108154, + 3.375408172607422, + 3.2628161907196045, + 3.473430633544922, + 3.3979501724243164, + 3.5328304767608643, + 3.387976884841919, + 3.3568387031555176, + 3.4019880294799805, + 3.449491500854492, + 3.288790702819824, + 3.2811241149902344, + 3.413360118865967, + 3.522514820098877, + 3.416184663772583, + 3.367243766784668, + 3.430569887161255, + 3.4162254333496094, + 3.316957473754883, + 3.5172324180603027, + 3.481950044631958, + 3.2808609008789062, + 3.3805809020996094, + 3.404008626937866, + 3.3258702754974365, + 3.5068233013153076, + 3.492882013320923, + 3.391446828842163, + 3.3717668056488037, + 3.4236676692962646, + 3.372178316116333, + 3.327331066131592, + 3.464132785797119, + 3.3890438079833984, + 3.266350507736206, + 3.5327067375183105, + 3.468928575515747, + 3.529924154281616, + 3.430002212524414, + 3.4956748485565186, + 3.408801317214966, + 3.3793833255767822, + 3.4705727100372314, + 3.375779390335083, + 3.439964771270752, + 3.3941380977630615, + 3.4278581142425537, + 3.2916767597198486, + 3.3778789043426514, + 3.2116458415985107, + 3.38106632232666, + 3.3630518913269043, + 3.3149518966674805, + 3.3873813152313232, + 3.5666558742523193, + 3.4252631664276123, + 3.4566574096679688, + 3.526684045791626, + 3.2703843116760254, + 3.3003664016723633, + 3.4643959999084473, + 3.3198604583740234, + 3.3226568698883057, + 3.323026418685913, + 3.5383546352386475, + 3.2557172775268555, + 3.4515836238861084, + 3.409557580947876, + 3.385468006134033, + 3.549940824508667, + 3.2864925861358643 + ], + "val_steps": [ + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 46.26675859755556, + 43.64418999353795, + 41.90260098734844, + 39.78036983703053, + 38.300356688912125, + 37.078712532524705, + 35.689692985403155, + 34.82243947104565, + 34.0391236225716, + 33.1849575380452, + 32.8350501156039, + 32.46469704257234, + 32.24005981018026, + 32.14067968240271, + 32.1163634449671 + ], + "best_ppl": 32.1163634449671, + "train_time_s": 18277.56588602066 +} \ No newline at end of file diff --git a/062_attention_modes/restricted_best.pt b/062_attention_modes/restricted_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ffabec485d988d1d921e9c6a862217799d462f4 --- /dev/null +++ b/062_attention_modes/restricted_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:141b50457993aaa8136ce7bda30a78b4a920739bf1144e57f356257519e80f6f +size 341436561 diff --git a/062_attention_modes/restricted_results.json b/062_attention_modes/restricted_results.json new file mode 100644 index 0000000000000000000000000000000000000000..90c2e80cbd15af134bcda69e9e75ea373c623dfa --- /dev/null +++ b/062_attention_modes/restricted_results.json @@ -0,0 +1,1448 @@ +{ + "label": "restricted", + "attention_mode": "restricted", + "top_k_blocks": 3, + "num_thin_blocks": 5, + "total_params": 85301248, + "router_params": 40960, + "dense_ffn": true, + "d_ff_thin": 1024, + "steps": [ + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 3.790450096130371, + 3.9492809772491455, + 3.9797823429107666, + 3.9332828521728516, + 3.830980062484741, + 3.7955896854400635, + 4.017726898193359, + 3.866920232772827, + 3.9171013832092285, + 3.918046712875366, + 3.835620164871216, + 3.8109030723571777, + 3.9960124492645264, + 3.947378158569336, + 3.933720827102661, + 3.9585633277893066, + 3.864603281021118, + 3.916750907897949, + 3.7522921562194824, + 3.906536340713501, + 3.8870508670806885, + 3.8548052310943604, + 3.9299824237823486, + 3.988797664642334, + 4.035221576690674, + 3.9168686866760254, + 3.9374642372131348, + 3.9298899173736572, + 3.8769469261169434, + 3.977003335952759, + 3.7917306423187256, + 3.83500599861145, + 3.858839273452759, + 3.8350718021392822, + 3.790501594543457, + 3.8022212982177734, + 3.852084159851074, + 3.9469690322875977, + 3.8733370304107666, + 3.83105731010437, + 3.787712812423706, + 3.7623419761657715, + 3.682813882827759, + 3.8674936294555664, + 3.8070590496063232, + 3.8814570903778076, + 3.7972605228424072, + 3.858717441558838, + 3.832369565963745, + 3.8612282276153564, + 3.8251731395721436, + 3.751882553100586, + 3.820927619934082, + 3.8186841011047363, + 3.8447766304016113, + 3.7583091259002686, + 3.894925355911255, + 3.812849283218384, + 3.7606775760650635, + 3.8450801372528076, + 3.8519535064697266, + 3.885822296142578, + 3.922405481338501, + 3.7828900814056396, + 3.863189697265625, + 3.799211025238037, + 3.8612289428710938, + 3.900042772293091, + 3.9777984619140625, + 3.8664536476135254, + 3.7615816593170166, + 3.8745481967926025, + 3.6616311073303223, + 3.7954888343811035, + 3.7560319900512695, + 3.7318615913391113, + 3.9434938430786133, + 3.7863519191741943, + 3.9047067165374756, + 3.916001081466675, + 3.7736523151397705, + 3.648625373840332, + 3.8040342330932617, + 3.862665891647339, + 3.985131025314331, + 3.9568727016448975, + 3.6743128299713135, + 3.852895975112915, + 3.8620193004608154, + 3.8334884643554688, + 3.961974859237671, + 3.91292405128479, + 3.7578399181365967, + 3.6346993446350098, + 3.908233404159546, + 3.8854191303253174, + 3.7394697666168213, + 3.8259007930755615, + 3.736670732498169, + 3.8549952507019043, + 3.7919487953186035, + 3.805284261703491, + 3.805678129196167, + 3.7120683193206787, + 3.8704535961151123, + 3.8104307651519775, + 3.73299503326416, + 4.028676509857178, + 3.7567508220672607, + 3.8991358280181885, + 3.8374183177948, + 3.7647197246551514, + 3.824688673019409, + 3.811861991882324, + 3.83363938331604, + 3.830821990966797, + 3.7296741008758545, + 3.906676769256592, + 3.8073198795318604, + 3.787229061126709, + 3.8703434467315674, + 3.8462772369384766, + 3.7034080028533936, + 3.840200901031494, + 3.804011583328247, + 3.8394765853881836, + 3.8292226791381836, + 3.725515365600586, + 3.7464747428894043, + 3.7961413860321045, + 3.722466230392456, + 3.799391508102417, + 3.7360124588012695, + 3.7576658725738525, + 3.687706232070923, + 3.6662886142730713, + 3.672675848007202, + 3.7638440132141113, + 3.7897510528564453, + 3.7425777912139893, + 3.687978982925415, + 3.7291100025177, + 3.7043933868408203, + 3.6160147190093994, + 3.798060655593872, + 3.625434160232544, + 3.726207971572876, + 3.7772984504699707, + 3.7429563999176025, + 3.7881741523742676, + 3.8374061584472656, + 3.6778743267059326, + 3.7265372276306152, + 3.7502286434173584, + 3.5711731910705566, + 3.733189821243286, + 3.7052478790283203, + 3.6984810829162598, + 3.6963605880737305, + 3.655102491378784, + 3.7530221939086914, + 3.7664859294891357, + 3.7170052528381348, + 3.7622547149658203, + 3.647644281387329, + 3.8243088722229004, + 3.8025829792022705, + 3.7842445373535156, + 3.66461181640625, + 3.548438787460327, + 3.7160329818725586, + 3.772213935852051, + 3.7843356132507324, + 3.67889666557312, + 3.8693172931671143, + 3.794039249420166, + 3.648141384124756, + 3.7435195446014404, + 3.803745985031128, + 3.749098300933838, + 3.7285923957824707, + 3.708791494369507, + 3.7543704509735107, + 3.708732843399048, + 3.7365074157714844, + 3.6802937984466553, + 3.581865072250366, + 3.696315050125122, + 3.6286659240722656, + 3.644857168197632, + 3.7799816131591797, + 3.7049386501312256, + 3.60666561126709, + 3.592907667160034, + 3.6614105701446533, + 3.58848237991333, + 3.590977907180786, + 3.75252366065979, + 3.740635871887207, + 3.603353500366211, + 3.7518653869628906, + 3.7207541465759277, + 3.7330400943756104, + 3.8014161586761475, + 3.6127336025238037, + 3.754516124725342, + 3.769547939300537, + 3.6908857822418213, + 3.7476210594177246, + 3.5778133869171143, + 3.576340436935425, + 3.8026981353759766, + 3.6149561405181885, + 3.7041032314300537, + 3.6629750728607178, + 3.5851073265075684, + 3.764371156692505, + 3.6607282161712646, + 3.651533365249634, + 3.5105509757995605, + 3.743040084838867, + 3.7763144969940186, + 3.5662920475006104, + 3.6607577800750732, + 3.7775964736938477, + 3.5435550212860107, + 3.707406759262085, + 3.7798733711242676, + 3.6262152194976807, + 3.7169766426086426, + 3.734595775604248, + 3.6860830783843994, + 3.6652705669403076, + 3.626790761947632, + 3.788580894470215, + 3.6390857696533203, + 3.644601345062256, + 3.6361870765686035, + 3.6162312030792236, + 3.551365852355957, + 3.6764371395111084, + 3.5881800651550293, + 3.668006420135498, + 3.7739498615264893, + 3.8019115924835205, + 3.694173812866211, + 3.6988420486450195, + 3.6065356731414795, + 3.73624324798584, + 3.686861991882324, + 3.665849208831787, + 3.572147846221924, + 3.62308406829834, + 3.6584041118621826, + 3.4308815002441406, + 3.5343403816223145, + 3.6190614700317383, + 3.679835796356201, + 3.514439344406128, + 3.633582592010498, + 3.6578567028045654, + 3.5130650997161865, + 3.63344669342041, + 3.5823872089385986, + 3.598372459411621, + 3.6517655849456787, + 3.6262764930725098, + 3.6529808044433594, + 3.623420476913452, + 3.5694379806518555, + 3.6936895847320557, + 3.477480888366699, + 3.7550535202026367, + 3.5233101844787598, + 3.6347179412841797, + 3.5229108333587646, + 3.6454882621765137, + 3.5593059062957764, + 3.4631950855255127, + 3.5386905670166016, + 3.5650203227996826, + 3.6202406883239746, + 3.460479974746704, + 3.6790058612823486, + 3.631376266479492, + 3.6229021549224854, + 3.572817325592041, + 3.508175849914551, + 3.5512077808380127, + 3.5677733421325684, + 3.4976744651794434, + 3.762092351913452, + 3.671143054962158, + 3.5339460372924805, + 3.6058359146118164, + 3.5847418308258057, + 3.4848058223724365, + 3.516012191772461, + 3.5609874725341797, + 3.472346544265747, + 3.641314744949341, + 3.6229326725006104, + 3.6701149940490723, + 3.6525895595550537, + 3.555858850479126, + 3.6057252883911133, + 3.5809195041656494, + 3.429701566696167, + 3.761251211166382, + 3.6535325050354004, + 3.7397665977478027, + 3.6729443073272705, + 3.4741499423980713, + 3.553271770477295, + 3.7246286869049072, + 3.611009359359741, + 3.613893508911133, + 3.641598701477051, + 3.4903204441070557, + 3.574833393096924, + 3.6085925102233887, + 3.6457467079162598, + 3.604571580886841, + 3.6789486408233643, + 3.5237598419189453, + 3.660799026489258, + 3.5183615684509277, + 3.5515544414520264, + 3.533116579055786, + 3.516873359680176, + 3.474924325942993, + 3.6372923851013184, + 3.531980037689209, + 3.675935745239258, + 3.5581295490264893, + 3.4627482891082764, + 3.5711891651153564, + 3.5280063152313232, + 3.607689380645752, + 3.5227136611938477, + 3.594686985015869, + 3.3941314220428467, + 3.6499526500701904, + 3.686235189437866, + 3.431478977203369, + 3.648231267929077, + 3.592911720275879, + 3.589581251144409, + 3.5459554195404053, + 3.7108404636383057, + 3.444159746170044, + 3.648117780685425, + 3.5936577320098877, + 3.544023275375366, + 3.5962767601013184, + 3.686030149459839, + 3.649594783782959, + 3.5571889877319336, + 3.5455398559570312, + 3.526122570037842, + 3.5869226455688477, + 3.608442544937134, + 3.555772542953491, + 3.439479351043701, + 3.6372780799865723, + 3.610511541366577, + 3.706793785095215, + 3.672445774078369, + 3.6536524295806885, + 3.680668830871582, + 3.593923807144165, + 3.5578129291534424, + 3.6067135334014893, + 3.4784743785858154, + 3.621500253677368, + 3.5848805904388428, + 3.5340301990509033, + 3.5938761234283447, + 3.5246450901031494, + 3.5503346920013428, + 3.4499623775482178, + 3.474820613861084, + 3.4713022708892822, + 3.4002795219421387, + 3.4454357624053955, + 3.4868035316467285, + 3.4412038326263428, + 3.5421204566955566, + 3.5127952098846436, + 3.612290620803833, + 3.4782934188842773, + 3.4144861698150635, + 3.5130581855773926, + 3.5222718715667725, + 3.6589267253875732, + 3.522958517074585, + 3.5025601387023926, + 3.5271527767181396, + 3.6896605491638184, + 3.439267873764038, + 3.3977084159851074, + 3.4949333667755127, + 3.515126943588257, + 3.535688638687134, + 3.5650527477264404, + 3.4056575298309326, + 3.584153890609741, + 3.5322647094726562, + 3.529726266860962, + 3.558642625808716, + 3.600797176361084, + 3.511141777038574, + 3.5404715538024902, + 3.4574263095855713, + 3.6540369987487793, + 3.4817678928375244, + 3.5981411933898926, + 3.467273473739624, + 3.462526798248291, + 3.5596182346343994, + 3.4431209564208984, + 3.397575855255127, + 3.537121057510376, + 3.5552175045013428, + 3.5947248935699463, + 3.547795295715332, + 3.500732898712158, + 3.493685483932495, + 3.4977567195892334, + 3.453871011734009, + 3.494386672973633, + 3.516038417816162, + 3.64460825920105, + 3.531801223754883, + 3.603135108947754, + 3.6503570079803467, + 3.5667436122894287, + 3.4398932456970215, + 3.473973512649536, + 3.539480686187744, + 3.5137665271759033, + 3.419259786605835, + 3.5594894886016846, + 3.5654313564300537, + 3.4288175106048584, + 3.443237781524658, + 3.594104528427124, + 3.496344566345215, + 3.5561180114746094, + 3.4954395294189453, + 3.48348069190979, + 3.5168731212615967, + 3.5004801750183105, + 3.47841477394104, + 3.512835741043091, + 3.4351460933685303, + 3.5541226863861084, + 3.5886740684509277, + 3.3716800212860107, + 3.569410562515259, + 3.5271706581115723, + 3.560610294342041, + 3.4716994762420654, + 3.512861490249634, + 3.50691294670105, + 3.506406784057617, + 3.5202224254608154, + 3.357218027114868, + 3.566636800765991, + 3.419064521789551, + 3.395268201828003, + 3.52838134765625, + 3.457144021987915, + 3.420983076095581, + 3.54579758644104, + 3.5511109828948975, + 3.445282459259033, + 3.3979687690734863, + 3.4910852909088135, + 3.5883498191833496, + 3.455587387084961, + 3.552382707595825, + 3.4465909004211426, + 3.4689388275146484, + 3.3871994018554688, + 3.325751781463623, + 3.515843152999878, + 3.4850692749023438, + 3.617766857147217, + 3.4456074237823486, + 3.408729076385498, + 3.589010238647461, + 3.595789670944214, + 3.4514644145965576, + 3.5627694129943848, + 3.4593405723571777, + 3.532005786895752, + 3.580838203430176, + 3.5206706523895264, + 3.5035808086395264, + 3.5358810424804688, + 3.51023530960083, + 3.5106630325317383, + 3.46213698387146, + 3.452677011489868, + 3.533653497695923, + 3.2547242641448975, + 3.4900028705596924, + 3.523132801055908, + 3.460688829421997, + 3.48016619682312, + 3.334113836288452, + 3.39797306060791, + 3.514037847518921, + 3.427321434020996, + 3.499401092529297, + 3.4417073726654053, + 3.3841588497161865, + 3.511582374572754, + 3.427225112915039, + 3.5392541885375977, + 3.3811113834381104, + 3.348762273788452, + 3.337156057357788, + 3.4207780361175537, + 3.468876600265503, + 3.528308629989624, + 3.4418184757232666, + 3.6073355674743652, + 3.3969626426696777, + 3.3006157875061035, + 3.479853868484497, + 3.4838032722473145, + 3.453148365020752, + 3.3367562294006348, + 3.4275062084198, + 3.4359982013702393, + 3.4931254386901855, + 3.4278576374053955, + 3.449589490890503, + 3.3835084438323975, + 3.4706642627716064, + 3.4227724075317383, + 3.550672769546509, + 3.4220376014709473, + 3.5117013454437256, + 3.3612372875213623, + 3.5746450424194336, + 3.4033820629119873, + 3.409935474395752, + 3.459061622619629, + 3.4672908782958984, + 3.534175395965576, + 3.4371931552886963, + 3.4721920490264893, + 3.487701416015625, + 3.3029625415802, + 3.340653896331787, + 3.352673292160034, + 3.4939334392547607, + 3.4610981941223145, + 3.4233577251434326, + 3.4605159759521484, + 3.449721336364746, + 3.337402582168579, + 3.4414141178131104, + 3.486272096633911, + 3.455885410308838, + 3.414763927459717, + 3.249502182006836, + 3.564873695373535, + 3.5960166454315186, + 3.5122416019439697, + 3.4477615356445312, + 3.3696577548980713, + 3.482866048812866, + 3.347792148590088, + 3.285337209701538, + 3.4856202602386475, + 3.525169849395752, + 3.43392276763916, + 3.3923211097717285, + 3.400099754333496, + 3.343458652496338, + 3.4236464500427246, + 3.442129373550415, + 3.3579440116882324, + 3.449191093444824, + 3.386547803878784, + 3.410463809967041, + 3.4693543910980225, + 3.458803415298462, + 3.4040439128875732, + 3.46061110496521, + 3.5099146366119385, + 3.446382999420166, + 3.391024351119995, + 3.37567138671875, + 3.3648555278778076, + 3.3815126419067383, + 3.3887860774993896, + 3.2960879802703857, + 3.4033141136169434, + 3.5048484802246094, + 3.331097364425659, + 3.5078060626983643, + 3.5272176265716553, + 3.4699795246124268, + 3.4774911403656006, + 3.4061193466186523, + 3.337980270385742, + 3.4224905967712402, + 3.33365797996521, + 3.503126621246338, + 3.6018271446228027, + 3.3843932151794434, + 3.5163302421569824, + 3.3573567867279053, + 3.4408857822418213, + 3.3017609119415283, + 3.3753459453582764, + 3.487358808517456, + 3.541088342666626, + 3.4168283939361572, + 3.4118056297302246, + 3.4956929683685303, + 3.4634897708892822, + 3.515498161315918, + 3.4329605102539062, + 3.457519769668579, + 3.4211783409118652, + 3.37626314163208, + 3.426966667175293, + 3.284271478652954, + 3.2976596355438232, + 3.4279918670654297, + 3.400728464126587, + 3.3819549083709717, + 3.3506460189819336, + 3.300379753112793, + 3.2663121223449707, + 3.394817590713501, + 3.366678476333618, + 3.283740997314453, + 3.437030553817749, + 3.3626272678375244, + 3.397613048553467, + 3.403170585632324, + 3.461005210876465, + 3.480410575866699, + 3.4302990436553955, + 3.3075573444366455, + 3.39839243888855, + 3.4103612899780273, + 3.48004412651062, + 3.4197771549224854, + 3.389474391937256, + 3.340935707092285, + 3.403606414794922, + 3.522568941116333, + 3.3928935527801514, + 3.31994366645813, + 3.401139736175537, + 3.410158634185791, + 3.4764232635498047, + 3.3968942165374756, + 3.328916311264038, + 3.431762456893921, + 3.423400402069092, + 3.4148294925689697, + 3.4744246006011963, + 3.3450493812561035, + 3.4298293590545654, + 3.3930296897888184, + 3.3083577156066895, + 3.386342763900757, + 3.2850844860076904, + 3.394247531890869, + 3.497939348220825, + 3.3107850551605225, + 3.3600213527679443, + 3.2686996459960938, + 3.460310459136963, + 3.3908958435058594, + 3.5122852325439453, + 3.379575490951538, + 3.348813056945801, + 3.3881540298461914, + 3.452545166015625, + 3.276597023010254, + 3.2906134128570557, + 3.3917441368103027, + 3.530339241027832, + 3.4178545475006104, + 3.3727259635925293, + 3.426248550415039, + 3.422724723815918, + 3.3250157833099365, + 3.508021116256714, + 3.488661050796509 + ], + "val_steps": [ + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 43.90837327750932, + 41.72490213042351, + 40.264477065578546, + 38.52010062532904, + 37.06677379766644, + 36.134965810074576, + 34.89622787756333, + 34.331926681490955, + 33.536405718339836, + 32.871576324424154, + 32.66295948028353, + 32.424402330379664, + 32.317061285424565, + 32.3059658066729 + ], + "best_ppl": 32.3059658066729, + "train_time_s": 15429.578290939331 +} \ No newline at end of file diff --git a/062_attention_modes/skip_proj_best.pt b/062_attention_modes/skip_proj_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2d97fa0860740d7958454723b2c6c4c315d003e --- /dev/null +++ b/062_attention_modes/skip_proj_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c03b61aa897a47e72e5c36f999cc36656da4a74e96a6b668479c5a47d022a442 +size 341438456 diff --git a/062_attention_modes/skip_proj_results.json b/062_attention_modes/skip_proj_results.json new file mode 100644 index 0000000000000000000000000000000000000000..30533f669dd6a0e4bb38be8e05b67f0bf4d35067 --- /dev/null +++ b/062_attention_modes/skip_proj_results.json @@ -0,0 +1,2060 @@ +{ + "label": "skip_proj", + "attention_mode": "skip_proj", + "top_k_blocks": 3, + "num_thin_blocks": 5, + "total_params": 85301256, + "router_params": 40960, + "dense_ffn": true, + "d_ff_thin": 1024, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.08596420288086, + 15.112953186035156, + 8.0263671875, + 7.335609436035156, + 7.162363529205322, + 6.974828243255615, + 6.853320121765137, + 6.744539737701416, + 6.7008185386657715, + 6.68684720993042, + 6.51012659072876, + 6.408962726593018, + 6.462562561035156, + 6.261905670166016, + 6.170009613037109, + 6.201412677764893, + 6.031660556793213, + 6.219124794006348, + 6.0319108963012695, + 5.954541206359863, + 5.957034587860107, + 5.849795341491699, + 5.803576469421387, + 5.802489757537842, + 5.684293746948242, + 5.58271598815918, + 5.591136932373047, + 5.695274353027344, + 5.6454057693481445, + 5.4506120681762695, + 5.335177421569824, + 5.554460048675537, + 5.3778581619262695, + 5.406036376953125, + 5.376981258392334, + 5.427975654602051, + 5.264865398406982, + 5.356331825256348, + 5.191220760345459, + 5.307287693023682, + 5.284099102020264, + 5.2416181564331055, + 5.135713577270508, + 5.274996280670166, + 5.154531478881836, + 5.080024242401123, + 5.206314563751221, + 5.009801864624023, + 5.136009693145752, + 5.040626525878906, + 5.06510066986084, + 5.130580902099609, + 4.930299282073975, + 5.097014904022217, + 5.0473127365112305, + 4.999566078186035, + 5.13397216796875, + 4.973996162414551, + 4.851098537445068, + 4.896252632141113, + 4.956125259399414, + 4.831935405731201, + 4.942730903625488, + 4.953147888183594, + 4.940771579742432, + 4.97278356552124, + 4.856249809265137, + 4.85536003112793, + 4.916842460632324, + 4.850441932678223, + 4.914752006530762, + 4.7794904708862305, + 4.7660393714904785, + 4.8482136726379395, + 4.797064304351807, + 4.8584394454956055, + 4.807549953460693, + 4.821938514709473, + 4.74113655090332, + 4.853353023529053, + 4.735696315765381, + 4.810214519500732, + 4.809896469116211, + 4.9083099365234375, + 4.808777332305908, + 4.642732620239258, + 4.678930282592773, + 4.69113302230835, + 4.712398529052734, + 4.701381683349609, + 4.617664337158203, + 4.860929489135742, + 4.620314598083496, + 4.654277801513672, + 4.711268901824951, + 4.691694259643555, + 4.499011993408203, + 4.626298427581787, + 4.69167947769165, + 4.525469779968262, + 4.7495903968811035, + 4.3980631828308105, + 4.722275733947754, + 4.693156719207764, + 4.640800952911377, + 4.580720901489258, + 4.687357425689697, + 4.591303825378418, + 4.632245063781738, + 4.350936412811279, + 4.54677152633667, + 4.6176066398620605, + 4.561553001403809, + 4.714099884033203, + 4.626282691955566, + 4.4241437911987305, + 4.4966888427734375, + 4.50506067276001, + 4.5307841300964355, + 4.590148448944092, + 4.455595016479492, + 4.575745105743408, + 4.545770645141602, + 4.49389123916626, + 4.477015018463135, + 4.621664047241211, + 4.534879207611084, + 4.603664875030518, + 4.332018852233887, + 4.539920330047607, + 4.489384651184082, + 4.415050506591797, + 4.567478656768799, + 4.438871383666992, + 4.46109676361084, + 4.517402172088623, + 4.498800277709961, + 4.420829772949219, + 4.471014499664307, + 4.524826526641846, + 4.435011386871338, + 4.316723823547363, + 4.331779479980469, + 4.646361351013184, + 4.3703203201293945, + 4.37485408782959, + 4.483345985412598, + 4.518678665161133, + 4.296083450317383, + 4.496599197387695, + 4.4600934982299805, + 4.450521469116211, + 4.328281402587891, + 4.494579315185547, + 4.393089771270752, + 4.272164344787598, + 4.4625444412231445, + 4.457320690155029, + 4.377748489379883, + 4.286036491394043, + 4.140857696533203, + 4.283174514770508, + 4.212278842926025, + 4.2368927001953125, + 4.35413932800293, + 4.394045352935791, + 4.34943962097168, + 4.361146926879883, + 4.360788345336914, + 4.290797710418701, + 4.365678310394287, + 4.2863359451293945, + 4.311891078948975, + 4.304139614105225, + 4.191216468811035, + 4.140519618988037, + 4.441884517669678, + 4.274441242218018, + 4.26220703125, + 4.345893859863281, + 4.339400768280029, + 4.209969520568848, + 4.191564559936523, + 4.251955509185791, + 4.346439361572266, + 4.133619785308838, + 4.129082679748535, + 4.205630779266357, + 4.299286842346191, + 4.319561004638672, + 4.25014066696167, + 4.048801898956299, + 4.318497180938721, + 4.354116439819336, + 4.292740821838379, + 4.313852310180664, + 4.305118083953857, + 4.138016700744629, + 4.23563289642334, + 4.221190929412842, + 4.218052387237549, + 4.345508098602295, + 4.158017158508301, + 4.254491329193115, + 4.180156707763672, + 4.092013835906982, + 4.127259731292725, + 4.245166778564453, + 4.2998433113098145, + 4.110184192657471, + 4.169925689697266, + 4.28633975982666, + 4.146646976470947, + 4.169228553771973, + 4.122506618499756, + 4.266409873962402, + 4.146570682525635, + 4.169849872589111, + 4.222439289093018, + 4.208511829376221, + 4.1810784339904785, + 4.193264484405518, + 4.024094104766846, + 4.177865982055664, + 4.083193778991699, + 4.20955228805542, + 4.307307720184326, + 4.146220684051514, + 4.062357425689697, + 4.294529438018799, + 4.2181878089904785, + 4.235630035400391, + 4.203685760498047, + 4.1151204109191895, + 4.0956292152404785, + 4.114746570587158, + 4.037007808685303, + 4.0697455406188965, + 4.147527694702148, + 4.306244373321533, + 4.218733787536621, + 4.109098434448242, + 4.090320110321045, + 4.080880641937256, + 4.1657280921936035, + 4.159702301025391, + 4.347695350646973, + 4.197429656982422, + 4.128251552581787, + 4.186938762664795, + 4.201210975646973, + 4.008620738983154, + 4.138296127319336, + 4.26945161819458, + 4.054989814758301, + 4.310856342315674, + 4.187769889831543, + 4.058160305023193, + 4.053998947143555, + 4.177056312561035, + 4.150158882141113, + 4.137502193450928, + 4.236532211303711, + 4.048389911651611, + 3.9339702129364014, + 4.061906814575195, + 4.1616716384887695, + 4.057629108428955, + 4.092045783996582, + 4.082358360290527, + 4.0935516357421875, + 4.060243606567383, + 4.088277339935303, + 4.072606563568115, + 4.067165851593018, + 4.013208389282227, + 4.147393226623535, + 4.028595447540283, + 4.183425426483154, + 4.082808494567871, + 4.188565254211426, + 4.0088653564453125, + 4.140608310699463, + 3.9888315200805664, + 4.0727643966674805, + 4.239567279815674, + 4.090461254119873, + 3.9598305225372314, + 4.095779895782471, + 3.913026809692383, + 4.028777122497559, + 4.264423370361328, + 3.943753719329834, + 4.1246137619018555, + 4.016755104064941, + 3.957732677459717, + 4.049840450286865, + 4.063866138458252, + 4.0336127281188965, + 3.8866515159606934, + 4.039880752563477, + 4.050871849060059, + 4.055139064788818, + 3.97371244430542, + 3.8989131450653076, + 4.036735534667969, + 3.960014581680298, + 4.1748552322387695, + 4.018701553344727, + 4.029501438140869, + 4.1451873779296875, + 3.9391255378723145, + 4.1266655921936035, + 3.9034900665283203, + 4.095554351806641, + 3.9309773445129395, + 4.051489353179932, + 4.035745620727539, + 3.982804298400879, + 3.9784796237945557, + 4.082574367523193, + 4.064925670623779, + 3.9101996421813965, + 4.069193363189697, + 4.177189350128174, + 4.088789939880371, + 3.984983205795288, + 3.9639430046081543, + 4.024221897125244, + 4.014024257659912, + 4.101980686187744, + 3.899986743927002, + 3.935299873352051, + 3.89492130279541, + 3.993849754333496, + 3.9384796619415283, + 4.02681827545166, + 3.753361225128174, + 4.014102935791016, + 3.8709867000579834, + 3.953214406967163, + 4.060172080993652, + 3.9971539974212646, + 3.702545166015625, + 3.9415440559387207, + 3.9543423652648926, + 4.121249675750732, + 4.013279914855957, + 3.893892765045166, + 3.957535982131958, + 3.9006264209747314, + 4.04348087310791, + 3.893193006515503, + 3.880720615386963, + 4.009401798248291, + 3.8015575408935547, + 3.9942073822021484, + 3.9524729251861572, + 3.845559597015381, + 3.731329917907715, + 3.9772653579711914, + 3.99229097366333, + 3.9753308296203613, + 3.9519150257110596, + 3.841139793395996, + 3.9313433170318604, + 3.925246000289917, + 4.118336200714111, + 4.07237434387207, + 3.955237865447998, + 4.029160022735596, + 3.992649555206299, + 4.014462947845459, + 3.92134165763855, + 3.968447208404541, + 4.044754981994629, + 3.9760031700134277, + 4.018696308135986, + 3.9734303951263428, + 3.780024290084839, + 3.9787721633911133, + 3.9742610454559326, + 3.9501099586486816, + 4.0439300537109375, + 3.7921552658081055, + 3.8809404373168945, + 4.006497859954834, + 3.873323440551758, + 3.9273147583007812, + 3.922809600830078, + 3.965951442718506, + 3.9060280323028564, + 3.967245578765869, + 3.8984248638153076, + 3.8747780323028564, + 4.0469889640808105, + 3.912890672683716, + 3.891556978225708, + 3.820467948913574, + 3.9826366901397705, + 3.788027048110962, + 3.879199743270874, + 3.8702585697174072, + 4.066781997680664, + 3.9999585151672363, + 3.8955018520355225, + 3.94968581199646, + 3.8528172969818115, + 3.850759744644165, + 4.034226894378662, + 3.9188053607940674, + 3.961162567138672, + 3.8670432567596436, + 4.009208679199219, + 3.8604893684387207, + 3.8165273666381836, + 3.9277119636535645, + 3.9067461490631104, + 3.8043127059936523, + 3.997068166732788, + 3.890028953552246, + 4.019509792327881, + 3.774484157562256, + 3.901820659637451, + 4.014969825744629, + 3.935299873352051, + 3.7242465019226074, + 3.981200933456421, + 3.9132251739501953, + 3.9868733882904053, + 3.985595226287842, + 3.978055953979492, + 3.8054275512695312, + 3.760118007659912, + 3.828514814376831, + 3.91749906539917, + 3.8263049125671387, + 3.9469404220581055, + 4.005192756652832, + 3.740673542022705, + 3.9124481678009033, + 3.8909382820129395, + 3.984639883041382, + 3.8231632709503174, + 3.9653830528259277, + 3.9797520637512207, + 3.8378360271453857, + 3.8493313789367676, + 3.93872332572937, + 3.8097105026245117, + 3.901761770248413, + 3.93644380569458, + 3.686256170272827, + 3.849897861480713, + 3.7636594772338867, + 3.9051120281219482, + 3.9146199226379395, + 3.7191741466522217, + 3.8878402709960938, + 3.8609087467193604, + 3.970632791519165, + 3.94317364692688, + 4.064704418182373, + 3.9241411685943604, + 3.95906138420105, + 3.954602003097534, + 3.8388025760650635, + 4.070152282714844, + 3.806033134460449, + 3.8903656005859375, + 3.9498119354248047, + 3.715920925140381, + 3.8230197429656982, + 3.7352023124694824, + 3.914660930633545, + 3.77030348777771, + 3.972745180130005, + 3.757124423980713, + 3.693021535873413, + 3.775299549102783, + 3.9227614402770996, + 3.816653251647949, + 3.776893377304077, + 3.8324666023254395, + 3.8708834648132324, + 3.852419853210449, + 3.8138999938964844, + 3.858811378479004, + 3.8338139057159424, + 3.8128128051757812, + 3.7936346530914307, + 3.9477169513702393, + 3.968468427658081, + 3.941634178161621, + 3.812743663787842, + 3.7824935913085938, + 3.9063220024108887, + 3.823521375656128, + 3.9402689933776855, + 3.828702926635742, + 3.7397782802581787, + 3.744950294494629, + 3.7325150966644287, + 3.9188461303710938, + 3.6377267837524414, + 3.8996636867523193, + 3.7179739475250244, + 3.838716983795166, + 3.7187860012054443, + 3.5923333168029785, + 3.7189764976501465, + 3.8144845962524414, + 3.602924346923828, + 3.8333213329315186, + 3.683295726776123, + 3.7295444011688232, + 3.743546724319458, + 3.9443628787994385, + 3.747615098953247, + 3.6898691654205322, + 3.529026746749878, + 3.6017863750457764, + 3.8568637371063232, + 3.796278476715088, + 3.727935791015625, + 3.808532238006592, + 3.734557867050171, + 3.7279186248779297, + 3.8765108585357666, + 3.7338221073150635, + 3.84126353263855, + 3.924421787261963, + 3.7064571380615234, + 3.8081071376800537, + 3.764094591140747, + 3.732940673828125, + 3.8280439376831055, + 3.73309063911438, + 3.6621253490448, + 3.631019353866577, + 3.7737512588500977, + 3.716798782348633, + 3.7936716079711914, + 3.6822938919067383, + 3.7987825870513916, + 3.5538218021392822, + 3.7302768230438232, + 3.6949291229248047, + 3.788316011428833, + 3.803739309310913, + 3.6564888954162598, + 3.90606689453125, + 4.0228118896484375, + 3.592679262161255, + 3.776718854904175, + 3.7050697803497314, + 3.695035219192505, + 3.7964653968811035, + 3.670964002609253, + 3.720205545425415, + 3.7099967002868652, + 3.815703868865967, + 3.733309745788574, + 3.7457361221313477, + 3.761735677719116, + 3.8668158054351807, + 3.833813190460205, + 3.787511110305786, + 3.765847682952881, + 3.8237364292144775, + 3.739912748336792, + 3.7234013080596924, + 3.6602673530578613, + 3.7048869132995605, + 3.824427604675293, + 3.702009439468384, + 3.713228702545166, + 3.6990275382995605, + 3.732243061065674, + 3.7340521812438965, + 3.8617310523986816, + 3.653820514678955, + 3.8139231204986572, + 3.6328935623168945, + 3.701763391494751, + 3.8330187797546387, + 3.71726655960083, + 3.9113121032714844, + 3.743748426437378, + 3.5777275562286377, + 3.7491962909698486, + 3.779327154159546, + 3.8315975666046143, + 3.694486141204834, + 3.6347880363464355, + 3.72171950340271, + 3.7569971084594727, + 3.712038993835449, + 3.551985263824463, + 3.7239022254943848, + 3.7240753173828125, + 3.5721678733825684, + 3.7493536472320557, + 3.7124876976013184, + 3.7534542083740234, + 3.692291021347046, + 3.694138526916504, + 3.737361192703247, + 3.5504889488220215, + 3.6964938640594482, + 3.7960009574890137, + 3.708327531814575, + 3.696687698364258, + 3.6952431201934814, + 3.8330845832824707, + 3.7381889820098877, + 3.6538658142089844, + 3.7009894847869873, + 3.7663655281066895, + 3.6667118072509766, + 3.701011896133423, + 3.5535521507263184, + 3.7822813987731934, + 3.6273365020751953, + 3.676969289779663, + 3.7647030353546143, + 3.574577569961548, + 3.543980360031128, + 3.7623558044433594, + 3.6729187965393066, + 3.7270169258117676, + 3.700514554977417, + 3.6975491046905518, + 3.781160593032837, + 3.627730369567871, + 3.7114109992980957, + 3.7039613723754883, + 3.8254377841949463, + 3.8272857666015625, + 3.6423494815826416, + 3.516512393951416, + 3.6619834899902344, + 3.8539516925811768, + 3.733736753463745, + 3.6514689922332764, + 3.703897714614868, + 3.8459739685058594, + 3.8417141437530518, + 3.6902637481689453, + 3.697580575942993, + 3.704631805419922, + 3.8165156841278076, + 3.6475753784179688, + 3.574460983276367, + 3.7707467079162598, + 3.651587724685669, + 3.5945394039154053, + 3.59765362739563, + 3.7024571895599365, + 3.708254098892212, + 3.767542600631714, + 3.5361340045928955, + 3.732144832611084, + 3.829630136489868, + 3.6577036380767822, + 3.6943280696868896, + 3.565087080001831, + 3.7206170558929443, + 3.8118515014648438, + 3.7850193977355957, + 3.6335785388946533, + 3.6788432598114014, + 3.5105834007263184, + 3.7418503761291504, + 3.6394293308258057, + 3.672574043273926, + 3.6800267696380615, + 3.5434091091156006, + 3.5144152641296387, + 3.6515071392059326, + 3.5934407711029053, + 3.6292977333068848, + 3.6385462284088135, + 3.6817750930786133, + 3.6061925888061523, + 3.775357961654663, + 3.520813226699829, + 3.7167105674743652, + 3.576526641845703, + 3.4948835372924805, + 3.7864367961883545, + 3.6154215335845947, + 3.6039178371429443, + 3.6999752521514893, + 3.5475010871887207, + 3.642014265060425, + 3.6460320949554443, + 3.7287046909332275, + 3.7744152545928955, + 3.737931251525879, + 3.463552951812744, + 3.5512330532073975, + 3.702157974243164, + 3.486721992492676, + 3.649043321609497, + 3.6004464626312256, + 3.581281900405884, + 3.6583597660064697, + 3.6640284061431885, + 3.658168077468872, + 3.53995418548584, + 3.7765233516693115, + 3.6754660606384277, + 3.6402719020843506, + 3.640415906906128, + 3.6641294956207275, + 3.6369426250457764, + 3.620039701461792, + 3.6726486682891846, + 3.6010825634002686, + 3.7462801933288574, + 3.5714187622070312, + 3.58567476272583, + 3.5932486057281494, + 3.6223690509796143, + 3.715517997741699, + 3.6280243396759033, + 3.797537088394165, + 3.5225462913513184, + 3.6174821853637695, + 3.710082769393921, + 3.676952362060547, + 3.5693461894989014, + 3.582477331161499, + 3.7284560203552246, + 3.70566463470459, + 3.64142107963562, + 3.594724655151367, + 3.662034749984741, + 3.6806864738464355, + 3.696040153503418, + 3.7563908100128174, + 3.738077163696289, + 3.676623821258545, + 3.5012893676757812, + 3.5391292572021484, + 3.5263383388519287, + 3.5559916496276855, + 3.5970325469970703, + 3.486393451690674, + 3.704353094100952, + 3.5648491382598877, + 3.5966756343841553, + 3.608762264251709, + 3.5060219764709473, + 3.692847967147827, + 3.719132661819458, + 3.6939773559570312, + 3.630653142929077, + 3.603914976119995, + 3.6560230255126953, + 3.6028289794921875, + 3.5694756507873535, + 3.6478233337402344, + 3.7825043201446533, + 3.636652708053589, + 3.7280113697052, + 3.6046411991119385, + 3.5739004611968994, + 3.6917452812194824, + 3.639514684677124, + 3.6001195907592773, + 3.663395404815674, + 3.687865972518921, + 3.738412618637085, + 3.667854070663452, + 3.516444683074951, + 3.6255037784576416, + 3.6289660930633545, + 3.594578981399536, + 3.5532329082489014, + 3.5661089420318604, + 3.7006351947784424, + 3.60062313079834, + 3.4116647243499756, + 3.5010523796081543, + 3.6491990089416504, + 3.659329652786255, + 3.6353585720062256, + 3.5765557289123535, + 3.532074451446533, + 3.6134192943573, + 3.4954378604888916, + 3.592421770095825, + 3.5442144870758057, + 3.494631052017212, + 3.601698637008667, + 3.660447835922241, + 3.4963834285736084, + 3.4693939685821533, + 3.5191547870635986, + 3.646174669265747, + 3.653712272644043, + 3.567798137664795, + 3.495927572250366, + 3.6814961433410645, + 3.4773404598236084, + 3.7332241535186768, + 3.461251735687256, + 3.5979020595550537, + 3.573049783706665, + 3.7344961166381836, + 3.449291229248047, + 3.6872317790985107, + 3.4392924308776855, + 3.5439629554748535, + 3.7055485248565674, + 3.6501972675323486, + 3.577589511871338, + 3.599369525909424, + 3.577558994293213, + 3.5437583923339844, + 3.635258436203003, + 3.615858793258667, + 3.4959840774536133, + 3.599799871444702, + 3.668705463409424, + 3.4389071464538574, + 3.6464128494262695, + 3.5538477897644043, + 3.651087760925293, + 3.566608428955078, + 3.696195125579834, + 3.5893759727478027, + 3.488834857940674, + 3.5512402057647705, + 3.532956600189209, + 3.6070311069488525, + 3.463216781616211, + 3.58147931098938, + 3.427995443344116, + 3.5444703102111816, + 3.537907361984253, + 3.376786708831787, + 3.4550883769989014, + 3.5039546489715576, + 3.5688014030456543, + 3.523670196533203, + 3.6208038330078125, + 3.516000986099243, + 3.5055339336395264, + 3.6307425498962402, + 3.5021607875823975, + 3.5332672595977783, + 3.5144600868225098, + 3.5133323669433594, + 3.5276949405670166, + 3.5391440391540527, + 3.491903781890869, + 3.428607702255249, + 3.491241931915283, + 3.64428448677063, + 3.510615348815918, + 3.661498785018921, + 3.5961999893188477, + 3.525993824005127, + 3.6233253479003906, + 3.4923782348632812, + 3.599966287612915, + 3.4727954864501953, + 3.434142827987671, + 3.6481971740722656, + 3.5498602390289307, + 3.4781761169433594, + 3.5332608222961426, + 3.337430953979492, + 3.5732686519622803, + 3.6127445697784424, + 3.5643863677978516, + 3.488826036453247, + 3.4283945560455322, + 3.5426907539367676, + 3.5316410064697266, + 3.5169429779052734, + 3.481266498565674, + 3.4309916496276855, + 3.446220874786377, + 3.513761281967163, + 3.6621811389923096, + 3.4264707565307617, + 3.6626555919647217, + 3.5715744495391846, + 3.5721964836120605, + 3.4780426025390625, + 3.5806539058685303, + 3.5961062908172607, + 3.6032941341400146, + 3.54577374458313, + 3.5105714797973633, + 3.588823080062866, + 3.4889931678771973, + 3.605034351348877, + 3.4824140071868896, + 3.5068600177764893, + 3.5166592597961426, + 3.5487608909606934, + 3.5656845569610596, + 3.416684627532959, + 3.5016767978668213, + 3.561370372772217, + 3.309856653213501, + 3.500694990158081, + 3.569624900817871, + 3.6860361099243164, + 3.3985321521759033, + 3.393435478210449, + 3.57133150100708, + 3.468966245651245, + 3.45312762260437, + 3.6048099994659424, + 3.3913469314575195, + 3.5273990631103516, + 3.6035330295562744, + 3.5475215911865234, + 3.4943764209747314, + 3.5180399417877197, + 3.6433353424072266, + 3.6485464572906494, + 3.5209527015686035, + 3.570690155029297, + 3.5629465579986572, + 3.465350389480591, + 3.5014662742614746, + 3.6729092597961426, + 3.2598254680633545, + 3.4834141731262207, + 3.480961799621582, + 3.553372859954834, + 3.600266456604004, + 3.318423271179199, + 3.5986392498016357, + 3.456359386444092, + 3.412593364715576, + 3.4062516689300537, + 3.494509220123291, + 3.4808032512664795, + 3.5113441944122314, + 3.5564279556274414, + 3.621683120727539, + 3.593409299850464, + 3.683541774749756, + 3.57108473777771, + 3.5724263191223145, + 3.379627227783203, + 3.4308269023895264, + 3.439018964767456, + 3.5357229709625244, + 3.553534507751465, + 3.62213397026062, + 3.476255178451538, + 3.5430350303649902, + 3.469322919845581, + 3.4920592308044434, + 3.5297203063964844, + 3.4751009941101074, + 3.487891674041748, + 3.5610954761505127, + 3.6309733390808105, + 3.5997142791748047, + 3.4949984550476074, + 3.481079578399658, + 3.4876136779785156, + 3.5419833660125732, + 3.415611743927002, + 3.646847724914551, + 3.585754156112671, + 3.6358165740966797, + 3.576122283935547, + 3.5067145824432373, + 3.3579139709472656, + 3.715747833251953, + 3.549863815307617, + 3.638221263885498, + 3.6054468154907227, + 3.5665807723999023, + 3.527003526687622, + 3.708829402923584, + 3.331552743911743, + 3.293576717376709, + 3.492276906967163, + 3.5220468044281006, + 3.5200231075286865, + 3.631563186645508, + 3.503305435180664, + 3.5694713592529297, + 3.4714574813842773, + 3.648686170578003, + 3.567528486251831, + 3.4816277027130127, + 3.558584690093994 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 145.19610151699837, + 92.3716582127769, + 71.20546758664132, + 61.43718125855477, + 55.67406624603518, + 50.90762366739235, + 48.19614582398203, + 45.75125000581415, + 43.65386024881173, + 41.94482397860581, + 40.48856137508182, + 38.86342631492585, + 37.731703680659685, + 36.920743567542566, + 35.990234403945855, + 35.31205295439143, + 34.89302743455298, + 34.63950858722547, + 34.48082158457548, + 34.47018268209238 + ], + "best_ppl": 34.47018268209238, + "train_time_s": 22547.67471933365 +} \ No newline at end of file diff --git a/063_sparse_dispatch/alldeltanet_1plus2of5_best.pt b/063_sparse_dispatch/alldeltanet_1plus2of5_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..b72a096f88f811467da49c49c505e6785f24ad52 --- /dev/null +++ b/063_sparse_dispatch/alldeltanet_1plus2of5_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8cad768f4ef64c3ca17a73392d267fad38c9825900e6787f1ded43b1ed0f680 +size 352653860 diff --git a/063_sparse_dispatch/alldeltanet_1plus2of5_results.json b/063_sparse_dispatch/alldeltanet_1plus2of5_results.json new file mode 100644 index 0000000000000000000000000000000000000000..ccb9252855a8dc5a5440373f749c5c1d47dc8e8e --- /dev/null +++ b/063_sparse_dispatch/alldeltanet_1plus2of5_results.json @@ -0,0 +1,429 @@ +{ + "label": "alldeltanet_1plus2of5", + "seed": 42, + "attention_mode": "sparse", + "shared_attn_type": "deltanet", + "routed_attn_type": "deltanet", + "top_k_blocks": 2, + "num_thin_blocks": 5, + "num_shared_blocks": 1, + "total_params": 88081216, + "steps": [ + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 3.191009283065796, + 3.328863739967346, + 3.415947914123535, + 3.2316702604293823, + 3.382833957672119, + 3.124008059501648, + 3.2027333974838257, + 3.2641021013259888, + 3.426743984222412, + 3.428125023841858, + 3.367152214050293, + 3.278505563735962, + 3.462581515312195, + 3.2879568338394165, + 3.1916502714157104, + 3.212552547454834, + 3.3022046089172363, + 3.1655482053756714, + 3.2660293579101562, + 3.2208882570266724, + 3.3859944343566895, + 3.3179856538772583, + 3.281696319580078, + 3.2022324800491333, + 3.3620890378952026, + 3.2239792346954346, + 3.216003894805908, + 3.2988672256469727, + 3.2124860286712646, + 3.316618800163269, + 3.3474607467651367, + 3.3079203367233276, + 3.3284828662872314, + 3.3102229833602905, + 3.322532296180725, + 3.28206467628479, + 3.315785765647888, + 3.312784194946289, + 3.335516095161438, + 3.359559416770935, + 3.3555291891098022, + 3.145307779312134, + 3.286845088005066, + 3.273396372795105, + 3.2949224710464478, + 3.3500232696533203, + 3.26678466796875, + 3.2964847087860107, + 3.361969828605652, + 3.257088541984558, + 3.3828060626983643, + 3.0366501808166504, + 3.247255563735962, + 3.2358914613723755, + 3.299847722053528, + 3.3237905502319336, + 3.3515087366104126, + 3.263141393661499, + 3.3021726608276367, + 3.323043704032898, + 3.233532667160034, + 3.314697742462158, + 3.3149819374084473, + 3.477187752723694, + 3.2339398860931396, + 3.2804248332977295, + 3.4242688417434692, + 3.2916712760925293, + 3.36748731136322, + 3.130074381828308, + 3.3141956329345703, + 3.3143627643585205, + 3.346090793609619, + 3.2531620264053345, + 3.2823760509490967, + 3.308228611946106, + 3.1535675525665283, + 3.260642409324646, + 3.2091482877731323, + 3.395833969116211, + 3.30565083026886, + 3.361516833305359, + 3.3984854221343994, + 3.242994546890259, + 3.298974633216858, + 3.266920804977417, + 3.3672393560409546, + 3.279137969017029, + 3.3808528184890747, + 3.331620931625366, + 3.227416157722473, + 3.3883129358291626, + 3.363086462020874, + 3.3585463762283325, + 3.318735361099243, + 3.253494620323181, + 3.331997871398926, + 3.30734384059906, + 3.245086431503296, + 3.328007459640503, + 3.337233781814575, + 3.2582403421401978, + 3.3222533464431763, + 3.210793972015381, + 3.2098299264907837, + 3.2652772665023804, + 3.3293458223342896, + 3.2253823280334473, + 3.3521727323532104, + 3.3217657804489136, + 3.1567418575286865, + 3.3199880123138428, + 3.32015597820282, + 3.311248540878296, + 3.2662731409072876, + 3.261398434638977, + 3.240964412689209, + 3.2887063026428223, + 3.3099323511123657, + 3.3993003368377686, + 3.395317554473877, + 3.2063169479370117, + 3.3246384859085083, + 3.2682676315307617, + 3.2480276823043823, + 3.320936918258667, + 3.373904585838318, + 3.3318439722061157, + 3.2063597440719604, + 3.3381389379501343, + 3.2591646909713745, + 3.2709685564041138, + 3.346736431121826, + 3.2756093740463257, + 3.275025248527527, + 3.1244016885757446, + 3.2753909826278687, + 3.3261804580688477, + 3.2849947214126587, + 3.302319645881653, + 3.1971189975738525, + 3.2646206617355347, + 3.3623400926589966, + 3.322376847267151, + 3.1334760189056396, + 3.28019917011261, + 3.2801499366760254, + 3.328059434890747, + 3.1762616634368896, + 3.310847282409668, + 3.2009763717651367, + 3.316709518432617, + 3.271262288093567, + 3.1959617137908936, + 3.4061949253082275, + 3.3021410703659058, + 3.327588438987732, + 3.4334702491760254, + 3.258755683898926, + 3.2574872970581055, + 3.11220645904541, + 3.3257079124450684, + 3.2220778465270996, + 3.2676130533218384, + 3.1585662364959717, + 3.189629316329956, + 3.3998475074768066, + 3.243172764778137, + 3.2481372356414795, + 3.248865246772766, + 3.3222776651382446, + 3.2382031679153442, + 3.256750226020813, + 3.358233332633972, + 3.4258028268814087, + 3.2402292490005493, + 3.3094077110290527, + 3.2269777059555054, + 3.280907392501831, + 3.2669597864151, + 3.2587358951568604, + 3.3182339668273926, + 3.3697680234909058, + 3.322334051132202, + 3.3249034881591797, + 3.194716215133667, + 3.2533066272735596, + 3.146541118621826, + 3.3767937421798706, + 3.2831578254699707, + 3.2430028915405273, + 3.3182345628738403, + 3.2724382877349854, + 3.280776858329773, + 3.307960629463196, + 3.2302061319351196, + 3.3128888607025146, + 3.1976616382598877, + 3.116324782371521, + 3.1689229011535645 + ], + "val_steps": [ + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 31.76339495631011, + 31.519708100944182, + 31.47150139549715, + 31.454479417602673 + ], + "best_ppl": 31.454479417602673, + "train_time_s": 4799.055575609207 +} \ No newline at end of file diff --git a/063_sparse_dispatch/alldeltanet_1plus2of5_s137_best.pt b/063_sparse_dispatch/alldeltanet_1plus2of5_s137_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..87dbf741af2ea0ea0f48a6112cdd8ccd91ae5daa --- /dev/null +++ b/063_sparse_dispatch/alldeltanet_1plus2of5_s137_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c165ddfe3f8eb4ffebc9240f8d5627adc801d1cd47d4095a8efbad48d5722c62 +size 352657745 diff --git a/063_sparse_dispatch/alldeltanet_1plus2of5_s137_results.json b/063_sparse_dispatch/alldeltanet_1plus2of5_s137_results.json new file mode 100644 index 0000000000000000000000000000000000000000..8a1848831a97355fb53128785286fd9283b620be --- /dev/null +++ b/063_sparse_dispatch/alldeltanet_1plus2of5_s137_results.json @@ -0,0 +1,2061 @@ +{ + "label": "alldeltanet_1plus2of5_s137", + "seed": 137, + "attention_mode": "sparse", + "shared_attn_type": "deltanet", + "routed_attn_type": "deltanet", + "top_k_blocks": 2, + "num_thin_blocks": 5, + "num_shared_blocks": 1, + "total_params": 88081216, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 18.998830795288086, + 15.902393817901611, + 8.135669708251953, + 7.324019193649292, + 7.115792989730835, + 7.064813613891602, + 6.938784837722778, + 6.715198516845703, + 6.613839387893677, + 6.542379856109619, + 6.410369157791138, + 6.3311638832092285, + 6.242779493331909, + 6.233959436416626, + 5.9862165451049805, + 6.01566481590271, + 5.841345548629761, + 5.80605411529541, + 5.83101487159729, + 5.712050676345825, + 5.702324628829956, + 5.543395042419434, + 5.561380863189697, + 5.570098638534546, + 5.389003753662109, + 5.383180856704712, + 5.284287929534912, + 5.341218948364258, + 5.250117063522339, + 5.189336538314819, + 5.2042176723480225, + 5.203203439712524, + 5.230615615844727, + 5.164457559585571, + 5.067585706710815, + 5.218745231628418, + 5.109286069869995, + 5.0779759883880615, + 4.985615968704224, + 4.979855537414551, + 4.874714612960815, + 4.979636192321777, + 4.886847019195557, + 4.9748852252960205, + 4.977687835693359, + 4.885562181472778, + 4.875044107437134, + 4.872605085372925, + 4.79611611366272, + 4.753376007080078, + 4.818694591522217, + 4.751328706741333, + 4.866663694381714, + 4.715213298797607, + 4.7770116329193115, + 4.650736093521118, + 4.748805284500122, + 4.712806940078735, + 4.703641653060913, + 4.801276206970215, + 4.5588366985321045, + 4.5834081172943115, + 4.512567043304443, + 4.502172946929932, + 4.5963380336761475, + 4.717924118041992, + 4.648649454116821, + 4.715246915817261, + 4.463584661483765, + 4.518133163452148, + 4.511329174041748, + 4.586673736572266, + 4.547655820846558, + 4.504882574081421, + 4.464513540267944, + 4.686469793319702, + 4.468414068222046, + 4.552989482879639, + 4.39307165145874, + 4.481320142745972, + 4.434136390686035, + 4.46428918838501, + 4.438634872436523, + 4.426852703094482, + 4.338860750198364, + 4.412120580673218, + 4.407027006149292, + 4.449138164520264, + 4.52044939994812, + 4.4191734790802, + 4.351147413253784, + 4.464174509048462, + 4.316612243652344, + 4.383476495742798, + 4.487093448638916, + 4.314574241638184, + 4.350242853164673, + 4.359611749649048, + 4.397294521331787, + 4.301522731781006, + 4.40493631362915, + 4.250093936920166, + 4.426027059555054, + 4.364105939865112, + 4.424943685531616, + 4.2064361572265625, + 4.248584032058716, + 4.321798086166382, + 4.321392297744751, + 4.208900451660156, + 4.225464582443237, + 4.304142951965332, + 4.156719446182251, + 4.09867537021637, + 4.27614426612854, + 4.352964162826538, + 4.028457045555115, + 4.314083814620972, + 4.219389200210571, + 4.24091911315918, + 4.222465753555298, + 4.285738468170166, + 4.2891387939453125, + 4.17737603187561, + 4.165592193603516, + 4.335840702056885, + 4.101284742355347, + 4.1548848152160645, + 4.19049596786499, + 4.083781003952026, + 4.112316370010376, + 4.153935432434082, + 4.139513254165649, + 4.1688947677612305, + 4.152753829956055, + 4.1667726039886475, + 4.224350214004517, + 4.198346853256226, + 4.131649017333984, + 4.139101982116699, + 4.214052200317383, + 4.091844320297241, + 4.103557109832764, + 4.289600133895874, + 4.212530851364136, + 4.151909589767456, + 4.149998426437378, + 4.155304908752441, + 4.113973617553711, + 4.031577706336975, + 4.1352012157440186, + 4.114476919174194, + 4.146446466445923, + 4.12694239616394, + 3.9836461544036865, + 4.146100282669067, + 4.095039367675781, + 4.135971307754517, + 4.1055296659469604, + 4.060878753662109, + 4.1591081619262695, + 4.049932241439819, + 4.1847511529922485, + 3.8756455183029175, + 4.0713067054748535, + 4.0493690967559814, + 4.174810886383057, + 4.018259167671204, + 4.042242527008057, + 4.0384215116500854, + 3.948588252067566, + 4.062283873558044, + 3.922053813934326, + 4.064862251281738, + 3.986765503883362, + 4.089112758636475, + 3.9434412717819214, + 4.059319734573364, + 4.177096128463745, + 4.1348676681518555, + 4.087693929672241, + 4.053100109100342, + 4.002689361572266, + 3.955136299133301, + 4.007729887962341, + 4.0467939376831055, + 4.05051589012146, + 4.003443479537964, + 3.9219433069229126, + 4.097153425216675, + 3.823634386062622, + 3.905206799507141, + 4.017326354980469, + 4.041743993759155, + 4.139641284942627, + 4.1574931144714355, + 3.929463267326355, + 4.0011231899261475, + 4.121377944946289, + 3.8872785568237305, + 4.055731773376465, + 3.9159021377563477, + 4.019008159637451, + 3.8959503173828125, + 3.9617968797683716, + 4.023158431053162, + 4.073160171508789, + 3.9099408388137817, + 4.0336527824401855, + 3.913222312927246, + 3.82514226436615, + 4.002493977546692, + 3.89626407623291, + 3.866461992263794, + 4.032644629478455, + 4.083093881607056, + 3.8960628509521484, + 3.8888808488845825, + 3.8866307735443115, + 4.006989598274231, + 3.967448115348816, + 3.9986644983291626, + 3.9677329063415527, + 3.973229765892029, + 3.980556845664978, + 3.9967238903045654, + 3.9732697010040283, + 3.972195625305176, + 3.9403018951416016, + 3.8477654457092285, + 3.860353112220764, + 4.023180723190308, + 3.9116647243499756, + 3.9754979610443115, + 3.8505417108535767, + 3.8584835529327393, + 3.8588685989379883, + 3.866559624671936, + 3.84561550617218, + 3.9838398694992065, + 3.9697065353393555, + 3.84236478805542, + 4.03624165058136, + 3.869160294532776, + 3.81305730342865, + 4.006822228431702, + 3.97523033618927, + 3.858354926109314, + 3.9034061431884766, + 3.8970268964767456, + 3.900955080986023, + 3.9816075563430786, + 3.799758791923523, + 3.7774200439453125, + 3.7437314987182617, + 3.821020007133484, + 3.8233903646469116, + 3.7579845190048218, + 3.8535592555999756, + 3.8858507871627808, + 3.832114338874817, + 3.8307234048843384, + 3.76480495929718, + 3.795080304145813, + 3.806410312652588, + 3.911821484565735, + 3.8622078895568848, + 3.8007956743240356, + 3.8660532236099243, + 3.852578043937683, + 3.7740538120269775, + 3.791263461112976, + 3.8572027683258057, + 3.859891891479492, + 3.886280655860901, + 3.883092761039734, + 3.709146022796631, + 3.7610872983932495, + 3.823358178138733, + 3.9073985815048218, + 3.651868462562561, + 3.8262230157852173, + 3.8142489194869995, + 3.6897839307785034, + 3.701169729232788, + 3.7030731439590454, + 3.848225951194763, + 3.8413671255111694, + 3.913910984992981, + 3.7865668535232544, + 3.7090139389038086, + 3.782328248023987, + 3.813398003578186, + 3.9115114212036133, + 3.6717092990875244, + 3.703883409500122, + 3.8283653259277344, + 3.8515931367874146, + 3.739478826522827, + 3.7533130645751953, + 3.805442690849304, + 3.8783682584762573, + 3.6724294424057007, + 3.8101296424865723, + 3.860557436943054, + 3.9089529514312744, + 3.717286705970764, + 3.814761161804199, + 3.6818259954452515, + 3.7831326723098755, + 3.918969988822937, + 3.84344744682312, + 3.7418699264526367, + 3.773910164833069, + 3.8140822649002075, + 3.8019096851348877, + 3.7954823970794678, + 3.8539979457855225, + 3.779366970062256, + 3.813989758491516, + 3.695577383041382, + 3.793730139732361, + 3.743077516555786, + 3.9378621578216553, + 3.7422913312911987, + 3.653027892112732, + 3.872283458709717, + 3.798112630844116, + 3.7767640352249146, + 3.7383464574813843, + 3.6804624795913696, + 3.7515379190444946, + 3.8153882026672363, + 3.8776211738586426, + 3.847944736480713, + 3.8039597272872925, + 3.8121931552886963, + 3.642123818397522, + 3.677533268928528, + 3.607884645462036, + 3.682467460632324, + 3.7020639181137085, + 3.803477168083191, + 3.8894057273864746, + 3.753091335296631, + 3.9497532844543457, + 3.7198896408081055, + 3.657642126083374, + 3.8444312810897827, + 3.798351764678955, + 3.7675222158432007, + 3.690181255340576, + 3.6701728105545044, + 3.776535391807556, + 3.8110504150390625, + 3.6328461170196533, + 3.7398868799209595, + 3.6572368144989014, + 3.753793239593506, + 3.7906423807144165, + 3.788413643836975, + 3.748659610748291, + 3.6396608352661133, + 3.8450639247894287, + 3.6626795530319214, + 3.8046975135803223, + 3.705195665359497, + 3.6619454622268677, + 3.606258988380432, + 3.80869197845459, + 3.759229302406311, + 3.6276663541793823, + 3.7133227586746216, + 3.8393174409866333, + 3.7589828968048096, + 3.804274797439575, + 3.779467463493347, + 3.749302387237549, + 3.6737449169158936, + 3.621023416519165, + 3.6493114233016968, + 3.5030773878097534, + 3.667122006416321, + 3.6237977743148804, + 3.6108150482177734, + 3.712266683578491, + 3.603633999824524, + 3.5620014667510986, + 3.5444889068603516, + 3.3859620094299316, + 3.577008843421936, + 3.632616400718689, + 3.515701651573181, + 3.6947470903396606, + 3.624481201171875, + 3.549383044242859, + 3.686112403869629, + 3.5946624279022217, + 3.5401724576950073, + 3.612841486930847, + 3.661042809486389, + 3.517255663871765, + 3.6761491298675537, + 3.672338128089905, + 3.6589304208755493, + 3.6063109636306763, + 3.736019015312195, + 3.575764298439026, + 3.5328363180160522, + 3.6549776792526245, + 3.654882788658142, + 3.5424089431762695, + 3.5229551792144775, + 3.587777853012085, + 3.6807717084884644, + 3.5462679862976074, + 3.6899009943008423, + 3.669807195663452, + 3.655733585357666, + 3.6646934747695923, + 3.6084760427474976, + 3.682489037513733, + 3.6864030361175537, + 3.61388623714447, + 3.4669289588928223, + 3.687342405319214, + 3.633343458175659, + 3.612823963165283, + 3.570981740951538, + 3.6611108779907227, + 3.6168404817581177, + 3.4550222158432007, + 3.7680652141571045, + 3.5577322244644165, + 3.6512033939361572, + 3.525450348854065, + 3.624173402786255, + 3.603409171104431, + 3.745275378227234, + 3.589213252067566, + 3.6044065952301025, + 3.7579989433288574, + 3.6552642583847046, + 3.536327600479126, + 3.5487130880355835, + 3.65525484085083, + 3.591292977333069, + 3.487553834915161, + 3.6089411973953247, + 3.6924889087677, + 3.5740519762039185, + 3.664687752723694, + 3.7954529523849487, + 3.579791784286499, + 3.6431127786636353, + 3.679202675819397, + 3.535749673843384, + 3.5547730922698975, + 3.676271438598633, + 3.5535776615142822, + 3.6805814504623413, + 3.5860432386398315, + 3.643311023712158, + 3.620045304298401, + 3.640904664993286, + 3.668442487716675, + 3.5678728818893433, + 3.6443320512771606, + 3.5468199253082275, + 3.6049857139587402, + 3.615994095802307, + 3.6010396480560303, + 3.665578603744507, + 3.5673599243164062, + 3.5209429264068604, + 3.575212240219116, + 3.68528950214386, + 3.6032750606536865, + 3.5936639308929443, + 3.624916195869446, + 3.547876715660095, + 3.6645947694778442, + 3.5710196495056152, + 3.6509076356887817, + 3.527952194213867, + 3.5389329195022583, + 3.6876262426376343, + 3.566441059112549, + 3.615090250968933, + 3.4466408491134644, + 3.567179560661316, + 3.6304162740707397, + 3.4789031744003296, + 3.600951075553894, + 3.6257681846618652, + 3.5756195783615112, + 3.6237220764160156, + 3.562214970588684, + 3.5858148336410522, + 3.536599636077881, + 3.6016359329223633, + 3.6153194904327393, + 3.642619252204895, + 3.5583548545837402, + 3.4989734888076782, + 3.6125071048736572, + 3.58587110042572, + 3.6077882051467896, + 3.410528540611267, + 3.442135214805603, + 3.5934964418411255, + 3.466873526573181, + 3.3380837440490723, + 3.5296844244003296, + 3.368082642555237, + 3.4803810119628906, + 3.4426097869873047, + 3.5136858224868774, + 3.3578667640686035, + 3.4190531969070435, + 3.497987389564514, + 3.4281054735183716, + 3.728802442550659, + 3.4807499647140503, + 3.505318284034729, + 3.396530508995056, + 3.4769870042800903, + 3.5014196634292603, + 3.446594476699829, + 3.4125099182128906, + 3.5743322372436523, + 3.498407483100891, + 3.461202025413513, + 3.5223448276519775, + 3.529191255569458, + 3.5765875577926636, + 3.438029646873474, + 3.427434802055359, + 3.60728657245636, + 3.415967583656311, + 3.540745973587036, + 3.4820445775985718, + 3.6148245334625244, + 3.4180608987808228, + 3.4525156021118164, + 3.613129138946533, + 3.4806121587753296, + 3.3911194801330566, + 3.482981562614441, + 3.5427536964416504, + 3.491361141204834, + 3.4627989530563354, + 3.531153082847595, + 3.4816982746124268, + 3.5198099613189697, + 3.5708805322647095, + 3.3534621000289917, + 3.5827447175979614, + 3.4488636255264282, + 3.5360634326934814, + 3.5793426036834717, + 3.531952738761902, + 3.546595573425293, + 3.4291568994522095, + 3.5934784412384033, + 3.505324959754944, + 3.4268972873687744, + 3.6329777240753174, + 3.5077606439590454, + 3.491404175758362, + 3.505107879638672, + 3.4934500455856323, + 3.5122203826904297, + 3.514194965362549, + 3.4848049879074097, + 3.5478222370147705, + 3.4696428775787354, + 3.6027528047561646, + 3.619149088859558, + 3.4361294507980347, + 3.5420628786087036, + 3.5073041915893555, + 3.443465828895569, + 3.454231858253479, + 3.56363308429718, + 3.4710850715637207, + 3.4259276390075684, + 3.5013989210128784, + 3.5731489658355713, + 3.740217089653015, + 3.6309854984283447, + 3.5998345613479614, + 3.408096671104431, + 3.3934890031814575, + 3.527074694633484, + 3.5016918182373047, + 3.4428783655166626, + 3.5190287828445435, + 3.5747302770614624, + 3.5799859762191772, + 3.5483521223068237, + 3.34212064743042, + 3.360977292060852, + 3.531426787376404, + 3.4036728143692017, + 3.548838257789612, + 3.484237313270569, + 3.4676707983016968, + 3.457768201828003, + 3.491345524787903, + 3.4098918437957764, + 3.5583548545837402, + 3.5740383863449097, + 3.420140504837036, + 3.4921882152557373, + 3.390168309211731, + 3.44605028629303, + 3.415993094444275, + 3.4745925664901733, + 3.478520631790161, + 3.588428020477295, + 3.4720436334609985, + 3.595000743865967, + 3.4886019229888916, + 3.404026985168457, + 3.421090245246887, + 3.4963423013687134, + 3.467875123023987, + 3.472922444343567, + 3.5878041982650757, + 3.534970998764038, + 3.474698066711426, + 3.3641624450683594, + 3.316718339920044, + 3.517506957054138, + 3.4768658876419067, + 3.3840389251708984, + 3.160529851913452, + 3.314793109893799, + 3.311537981033325, + 3.3102192878723145, + 3.3442219495773315, + 3.4112839698791504, + 3.192120313644409, + 3.444401502609253, + 3.2690012454986572, + 3.359909176826477, + 3.4075154066085815, + 3.4455008506774902, + 3.323975682258606, + 3.421507239341736, + 3.392412781715393, + 3.2887091636657715, + 3.55082905292511, + 3.4426130056381226, + 3.4450511932373047, + 3.5477157831192017, + 3.2614632844924927, + 3.4085659980773926, + 3.3366516828536987, + 3.192799210548401, + 3.4773645401000977, + 3.335031509399414, + 3.410867214202881, + 3.4550243616104126, + 3.3792723417282104, + 3.4016271829605103, + 3.5427252054214478, + 3.290869116783142, + 3.4162875413894653, + 3.3337368965148926, + 3.3074110746383667, + 3.4247820377349854, + 3.4438369274139404, + 3.3693461418151855, + 3.369502902030945, + 3.423094153404236, + 3.291845917701721, + 3.40224552154541, + 3.339176654815674, + 3.296999931335449, + 3.3929882049560547, + 3.3918853998184204, + 3.372926354408264, + 3.396432876586914, + 3.447497844696045, + 3.19490909576416, + 3.392892837524414, + 3.3146270513534546, + 3.3733900785446167, + 3.2880433797836304, + 3.465716004371643, + 3.4466546773910522, + 3.3214343786239624, + 3.3938976526260376, + 3.444453001022339, + 3.3173491954803467, + 3.2717233896255493, + 3.2585455179214478, + 3.5269713401794434, + 3.4620776176452637, + 3.3686437606811523, + 3.5054972171783447, + 3.3989003896713257, + 3.4058542251586914, + 3.3951841592788696, + 3.354646682739258, + 3.380639672279358, + 3.449304938316345, + 3.4132933616638184, + 3.3479177951812744, + 3.3609135150909424, + 3.433652400970459, + 3.3350909948349, + 3.35648512840271, + 3.374131917953491, + 3.423043727874756, + 3.420132637023926, + 3.4651626348495483, + 3.4749062061309814, + 3.3724987506866455, + 3.4204097986221313, + 3.361490845680237, + 3.436543583869934, + 3.3111331462860107, + 3.415375232696533, + 3.3422341346740723, + 3.221873641014099, + 3.370168924331665, + 3.4724011421203613, + 3.3375701904296875, + 3.4342103004455566, + 3.370037317276001, + 3.30540132522583, + 3.2690407037734985, + 3.334922432899475, + 3.3190507888793945, + 3.4149186611175537, + 3.2975540161132812, + 3.3733614683151245, + 3.375919818878174, + 3.4497960805892944, + 3.3663244247436523, + 3.453156590461731, + 3.2468005418777466, + 3.4669978618621826, + 3.310835838317871, + 3.386649489402771, + 3.298877716064453, + 3.4281643629074097, + 3.378668785095215, + 3.280370831489563, + 3.4741233587265015, + 3.3210500478744507, + 3.21358859539032, + 3.4655601978302, + 3.304251790046692, + 3.2748299837112427, + 3.2458198070526123, + 3.3402960300445557, + 3.303499937057495, + 3.3754454851150513, + 3.2160704135894775, + 3.1292636394500732, + 3.230453610420227, + 3.2507048845291138, + 3.419630289077759, + 3.3151681423187256, + 3.260809540748596, + 3.277487635612488, + 3.284781813621521, + 3.21893310546875, + 3.3400439023971558, + 3.311864495277405, + 3.392420768737793, + 3.3701019287109375, + 3.264276385307312, + 3.3898879289627075, + 3.368515968322754, + 3.263152241706848, + 3.3245391845703125, + 3.1779630184173584, + 3.334746837615967, + 3.2920159101486206, + 3.2097009420394897, + 3.2962288856506348, + 3.200961112976074, + 3.314592957496643, + 3.2399230003356934, + 3.445597767829895, + 3.4994553327560425, + 3.2688013315200806, + 3.349411964416504, + 3.4291832447052, + 3.225677728652954, + 3.3066598176956177, + 3.255475401878357, + 3.2427663803100586, + 3.334477663040161, + 3.3006222248077393, + 3.4067184925079346, + 3.4578715562820435, + 3.2700220346450806, + 3.243617057800293, + 3.279197573661804, + 3.283093810081482, + 3.2025914192199707, + 3.236183524131775, + 3.1864073276519775, + 3.3662421703338623, + 3.330789804458618, + 3.324222207069397, + 3.177940845489502, + 3.2722887992858887, + 3.3490861654281616, + 3.3144606351852417, + 3.3128275871276855, + 3.383549690246582, + 3.3404756784439087, + 3.14749014377594, + 3.2391306161880493, + 3.2919281721115112, + 3.3551939725875854, + 3.3464608192443848, + 3.3908331394195557, + 3.3758416175842285, + 3.3311800956726074, + 3.3007750511169434, + 3.365251660346985, + 3.2534343004226685, + 3.40329909324646, + 3.271202802658081, + 3.4148221015930176, + 3.302709221839905, + 3.2335857152938843, + 3.4193261861801147, + 3.2107889652252197, + 3.2835311889648438, + 3.1733148097991943, + 3.346213936805725, + 3.2460445165634155, + 3.3524104356765747, + 3.349741578102112, + 3.3509615659713745, + 3.4525562524795532, + 3.3268498182296753, + 3.2562891244888306, + 3.364031434059143, + 3.147802233695984, + 3.3598618507385254, + 3.2691240310668945, + 3.2650513648986816, + 3.2580044269561768, + 3.2672570943832397, + 3.2180153131484985, + 3.219757318496704, + 3.262286424636841, + 3.2135732173919678, + 3.3376165628433228, + 3.2698862552642822, + 3.2414110898971558, + 3.296245574951172, + 3.1102824211120605, + 3.324921488761902, + 3.4032135009765625, + 3.355102062225342, + 3.221597909927368, + 3.3697067499160767, + 3.3733794689178467, + 3.2654017210006714, + 3.316105604171753, + 3.346908211708069, + 3.271397352218628, + 3.2173025608062744, + 3.3281655311584473, + 3.3319461345672607, + 3.3792290687561035, + 3.301990032196045, + 3.2557483911514282, + 3.4133758544921875, + 3.2559350728988647, + 3.252989649772644, + 3.4374083280563354, + 3.243988037109375, + 3.382023572921753, + 3.1744292974472046, + 3.323523163795471, + 3.360379934310913, + 3.138803243637085, + 3.21491801738739, + 3.235256314277649, + 3.2845007181167603, + 3.12716281414032, + 3.225551962852478, + 3.115045189857483, + 3.289994478225708, + 3.401164174079895, + 3.2512223720550537, + 3.2155410051345825, + 3.2602802515029907, + 3.1151710748672485, + 3.3385804891586304, + 3.3752673864364624, + 3.2036375999450684, + 3.266953468322754, + 3.3013168573379517, + 3.1290531158447266, + 3.21755850315094, + 3.2785524129867554, + 3.168889880180359, + 3.3382441997528076, + 3.2355856895446777, + 3.1339327096939087, + 3.221031427383423, + 3.349816083908081, + 3.1372504234313965, + 3.215782403945923, + 3.229736089706421, + 3.249935030937195, + 3.3534621000289917, + 3.2819842100143433, + 3.326842784881592, + 3.1545780897140503, + 3.1948453187942505, + 3.265215754508972, + 3.2713218927383423, + 3.3039532899856567, + 3.2256386280059814, + 3.2074464559555054, + 3.3838146924972534, + 3.2510969638824463, + 3.3432141542434692, + 3.3325183391571045, + 3.308952569961548, + 3.105363965034485, + 3.270874500274658, + 3.3109946250915527, + 3.2289061546325684, + 3.2020472288131714, + 3.2007789611816406, + 3.4425854682922363, + 3.0917961597442627, + 3.3621513843536377, + 3.23023784160614, + 3.3827412128448486, + 3.2578223943710327, + 3.17236590385437, + 3.1963276863098145, + 3.266272783279419, + 3.293743371963501, + 3.2286343574523926, + 3.250296711921692, + 3.3083410263061523, + 3.317312479019165, + 3.144782543182373, + 3.258291244506836, + 3.21946120262146, + 3.2407562732696533, + 3.174214720726013, + 3.306045651435852, + 3.1964091062545776, + 3.2834869623184204, + 3.3207398653030396, + 3.2084649801254272, + 3.1850613355636597, + 3.079409599304199, + 3.310445547103882, + 3.324708342552185, + 3.3314799070358276, + 3.260048031806946, + 3.337706208229065, + 3.4071842432022095, + 3.375268578529358, + 3.2727071046829224, + 3.1365272998809814, + 3.2788045406341553, + 3.3435782194137573, + 3.199700355529785, + 3.2216211557388306, + 3.158092498779297, + 3.311465263366699, + 3.2462629079818726, + 3.2669178247451782, + 3.199223518371582, + 3.2521859407424927, + 3.197777271270752, + 3.163889765739441, + 3.241421937942505, + 3.261011838912964, + 3.253145456314087, + 3.2845537662506104, + 3.26407253742218, + 3.4115915298461914, + 3.232579231262207, + 3.2489051818847656, + 3.202889561653137, + 3.3397955894470215, + 3.3210495710372925, + 3.30639386177063, + 3.220300316810608 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 112.74004630121391, + 70.18333745133867, + 56.52099195412605, + 50.12498752351723, + 45.94783726815109, + 42.97035930153135, + 40.872524944137425, + 39.03065753107409, + 37.475176210044644, + 36.08006750788051, + 35.121876372481545, + 34.02011136569953, + 33.30483801540374, + 32.635665328418625, + 32.05696368568822, + 31.77630278361898, + 31.47081359220405, + 31.318975608938814, + 31.235075853291086, + 31.209568086658646 + ], + "best_ppl": 31.209568086658646, + "train_time_s": 24037.091883659363 +} \ No newline at end of file diff --git a/063_sparse_dispatch/balanced_deltanet_1plus2of14_best.pt b/063_sparse_dispatch/balanced_deltanet_1plus2of14_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..fab34a6c88a30bd78c9d14fc1003eb9e2324511f --- /dev/null +++ b/063_sparse_dispatch/balanced_deltanet_1plus2of14_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66960cd5d4e7b849a7a453be0e33522a4df1ef4e87c942cdba146ed8dbc35291 +size 793625734 diff --git a/063_sparse_dispatch/balanced_deltanet_1plus2of14_results.json b/063_sparse_dispatch/balanced_deltanet_1plus2of14_results.json new file mode 100644 index 0000000000000000000000000000000000000000..54baba89577377663f09b8617e607ac5966c404b --- /dev/null +++ b/063_sparse_dispatch/balanced_deltanet_1plus2of14_results.json @@ -0,0 +1,2056 @@ +{ + "label": "balanced_deltanet_1plus2of14", + "seed": 42, + "balanced_shared_weight": true, + "total_params": 198162304, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.007029056549072, + 15.744546890258789, + 8.096514701843262, + 7.200287103652954, + 7.07413375377655, + 6.95750105381012, + 6.825071215629578, + 6.728790760040283, + 6.5309847593307495, + 6.566293835639954, + 6.426525592803955, + 6.300367116928101, + 6.271498084068298, + 6.129087686538696, + 6.023912310600281, + 5.960632562637329, + 5.983098149299622, + 5.774940490722656, + 5.810048818588257, + 5.765506982803345, + 5.7465338706970215, + 5.6365591287612915, + 5.48430609703064, + 5.516469359397888, + 5.436890363693237, + 5.6016035079956055, + 5.3401395082473755, + 5.399532675743103, + 5.344425320625305, + 5.239517092704773, + 5.399593830108643, + 5.1822367906570435, + 5.218731880187988, + 5.250670075416565, + 5.094865083694458, + 5.1221684217453, + 5.006512403488159, + 5.095776200294495, + 5.2094831466674805, + 5.100244045257568, + 5.106192708015442, + 5.020656228065491, + 5.092447876930237, + 4.933869242668152, + 5.003573894500732, + 4.934317350387573, + 4.883657336235046, + 4.966542363166809, + 5.070444703102112, + 4.927702307701111, + 4.761173963546753, + 4.865741848945618, + 4.95940625667572, + 4.828409194946289, + 4.84242582321167, + 4.824490904808044, + 4.751731991767883, + 4.836623311042786, + 4.8206707239151, + 4.699794292449951, + 4.690372705459595, + 4.86928117275238, + 4.7513569593429565, + 4.706098198890686, + 4.740211844444275, + 4.764340400695801, + 4.720950484275818, + 4.635097026824951, + 4.765880346298218, + 4.695482492446899, + 4.781808972358704, + 4.645086050033569, + 4.572265267372131, + 4.511196970939636, + 4.607608795166016, + 4.674662232398987, + 4.549403190612793, + 4.616653800010681, + 4.522834062576294, + 4.603036284446716, + 4.685214638710022, + 4.494112133979797, + 4.639693856239319, + 4.550933361053467, + 4.567022204399109, + 4.616014361381531, + 4.55056095123291, + 4.432691812515259, + 4.772720217704773, + 4.542523384094238, + 4.54993200302124, + 4.483271479606628, + 4.562663555145264, + 4.523924469947815, + 4.494808197021484, + 4.547679662704468, + 4.489535093307495, + 4.459555149078369, + 4.555750370025635, + 4.453930854797363, + 4.447753548622131, + 4.465154409408569, + 4.466107368469238, + 4.377436280250549, + 4.403732419013977, + 4.462416410446167, + 4.4501917362213135, + 4.371693849563599, + 4.246965765953064, + 4.575590014457703, + 4.396987676620483, + 4.354585647583008, + 4.380972266197205, + 4.271614730358124, + 4.33528995513916, + 4.3450716733932495, + 4.422542452812195, + 4.4045504331588745, + 4.366891264915466, + 4.415210485458374, + 4.298605024814606, + 4.2014018297195435, + 4.410493850708008, + 4.41191291809082, + 4.404997229576111, + 4.340571761131287, + 4.178236603736877, + 4.213701367378235, + 4.362919449806213, + 4.33405065536499, + 4.107849597930908, + 4.1695069670677185, + 4.225027561187744, + 4.274167776107788, + 4.184337854385376, + 4.2325708866119385, + 4.246656656265259, + 4.209238648414612, + 4.296964406967163, + 4.229635238647461, + 4.322724103927612, + 4.309017896652222, + 4.191133379936218, + 4.23939061164856, + 4.197058796882629, + 4.321786642074585, + 4.309346675872803, + 4.196440637111664, + 4.150241732597351, + 4.267251193523407, + 4.247180700302124, + 4.288204312324524, + 4.052735328674316, + 4.267022252082825, + 4.351259112358093, + 4.199983835220337, + 4.192804634571075, + 4.16526198387146, + 4.251654744148254, + 4.15409255027771, + 4.2150068283081055, + 4.081317603588104, + 4.1541759967803955, + 4.101630091667175, + 4.19915509223938, + 4.085056006908417, + 4.107523798942566, + 4.232376456260681, + 4.1363712549209595, + 4.290483713150024, + 4.075610995292664, + 4.152762293815613, + 4.109402179718018, + 4.108001351356506, + 4.099833428859711, + 4.136088967323303, + 4.08637809753418, + 4.026861488819122, + 4.140812993049622, + 4.232643246650696, + 4.008289873600006, + 4.071905553340912, + 4.090707540512085, + 4.102768778800964, + 4.191878914833069, + 4.172437787055969, + 4.138423323631287, + 4.179016590118408, + 4.039000451564789, + 4.137966811656952, + 4.072010636329651, + 4.161252379417419, + 4.1526142954826355, + 4.138616740703583, + 4.088387310504913, + 4.043177425861359, + 4.218201160430908, + 4.174187064170837, + 4.071493923664093, + 4.048209607601166, + 4.1007901430130005, + 4.1442039012908936, + 4.263700246810913, + 4.098412752151489, + 4.027653634548187, + 4.128334641456604, + 4.053989946842194, + 3.9730498790740967, + 4.085931479930878, + 4.038202226161957, + 4.125645041465759, + 4.066151976585388, + 4.096789598464966, + 4.187631249427795, + 4.094697594642639, + 4.10444700717926, + 4.075880467891693, + 3.9534595012664795, + 4.11457622051239, + 3.9766867756843567, + 4.0241557359695435, + 4.045379161834717, + 4.075027704238892, + 4.003383100032806, + 4.158282279968262, + 4.005368590354919, + 3.941079258918762, + 4.098685026168823, + 3.9974223375320435, + 3.969735026359558, + 4.022302985191345, + 4.000357985496521, + 4.082313716411591, + 4.099744260311127, + 4.102176904678345, + 4.068508148193359, + 4.124389588832855, + 4.038383483886719, + 4.040302217006683, + 3.947681248188019, + 4.116657376289368, + 3.956985652446747, + 4.026710689067841, + 4.021183609962463, + 4.0896036028862, + 4.046030938625336, + 3.9023674726486206, + 4.053390741348267, + 3.985423982143402, + 4.1022579073905945, + 4.113726496696472, + 3.912636876106262, + 3.940497636795044, + 3.9439421892166138, + 3.9760903120040894, + 3.8901697397232056, + 3.873382031917572, + 3.869977116584778, + 3.824986755847931, + 3.935152053833008, + 3.981752336025238, + 3.892707586288452, + 3.937479317188263, + 3.7739111185073853, + 3.9016400575637817, + 3.8394384384155273, + 3.828939139842987, + 3.884597957134247, + 3.9015448093414307, + 3.8136889338493347, + 3.8826528787612915, + 3.9895623922348022, + 3.8207748532295227, + 3.8640982508659363, + 3.95294988155365, + 3.8784157633781433, + 3.84086537361145, + 3.8452189564704895, + 3.8796722292900085, + 3.8614587783813477, + 3.9107221961021423, + 3.829412817955017, + 3.9026519060134888, + 3.6896952986717224, + 3.8207370042800903, + 3.8825061321258545, + 3.896413743495941, + 3.894048750400543, + 3.8271952271461487, + 3.866149365901947, + 3.995390772819519, + 3.941170632839203, + 4.039530456066132, + 3.7929213643074036, + 3.762755274772644, + 3.8592609167099, + 3.901469647884369, + 3.8494536876678467, + 3.960483968257904, + 3.931572735309601, + 3.8239455819129944, + 3.8550915718078613, + 3.809568226337433, + 3.918505370616913, + 3.887548089027405, + 3.6777948141098022, + 3.9376019835472107, + 3.959823429584503, + 3.944943130016327, + 3.7583335638046265, + 3.709763288497925, + 3.8516541123390198, + 3.9864314198493958, + 3.8521979451179504, + 3.7318177223205566, + 3.8888303637504578, + 3.928228437900543, + 3.934461295604706, + 3.848988175392151, + 3.807724893093109, + 3.7490667700767517, + 3.7937917709350586, + 3.884846568107605, + 3.836880385875702, + 3.8000001311302185, + 3.785006046295166, + 3.8115394115448, + 3.9892078638076782, + 3.796084403991699, + 3.7534388303756714, + 3.8985610604286194, + 3.6651576161384583, + 3.859231412410736, + 3.725500702857971, + 3.820160686969757, + 3.745773494243622, + 3.7781019806861877, + 3.7360565066337585, + 3.8166192173957825, + 4.006020367145538, + 3.8464730978012085, + 3.8950098156929016, + 3.8575876355171204, + 3.8538021445274353, + 3.825095474720001, + 3.733938753604889, + 3.7413015365600586, + 3.8945114612579346, + 3.869007408618927, + 3.6993778944015503, + 3.9148892164230347, + 3.970922350883484, + 3.842940330505371, + 3.732559859752655, + 3.852678596973419, + 3.893264412879944, + 3.8102806210517883, + 3.778937816619873, + 3.889963150024414, + 3.9158109426498413, + 3.63587087392807, + 3.7370256185531616, + 3.64510440826416, + 3.8409169912338257, + 3.841382682323456, + 3.8307185769081116, + 4.001526296138763, + 3.741424560546875, + 3.8941405415534973, + 3.7257795929908752, + 3.844810962677002, + 3.6838313341140747, + 3.810392916202545, + 3.7998202443122864, + 3.79592764377594, + 3.8024572134017944, + 3.739764451980591, + 3.8055062890052795, + 3.8543132543563843, + 3.642619013786316, + 3.5178913474082947, + 3.666505992412567, + 3.6771510243415833, + 3.533648729324341, + 3.787630081176758, + 3.6617390513420105, + 3.8582887649536133, + 3.7139276266098022, + 3.7798765301704407, + 3.7214419841766357, + 3.7267187237739563, + 3.673634350299835, + 3.558131456375122, + 3.8593090772628784, + 3.7210397124290466, + 3.7617361545562744, + 3.6193899512290955, + 3.6121144890785217, + 3.723844051361084, + 3.637888252735138, + 3.6057775020599365, + 3.679061710834503, + 3.7307745218276978, + 3.6424118876457214, + 3.760043740272522, + 3.5426050424575806, + 3.697793960571289, + 3.7605096101760864, + 3.7284671664237976, + 3.6107804775238037, + 3.70669949054718, + 3.7617433071136475, + 3.6118592023849487, + 3.5672801733016968, + 3.6745582222938538, + 3.6514071226119995, + 3.663335859775543, + 3.6108641624450684, + 3.661179780960083, + 3.6779531836509705, + 3.610503077507019, + 3.6853919625282288, + 3.7230613827705383, + 3.703774333000183, + 3.5854074954986572, + 3.663912832736969, + 3.8821557760238647, + 3.6986547112464905, + 3.7897230982780457, + 3.741970121860504, + 3.5928168296813965, + 3.757745623588562, + 3.6002541184425354, + 3.7940359115600586, + 3.576808452606201, + 3.680070698261261, + 3.6546184420585632, + 3.679753065109253, + 3.726378321647644, + 3.6292850375175476, + 3.7007378935813904, + 3.73223477602005, + 3.6332093477249146, + 3.720688760280609, + 3.6671735048294067, + 3.61959308385849, + 3.794495403766632, + 3.7525919675827026, + 3.7644433975219727, + 3.595315635204315, + 3.6116411685943604, + 3.600514590740204, + 3.7033591866493225, + 3.7219821214675903, + 3.54790723323822, + 3.5841710567474365, + 3.6556584239006042, + 3.7136719822883606, + 3.747722327709198, + 3.681504189968109, + 3.594109892845154, + 3.556330442428589, + 3.546804368495941, + 3.598013460636139, + 3.6279892921447754, + 3.740383505821228, + 3.5689139366149902, + 3.62203449010849, + 3.5915489196777344, + 3.5592015981674194, + 3.6982422471046448, + 3.7518842816352844, + 3.6372109055519104, + 3.6778676509857178, + 3.6928138732910156, + 3.655611574649811, + 3.6661707758903503, + 3.445195198059082, + 3.6120129227638245, + 3.6502230763435364, + 3.6693835854530334, + 3.6712623238563538, + 3.5089685320854187, + 3.628869593143463, + 3.638764977455139, + 3.652334213256836, + 3.709218382835388, + 3.749295175075531, + 3.701918601989746, + 3.629084289073944, + 3.5205920934677124, + 3.6980855464935303, + 3.645692765712738, + 3.548923671245575, + 3.4961034059524536, + 3.694467008113861, + 3.6566529870033264, + 3.704724907875061, + 3.594532608985901, + 3.6875813603401184, + 3.61167311668396, + 3.618522882461548, + 3.6705610156059265, + 3.62267404794693, + 3.6387943029403687, + 3.5813379883766174, + 3.4476893544197083, + 3.5449549555778503, + 3.394130766391754, + 3.5161473751068115, + 3.4708633422851562, + 3.525272011756897, + 3.5334867238998413, + 3.543613612651825, + 3.522661566734314, + 3.505491852760315, + 3.5864442586898804, + 3.458276927471161, + 3.406184196472168, + 3.4483708143234253, + 3.607776701450348, + 3.5533942580223083, + 3.4406712651252747, + 3.574903666973114, + 3.5208892822265625, + 3.4649507999420166, + 3.5058361887931824, + 3.555130183696747, + 3.5689972639083862, + 3.43949818611145, + 3.55232435464859, + 3.6643142700195312, + 3.473298490047455, + 3.5304969549179077, + 3.486370027065277, + 3.516752600669861, + 3.4204118847846985, + 3.437906265258789, + 3.5314783453941345, + 3.587952673435211, + 3.476911187171936, + 3.509238839149475, + 3.4970558881759644, + 3.501465678215027, + 3.5572405457496643, + 3.5167710185050964, + 3.4871893525123596, + 3.5572299361228943, + 3.5371997952461243, + 3.5466983914375305, + 3.493237018585205, + 3.5247625708580017, + 3.5753708481788635, + 3.531292974948883, + 3.508563458919525, + 3.426444172859192, + 3.530267059803009, + 3.5337369441986084, + 3.52803498506546, + 3.443161904811859, + 3.621330201625824, + 3.5283252000808716, + 3.337467074394226, + 3.3822349309921265, + 3.61822247505188, + 3.4067978262901306, + 3.4032986164093018, + 3.4984483122825623, + 3.599613904953003, + 3.428886830806732, + 3.4442883133888245, + 3.4766953587532043, + 3.441937565803528, + 3.5197020173072815, + 3.486977279186249, + 3.379583775997162, + 3.454721689224243, + 3.5241026878356934, + 3.591631591320038, + 3.503621459007263, + 3.6160948276519775, + 3.6443010568618774, + 3.365467369556427, + 3.60132098197937, + 3.559010922908783, + 3.5810182094573975, + 3.6416704058647156, + 3.5315659046173096, + 3.4592657685279846, + 3.4143880009651184, + 3.4435515999794006, + 3.33810818195343, + 3.617700219154358, + 3.3744266033172607, + 3.3490843176841736, + 3.4662532806396484, + 3.571637451648712, + 3.4692800045013428, + 3.5665860176086426, + 3.3947179913520813, + 3.5788954496383667, + 3.473336338996887, + 3.7005592584609985, + 3.546187937259674, + 3.5485023260116577, + 3.549393057823181, + 3.6411470770835876, + 3.3872397541999817, + 3.503234386444092, + 3.5779020190238953, + 3.4777398705482483, + 3.516692817211151, + 3.5366405248641968, + 3.4718490839004517, + 3.605877637863159, + 3.5471079349517822, + 3.5887807607650757, + 3.409898340702057, + 3.425763785839081, + 3.4755314588546753, + 3.2483492493629456, + 3.393581211566925, + 3.4814165830612183, + 3.313901364803314, + 3.615340292453766, + 3.6127726435661316, + 3.5721418261528015, + 3.4383354783058167, + 3.487226366996765, + 3.462518095970154, + 3.423736035823822, + 3.5637263655662537, + 3.4054301977157593, + 3.320780396461487, + 3.395626187324524, + 3.3885459303855896, + 3.3107191920280457, + 3.3878783583641052, + 3.5373337864875793, + 3.2462053894996643, + 3.314415395259857, + 3.236880838871002, + 3.3332250714302063, + 3.3105971217155457, + 3.3718199729919434, + 3.3510196208953857, + 3.405950367450714, + 3.2885642051696777, + 3.373809516429901, + 3.4906917810440063, + 3.4037766456604004, + 3.40459942817688, + 3.5330355167388916, + 3.385464668273926, + 3.3481326699256897, + 3.3348132371902466, + 3.4067582488059998, + 3.3592904210090637, + 3.333524525165558, + 3.380660057067871, + 3.3091734647750854, + 3.2748823165893555, + 3.4415799975395203, + 3.2515543699264526, + 3.3411168456077576, + 3.4248584508895874, + 3.412724494934082, + 3.2791712880134583, + 3.3160974979400635, + 3.4300490617752075, + 3.3628775477409363, + 3.329318642616272, + 3.402980387210846, + 3.4427759051322937, + 3.4423593282699585, + 3.3244038820266724, + 3.358207404613495, + 3.327985942363739, + 3.383539140224457, + 3.4405596256256104, + 3.4258177876472473, + 3.4219416975975037, + 3.381610333919525, + 3.472897231578827, + 3.2950326204299927, + 3.344038724899292, + 3.506730794906616, + 3.424596071243286, + 3.4430994987487793, + 3.3362362384796143, + 3.4176653027534485, + 3.386411130428314, + 3.3794225454330444, + 3.266981363296509, + 3.4120656847953796, + 3.4131357073783875, + 3.4244343638420105, + 3.2746670842170715, + 3.260496735572815, + 3.34052312374115, + 3.373394787311554, + 3.3610286712646484, + 3.303013324737549, + 3.2739111185073853, + 3.390907049179077, + 3.468145728111267, + 3.4973710775375366, + 3.423750340938568, + 3.3275778889656067, + 3.319582462310791, + 3.485335350036621, + 3.3511610627174377, + 3.428451359272003, + 3.393562078475952, + 3.365213692188263, + 3.216444730758667, + 3.4045000076293945, + 3.351387679576874, + 3.377753436565399, + 3.3692750334739685, + 3.3818345069885254, + 3.394642651081085, + 3.4213207364082336, + 3.349533438682556, + 3.473477840423584, + 3.5229434967041016, + 3.4397106766700745, + 3.367410659790039, + 3.2582120299339294, + 3.382024884223938, + 3.382887840270996, + 3.442626655101776, + 3.323520302772522, + 3.3920018672943115, + 3.449468195438385, + 3.3593021631240845, + 3.379427969455719, + 3.3620177507400513, + 3.3228018283843994, + 3.318937599658966, + 3.4515005946159363, + 3.4791728854179382, + 3.368217706680298, + 3.486562967300415, + 3.43165522813797, + 3.3200100660324097, + 3.4063815474510193, + 3.376468002796173, + 3.492992103099823, + 3.2689907550811768, + 3.300323784351349, + 3.4402114748954773, + 3.288609266281128, + 3.3388859629631042, + 3.4603845477104187, + 3.250612497329712, + 3.44053715467453, + 3.4345330595970154, + 3.3658493161201477, + 3.2147616147994995, + 3.353416860103607, + 3.226531684398651, + 3.376012086868286, + 3.4117690920829773, + 3.2856377363204956, + 3.442535102367401, + 3.246853828430176, + 3.2096789479255676, + 3.2027925848960876, + 3.276724100112915, + 3.2478445172309875, + 3.171311140060425, + 3.3369319438934326, + 3.228361129760742, + 3.3939632177352905, + 3.2980490922927856, + 3.3016237616539, + 3.3138691186904907, + 3.2456050515174866, + 3.357860803604126, + 3.4077190160751343, + 3.3320887684822083, + 3.2914960980415344, + 3.3239960074424744, + 3.214726984500885, + 3.2777644395828247, + 3.184890568256378, + 3.2073208689689636, + 3.246901512145996, + 3.2918217182159424, + 3.311216413974762, + 3.2850202322006226, + 3.3579817414283752, + 3.259308874607086, + 3.289494574069977, + 3.251686155796051, + 3.3702220916748047, + 3.323692739009857, + 3.2970107793807983, + 3.3560274243354797, + 3.096928119659424, + 3.2632140517234802, + 3.2637699842453003, + 3.354560375213623, + 3.2564483284950256, + 3.2692389488220215, + 3.437110483646393, + 3.2851531505584717, + 3.382664978504181, + 3.254937708377838, + 3.4005895853042603, + 3.323284149169922, + 3.298140823841095, + 3.3006730675697327, + 3.318230628967285, + 3.3902058005332947, + 3.2050209641456604, + 3.3346303701400757, + 3.253235340118408, + 3.2496114373207092, + 3.2570987343788147, + 3.315154016017914, + 3.3577839732170105, + 3.4092636704444885, + 3.190271019935608, + 3.394024074077606, + 3.2770082354545593, + 3.3311198353767395, + 3.254634201526642, + 3.309836983680725, + 3.24170184135437, + 3.2993884086608887, + 3.273933529853821, + 3.3305102586746216, + 3.3516713976860046, + 3.336953043937683, + 3.2708035111427307, + 3.344735026359558, + 3.2716383934020996, + 3.2988362312316895, + 3.3469311594963074, + 3.2809232473373413, + 3.1368669271469116, + 3.2939584851264954, + 3.3009403347969055, + 3.424804389476776, + 3.285794258117676, + 3.189984679222107, + 3.210780918598175, + 3.319199562072754, + 3.374878406524658, + 3.492256462574005, + 3.4707473516464233, + 3.281998097896576, + 3.2820500135421753, + 3.324783444404602, + 3.1488601565361023, + 3.138077974319458, + 3.1074076890945435, + 3.358904242515564, + 3.2096647024154663, + 3.4350916147232056, + 3.2309131622314453, + 3.2373359203338623, + 3.3022085428237915, + 3.4377193450927734, + 3.3872653245925903, + 3.4030967354774475, + 3.360609710216522, + 3.310350000858307, + 3.397625148296356, + 3.2593401074409485, + 3.1725522875785828, + 3.2998780608177185, + 3.2437320947647095, + 3.3542473912239075, + 3.3646947741508484, + 3.3371899724006653, + 3.3667120933532715, + 3.322721242904663, + 3.328936278820038, + 3.292581796646118, + 3.318600654602051, + 3.346566915512085, + 3.2666571140289307, + 3.2178702354431152, + 3.260078489780426, + 3.237205684185028, + 3.234681487083435, + 3.2550365328788757, + 3.3643633127212524, + 3.22128564119339, + 3.277369260787964, + 3.276298999786377, + 3.2483805418014526, + 3.3858906030654907, + 3.2534974813461304, + 3.205005645751953, + 3.140721082687378, + 3.3546706438064575, + 3.176580548286438, + 3.335087239742279, + 3.108737528324127, + 3.144282341003418, + 3.2223201394081116, + 3.1865015029907227, + 3.246525526046753, + 3.3272278904914856, + 3.2829355597496033, + 3.2892428636550903, + 3.3326780796051025, + 3.2529075741767883, + 3.2141535878181458, + 3.143064558506012, + 3.353688657283783, + 3.1298601031303406, + 3.1487892270088196, + 3.3221879601478577, + 3.2441176772117615, + 3.1429904103279114, + 3.203442394733429, + 3.2025593519210815, + 3.2302772402763367, + 3.2522717714309692, + 3.215831220149994, + 3.4004764556884766, + 3.241536855697632, + 3.131673514842987, + 3.124953269958496, + 3.3281795978546143, + 3.2628523111343384, + 3.144668400287628, + 3.0588754415512085, + 3.237188220024109, + 3.2635130882263184, + 3.145736336708069, + 3.2131412625312805, + 3.1861178874969482, + 3.262065827846527, + 3.09612375497818, + 3.291422665119171, + 3.2147578597068787, + 3.2617260217666626, + 3.262902617454529, + 3.168292760848999, + 3.238015115261078, + 3.3127917647361755, + 3.318225383758545, + 3.1634413599967957, + 3.2688133120536804, + 3.219923675060272, + 3.2421478033065796, + 3.31657475233078, + 3.1322668194770813, + 3.268855333328247, + 3.2221221327781677, + 3.3262423872947693, + 3.190935969352722, + 3.161631405353546, + 3.124063491821289, + 3.2193883657455444, + 3.0401601791381836, + 3.1131868958473206, + 3.2360206842422485, + 3.1788150668144226, + 3.102247714996338, + 3.318631410598755, + 3.1782225370407104, + 3.273117780685425, + 3.3134445548057556, + 3.2784950733184814, + 3.3556042313575745, + 3.2482816576957703, + 3.2810553312301636, + 3.251431941986084, + 3.171278238296509, + 3.1784600019454956, + 3.362046241760254, + 3.305746555328369, + 3.2928380966186523, + 3.2933608889579773, + 3.1490713357925415, + 3.368197500705719, + 3.1445156931877136, + 3.1731978058815002, + 3.1377981305122375, + 3.2057541608810425, + 3.140432298183441, + 3.330853223800659, + 3.1850080490112305, + 3.1904314160346985, + 3.2269003987312317, + 3.3348873257637024, + 3.218994140625, + 3.277595639228821, + 3.279152512550354, + 3.1776480078697205, + 3.233492076396942, + 3.140605390071869, + 3.1646828651428223 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 119.44825357322371, + 76.49451383643928, + 61.53399442592188, + 53.9702523822168, + 48.18650430001018, + 45.32106690267954, + 42.196802220859524, + 40.19377352594166, + 38.15843378355969, + 36.5756899955108, + 35.79859116760209, + 34.25852510705239, + 33.538988955067595, + 32.76574708906353, + 31.941866223351678, + 31.77778472936944, + 31.33854080596222, + 31.223129286236617, + 31.142865473650133, + 31.12561654351393 + ], + "best_ppl": 31.12561654351393, + "train_time_s": 96350.71798038483 +} \ No newline at end of file diff --git a/063_sparse_dispatch/balanced_deltanet_1plus2of5_best.pt b/063_sparse_dispatch/balanced_deltanet_1plus2of5_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f3c207d2abdc28f4f6a4810e0dedd723cc3b7af --- /dev/null +++ b/063_sparse_dispatch/balanced_deltanet_1plus2of5_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ef9a8651609ab13346cd9fffa414b8b886d0b20ff6826761a9bf733df3077ad +size 350409602 diff --git a/063_sparse_dispatch/balanced_deltanet_1plus2of5_results.json b/063_sparse_dispatch/balanced_deltanet_1plus2of5_results.json new file mode 100644 index 0000000000000000000000000000000000000000..e9fe7ff728735629ded4e514749dd2c703ef45ec --- /dev/null +++ b/063_sparse_dispatch/balanced_deltanet_1plus2of5_results.json @@ -0,0 +1,2056 @@ +{ + "label": "balanced_deltanet_1plus2of5", + "seed": 42, + "balanced_shared_weight": true, + "total_params": 87523584, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.06377410888672, + 16.01507806777954, + 8.050448417663574, + 7.302537441253662, + 7.104335069656372, + 6.969830274581909, + 6.93603253364563, + 6.763523817062378, + 6.563542127609253, + 6.370935440063477, + 6.4972474575042725, + 6.329789876937866, + 6.13366436958313, + 6.1089019775390625, + 6.092848539352417, + 5.972782611846924, + 5.9810261726379395, + 5.8788228034973145, + 5.704522132873535, + 5.680000305175781, + 5.712233304977417, + 5.608947038650513, + 5.613853693008423, + 5.58836030960083, + 5.459403038024902, + 5.3757030963897705, + 5.3833582401275635, + 5.445712566375732, + 5.108971834182739, + 5.355480194091797, + 5.190803527832031, + 5.277824878692627, + 5.236246347427368, + 5.142643213272095, + 5.229192018508911, + 5.156345844268799, + 4.8695454597473145, + 5.149953603744507, + 5.063090562820435, + 5.05802845954895, + 5.115422010421753, + 5.089590311050415, + 5.0074989795684814, + 4.9873456954956055, + 5.004148960113525, + 4.987397909164429, + 4.831067323684692, + 5.00829815864563, + 4.828976392745972, + 4.921891212463379, + 4.878847122192383, + 4.731806039810181, + 4.799510955810547, + 4.771551609039307, + 4.766524791717529, + 4.863228797912598, + 4.743756055831909, + 4.741974592208862, + 4.795232057571411, + 4.722021102905273, + 4.697370767593384, + 4.569409608840942, + 4.707533121109009, + 4.851870775222778, + 4.591017007827759, + 4.79823637008667, + 4.852752447128296, + 4.610065698623657, + 4.627712249755859, + 4.579732418060303, + 4.654018878936768, + 4.647151708602905, + 4.62382435798645, + 4.543317794799805, + 4.430135488510132, + 4.5591113567352295, + 4.6411051750183105, + 4.5455145835876465, + 4.637141704559326, + 4.513439655303955, + 4.545490503311157, + 4.524937868118286, + 4.48994517326355, + 4.503901720046997, + 4.396118879318237, + 4.531040906906128, + 4.475537061691284, + 4.620260238647461, + 4.353945970535278, + 4.427781581878662, + 4.4299585819244385, + 4.502310752868652, + 4.462902307510376, + 4.45287299156189, + 4.435248851776123, + 4.4303083419799805, + 4.3331849575042725, + 4.337235927581787, + 4.327529430389404, + 4.30165433883667, + 4.401233434677124, + 4.4690656661987305, + 4.303806781768799, + 4.271132230758667, + 4.334232568740845, + 4.373554944992065, + 4.398875951766968, + 4.354549169540405, + 4.3251259326934814, + 4.372986555099487, + 4.308181524276733, + 4.385535001754761, + 4.3299243450164795, + 4.347604751586914, + 4.227386236190796, + 4.1492698192596436, + 4.173330068588257, + 4.2884886264801025, + 4.286467790603638, + 4.097751140594482, + 4.284489631652832, + 4.205770492553711, + 4.299149036407471, + 4.270694017410278, + 4.123234272003174, + 4.237138509750366, + 4.130246162414551, + 4.092962741851807, + 4.259333610534668, + 4.151477098464966, + 4.181407928466797, + 4.144507646560669, + 4.205508708953857, + 4.143828868865967, + 4.177074193954468, + 4.195450067520142, + 4.111290216445923, + 4.174194097518921, + 4.230637788772583, + 4.143279075622559, + 4.061154127120972, + 4.089140176773071, + 4.159973621368408, + 4.063915967941284, + 4.068528413772583, + 4.225515127182007, + 4.014580249786377, + 4.099003791809082, + 4.171147346496582, + 4.106368780136108, + 4.077219724655151, + 3.994983673095703, + 4.086649179458618, + 4.124301910400391, + 4.196176528930664, + 4.0234386920928955, + 4.263610363006592, + 4.101058125495911, + 4.192416667938232, + 3.9848084449768066, + 4.005829095840454, + 4.178783655166626, + 4.034945607185364, + 4.182332277297974, + 4.150817155838013, + 4.09662127494812, + 4.221533298492432, + 4.102335691452026, + 4.173261880874634, + 4.040624618530273, + 4.077983379364014, + 4.065675258636475, + 4.196960210800171, + 4.1549177169799805, + 4.0456321239471436, + 4.146359205245972, + 4.052532076835632, + 4.086310863494873, + 4.275444507598877, + 4.130861520767212, + 4.055213451385498, + 4.022055625915527, + 3.970084309577942, + 4.075616359710693, + 4.071520090103149, + 3.981216549873352, + 4.052875995635986, + 4.164492130279541, + 3.9430359601974487, + 4.012874364852905, + 4.124771356582642, + 4.084800720214844, + 3.787041425704956, + 3.9932762384414673, + 4.087776303291321, + 4.0488903522491455, + 4.08594536781311, + 4.008386969566345, + 3.8737781047821045, + 4.095241665840149, + 3.901885986328125, + 3.9867604970932007, + 4.000642895698547, + 3.9106186628341675, + 3.900531053543091, + 4.02276086807251, + 3.8254154920578003, + 4.045905113220215, + 3.932168126106262, + 4.044691681861877, + 3.9793392419815063, + 3.9650849103927612, + 3.9900585412979126, + 4.01789927482605, + 4.080521821975708, + 3.9288054704666138, + 3.917252540588379, + 4.022040963172913, + 3.957750678062439, + 4.047248005867004, + 3.957527995109558, + 3.9349985122680664, + 4.000323414802551, + 3.8729217052459717, + 3.9122135639190674, + 4.000458002090454, + 3.8519922494888306, + 4.104188680648804, + 3.9615581035614014, + 3.9640146493911743, + 3.9148088693618774, + 3.835781455039978, + 3.888941764831543, + 4.009835600852966, + 3.9381245374679565, + 3.9850125312805176, + 3.9738478660583496, + 3.8813215494155884, + 3.9800411462783813, + 3.9124122858047485, + 3.8573750257492065, + 3.8595257997512817, + 3.961948871612549, + 3.977752208709717, + 3.823517322540283, + 3.9663615226745605, + 3.9134901762008667, + 3.9450958967208862, + 3.8430896997451782, + 3.881230354309082, + 3.8127366304397583, + 3.8931422233581543, + 3.890659809112549, + 3.7718394994735718, + 3.6904592514038086, + 3.732770562171936, + 3.8839155435562134, + 3.790118455886841, + 3.6267404556274414, + 3.8167864084243774, + 3.8205615282058716, + 3.756718635559082, + 3.824265718460083, + 3.8317558765411377, + 3.9956554174423218, + 3.8121073246002197, + 3.77086079120636, + 3.864113926887512, + 3.7210826873779297, + 3.798322558403015, + 3.7590795755386353, + 3.8324400186538696, + 3.964581608772278, + 3.9154661893844604, + 3.759956955909729, + 3.7142704725265503, + 3.7305768728256226, + 3.8203535079956055, + 3.915408730506897, + 3.7409223318099976, + 3.9259607791900635, + 3.840551495552063, + 3.810413122177124, + 3.829560875892639, + 3.8453508615493774, + 3.6110888719558716, + 3.8537579774856567, + 3.832363486289978, + 3.824670910835266, + 3.773284077644348, + 3.7591863870620728, + 3.8874958753585815, + 3.826709270477295, + 3.8503812551498413, + 3.7161144018173218, + 3.834787607192993, + 3.8738983869552612, + 3.738853693008423, + 3.831367015838623, + 3.9569164514541626, + 3.828178882598877, + 3.7215802669525146, + 3.7631523609161377, + 3.7628417015075684, + 3.9259344339370728, + 3.8759292364120483, + 3.778587818145752, + 3.8184902667999268, + 3.8767353296279907, + 3.773881435394287, + 3.7378252744674683, + 3.857879638671875, + 3.919119358062744, + 3.7183183431625366, + 3.9095523357391357, + 3.8193544149398804, + 3.814902663230896, + 3.7793198823928833, + 3.7064547538757324, + 3.8731645345687866, + 3.7702536582946777, + 3.7890573740005493, + 3.6258729696273804, + 3.808646082878113, + 3.7474018335342407, + 3.76917564868927, + 3.8167792558670044, + 3.8273426294326782, + 3.7729753255844116, + 3.814110040664673, + 3.897618532180786, + 3.7893502712249756, + 3.8437660932540894, + 3.7610214948654175, + 3.724684476852417, + 3.7217395305633545, + 3.7013049125671387, + 3.867881178855896, + 3.7379486560821533, + 3.793040633201599, + 3.8358418941497803, + 3.743691563606262, + 3.7164340019226074, + 3.8866270780563354, + 3.807172179222107, + 3.7000210285186768, + 3.861751437187195, + 3.608059287071228, + 3.6614140272140503, + 3.7949150800704956, + 3.8756715059280396, + 3.813953399658203, + 3.6946502923965454, + 3.789566397666931, + 3.741660952568054, + 3.8699235916137695, + 3.862525701522827, + 3.8854111433029175, + 3.8023040294647217, + 3.8358192443847656, + 3.747431993484497, + 3.717613458633423, + 3.6503015756607056, + 3.8587212562561035, + 3.7673439979553223, + 3.7503732442855835, + 3.7129170894622803, + 3.815732002258301, + 3.78299081325531, + 3.8785789012908936, + 3.82906711101532, + 3.678605079650879, + 3.7184643745422363, + 3.5799214839935303, + 3.6920411586761475, + 3.689605236053467, + 3.7708154916763306, + 3.6944773197174072, + 3.7425559759140015, + 3.518383264541626, + 3.7393077611923218, + 3.6818082332611084, + 3.647624373435974, + 3.5535932779312134, + 3.715991258621216, + 3.6338932514190674, + 3.7875818014144897, + 3.725663185119629, + 3.6496835947036743, + 3.611259698867798, + 3.6075345277786255, + 3.712296724319458, + 3.6964699029922485, + 3.71386981010437, + 3.7555726766586304, + 3.6187859773635864, + 3.5657676458358765, + 3.674143075942993, + 3.5584269762039185, + 3.7366573810577393, + 3.6286020278930664, + 3.5907785892486572, + 3.531205892562866, + 3.686668872833252, + 3.699129104614258, + 3.633247494697571, + 3.675019383430481, + 3.7043181657791138, + 3.6890605688095093, + 3.6022218465805054, + 3.666061043739319, + 3.717509150505066, + 3.580301523208618, + 3.782356381416321, + 3.6003241539001465, + 3.587449789047241, + 3.6573898792266846, + 3.515003800392151, + 3.6336385011672974, + 3.5941535234451294, + 3.6331945657730103, + 3.73340380191803, + 3.649920701980591, + 3.61410915851593, + 3.7007131576538086, + 3.677590012550354, + 3.75552761554718, + 3.68620765209198, + 3.650067090988159, + 3.757141590118408, + 3.5521013736724854, + 3.653264284133911, + 3.7980841398239136, + 3.75960373878479, + 3.651340961456299, + 3.5235875844955444, + 3.5626220703125, + 3.6759612560272217, + 3.685231566429138, + 3.599466323852539, + 3.6689640283584595, + 3.5143157243728638, + 3.6417081356048584, + 3.633180022239685, + 3.6226518154144287, + 3.5629976987838745, + 3.558014750480652, + 3.557093858718872, + 3.582646608352661, + 3.651623249053955, + 3.6664246320724487, + 3.736603260040283, + 3.7188788652420044, + 3.589044690132141, + 3.457908034324646, + 3.6459158658981323, + 3.617925763130188, + 3.843536376953125, + 3.685819983482361, + 3.578134298324585, + 3.538520336151123, + 3.7021067142486572, + 3.6479653120040894, + 3.466251492500305, + 3.4952701330184937, + 3.5750004053115845, + 3.590132713317871, + 3.729931950569153, + 3.7488754987716675, + 3.631823182106018, + 3.734391927719116, + 3.636057138442993, + 3.598333716392517, + 3.55400550365448, + 3.5021318197250366, + 3.6165056228637695, + 3.702807068824768, + 3.6327996253967285, + 3.5689934492111206, + 3.676620841026306, + 3.6259185075759888, + 3.648385524749756, + 3.6378540992736816, + 3.628057599067688, + 3.6281813383102417, + 3.654952645301819, + 3.710960268974304, + 3.6085721254348755, + 3.61719012260437, + 3.6398067474365234, + 3.626291275024414, + 3.61726713180542, + 3.721968650817871, + 3.613325834274292, + 3.690042734146118, + 3.7110915184020996, + 3.6856056451797485, + 3.421865940093994, + 3.6565041542053223, + 3.5318591594696045, + 3.7271316051483154, + 3.5336520671844482, + 3.651095747947693, + 3.6689207553863525, + 3.6518222093582153, + 3.7180906534194946, + 3.4695661067962646, + 3.4686636924743652, + 3.6366100311279297, + 3.539698839187622, + 3.526715397834778, + 3.4939807653427124, + 3.5860766172409058, + 3.4765753746032715, + 3.503576636314392, + 3.619803309440613, + 3.6495158672332764, + 3.5955169200897217, + 3.6049978733062744, + 3.4167678356170654, + 3.512558937072754, + 3.566495180130005, + 3.5197789669036865, + 3.4643714427948, + 3.4829238653182983, + 3.4602224826812744, + 3.6097301244735718, + 3.5304418802261353, + 3.4974969625473022, + 3.528473973274231, + 3.7560250759124756, + 3.465168595314026, + 3.5281381607055664, + 3.4993932247161865, + 3.4509202241897583, + 3.4376237392425537, + 3.419855833053589, + 3.6004282236099243, + 3.4610384702682495, + 3.624283790588379, + 3.5244009494781494, + 3.497830629348755, + 3.417862057685852, + 3.559911012649536, + 3.4377957582473755, + 3.5774303674697876, + 3.4377787113189697, + 3.4993512630462646, + 3.6475682258605957, + 3.4447537660598755, + 3.4652668237686157, + 3.4860564470291138, + 3.4820441007614136, + 3.606688976287842, + 3.3521268367767334, + 3.6377511024475098, + 3.4097920656204224, + 3.4705042839050293, + 3.5804309844970703, + 3.4295737743377686, + 3.4452617168426514, + 3.4560989141464233, + 3.416171669960022, + 3.5723648071289062, + 3.584526300430298, + 3.4907878637313843, + 3.4359270334243774, + 3.5652501583099365, + 3.470976233482361, + 3.41413950920105, + 3.3886605501174927, + 3.6432950496673584, + 3.5987164974212646, + 3.4647916555404663, + 3.553085446357727, + 3.564383625984192, + 3.5317983627319336, + 3.5010870695114136, + 3.4871867895126343, + 3.555480122566223, + 3.4652997255325317, + 3.5538723468780518, + 3.5094441175460815, + 3.4682050943374634, + 3.5693023204803467, + 3.5811718702316284, + 3.500525951385498, + 3.3897565603256226, + 3.643614411354065, + 3.477160096168518, + 3.449567198753357, + 3.6052385568618774, + 3.4327175617218018, + 3.4170044660568237, + 3.475111246109009, + 3.3907567262649536, + 3.475585460662842, + 3.506913185119629, + 3.6098852157592773, + 3.5151437520980835, + 3.568898320198059, + 3.510499119758606, + 3.5527127981185913, + 3.4573957920074463, + 3.4591503143310547, + 3.5428677797317505, + 3.5009868144989014, + 3.5696057081222534, + 3.551800012588501, + 3.424731731414795, + 3.533178210258484, + 3.4332618713378906, + 3.5553369522094727, + 3.524976372718811, + 3.559033751487732, + 3.63100528717041, + 3.387637495994568, + 3.438209056854248, + 3.322926163673401, + 3.4926412105560303, + 3.349120616912842, + 3.524866223335266, + 3.487138509750366, + 3.5590155124664307, + 3.489258050918579, + 3.522557497024536, + 3.5395208597183228, + 3.420214891433716, + 3.526695728302002, + 3.4155973196029663, + 3.4336977005004883, + 3.6407039165496826, + 3.3865208625793457, + 3.3231332302093506, + 3.4751604795455933, + 3.4411247968673706, + 3.3509410619735718, + 3.4677398204803467, + 3.4399757385253906, + 3.362837314605713, + 3.3291149139404297, + 3.3677854537963867, + 3.3927183151245117, + 3.4324851036071777, + 3.4495604038238525, + 3.312003254890442, + 3.4418365955352783, + 3.353217840194702, + 3.3496090173721313, + 3.3093013763427734, + 3.346621870994568, + 3.3511528968811035, + 3.3583563566207886, + 3.501634359359741, + 3.42975914478302, + 3.477685809135437, + 3.4550687074661255, + 3.513934373855591, + 3.377328038215637, + 3.3564287424087524, + 3.226001501083374, + 3.481455445289612, + 3.3489311933517456, + 3.5189379453659058, + 3.4578278064727783, + 3.358473062515259, + 3.4835060834884644, + 3.5429701805114746, + 3.5190634727478027, + 3.415704131126404, + 3.554509401321411, + 3.4199832677841187, + 3.298734188079834, + 3.410587430000305, + 3.294886589050293, + 3.4949110746383667, + 3.4450260400772095, + 3.3661749362945557, + 3.401394248008728, + 3.3664709329605103, + 3.49123215675354, + 3.3848308324813843, + 3.4139755964279175, + 3.5556923151016235, + 3.363577723503113, + 3.3493235111236572, + 3.347380757331848, + 3.443648338317871, + 3.498611807823181, + 3.42507541179657, + 3.411870002746582, + 3.357208013534546, + 3.3701796531677246, + 3.365540027618408, + 3.473201274871826, + 3.4737818241119385, + 3.4660747051239014, + 3.4523463249206543, + 3.3475801944732666, + 3.5195281505584717, + 3.48890221118927, + 3.551285743713379, + 3.386815071105957, + 3.3594610691070557, + 3.3869515657424927, + 3.2643362283706665, + 3.3372385501861572, + 3.3718453645706177, + 3.468748092651367, + 3.5307741165161133, + 3.5320639610290527, + 3.380175232887268, + 3.416056752204895, + 3.359110951423645, + 3.4327200651168823, + 3.379326820373535, + 3.3888437747955322, + 3.3919533491134644, + 3.407154679298401, + 3.4223352670669556, + 3.455893039703369, + 3.327970862388611, + 3.4927114248275757, + 3.4241117238998413, + 3.3627315759658813, + 3.30581271648407, + 3.2858227491378784, + 3.435559391975403, + 3.428718328475952, + 3.4679912328720093, + 3.425679326057434, + 3.422518730163574, + 3.442650556564331, + 3.350014805793762, + 3.4689441919326782, + 3.3795876502990723, + 3.4548733234405518, + 3.357605457305908, + 3.2471537590026855, + 3.513254404067993, + 3.423876404762268, + 3.47690212726593, + 3.38931667804718, + 3.404360294342041, + 3.3724290132522583, + 3.5194119215011597, + 3.311945080757141, + 3.5729650259017944, + 3.4118053913116455, + 3.4073359966278076, + 3.4344427585601807, + 3.363187551498413, + 3.360482692718506, + 3.398476243019104, + 3.4347143173217773, + 3.3108699321746826, + 3.399821639060974, + 3.3049988746643066, + 3.3655319213867188, + 3.19239604473114, + 3.2104084491729736, + 3.294224739074707, + 3.2539600133895874, + 3.2754151821136475, + 3.1818639039993286, + 3.369205117225647, + 3.3950400352478027, + 3.295725107192993, + 3.404174566268921, + 3.4565513134002686, + 3.335212469100952, + 3.412765622138977, + 3.3246456384658813, + 3.3764140605926514, + 3.3569306135177612, + 3.418075442314148, + 3.3811835050582886, + 3.3171435594558716, + 3.2853636741638184, + 3.402900218963623, + 3.3735769987106323, + 3.355115532875061, + 3.4099103212356567, + 3.4609228372573853, + 3.4953962564468384, + 3.3206080198287964, + 3.320672392845154, + 3.3442846536636353, + 3.2917269468307495, + 3.3090598583221436, + 3.3410667181015015, + 3.378603219985962, + 3.4606064558029175, + 3.3243273496627808, + 3.3021315336227417, + 3.342462182044983, + 3.224645495414734, + 3.4005388021469116, + 3.3392210006713867, + 3.4320753812789917, + 3.3383244276046753, + 3.1827253103256226, + 3.3630863428115845, + 3.453102231025696, + 3.258752465248108, + 3.291088581085205, + 3.3439996242523193, + 3.3378310203552246, + 3.2936519384384155, + 3.2876685857772827, + 3.5387459993362427, + 3.3702876567840576, + 3.2573964595794678, + 3.1523942947387695, + 3.3078103065490723, + 3.2910205125808716, + 3.372058391571045, + 3.430067539215088, + 3.2757648229599, + 3.4828726053237915, + 3.359042167663574, + 3.476567268371582, + 3.465182065963745, + 3.3757705688476562, + 3.292293429374695, + 3.250236988067627, + 3.43453049659729, + 3.440784215927124, + 3.3015867471694946, + 3.2663952112197876, + 3.3669687509536743, + 3.336418628692627, + 3.4012404680252075, + 3.2775778770446777, + 3.255927324295044, + 3.2960857152938843, + 3.349342107772827, + 3.3393847942352295, + 3.4397876262664795, + 3.3019102811813354, + 3.250092625617981, + 3.2669306993484497, + 3.286477208137512, + 3.124701976776123, + 3.496988534927368, + 3.383002519607544, + 3.3826054334640503, + 3.4199942350387573, + 3.3386592864990234, + 3.3134325742721558, + 3.186792492866516, + 3.242035984992981, + 3.4356521368026733, + 3.4032636880874634, + 3.3258674144744873, + 3.2466013431549072, + 3.425989270210266, + 3.426310181617737, + 3.2460906505584717, + 3.2795969247817993, + 3.3884918689727783, + 3.3982083797454834, + 3.420872926712036, + 3.3414106369018555, + 3.565899610519409, + 3.397579789161682, + 3.3051220178604126, + 3.306242823600769, + 3.3994251489639282, + 3.3335235118865967, + 3.364367365837097, + 3.3751413822174072, + 3.2750189304351807, + 3.3282251358032227, + 3.4230445623397827, + 3.255243420600891, + 3.3679137229919434, + 3.272101879119873, + 3.353832960128784, + 3.3539587259292603, + 3.232949137687683, + 3.442306399345398, + 3.2802984714508057, + 3.210158944129944, + 3.424932599067688, + 3.361391305923462, + 3.2147289514541626, + 3.3379952907562256, + 3.1072815656661987, + 3.4633820056915283, + 3.1307711601257324, + 3.212860584259033, + 3.312686324119568, + 3.2491650581359863, + 3.2416210174560547, + 3.257210612297058, + 3.252503991127014, + 3.2894927263259888, + 3.1152923107147217, + 3.317408561706543, + 3.20628023147583, + 3.348110795021057, + 3.273751735687256, + 3.28776478767395, + 3.3605858087539673, + 3.3309444189071655, + 3.0836814641952515, + 3.254409432411194, + 3.2928332090377808, + 3.311555027961731, + 3.225108027458191, + 3.2152074575424194, + 3.4862459897994995, + 3.2230619192123413, + 3.2988914251327515, + 3.324784278869629, + 3.28733229637146, + 3.3928967714309692, + 3.271755576133728, + 3.2979652881622314, + 3.3122808933258057, + 3.269972801208496, + 3.3417060375213623, + 3.316786527633667, + 3.232014298439026, + 3.34953236579895, + 3.2675153017044067, + 3.347611665725708, + 3.3536099195480347, + 3.247461676597595, + 3.2662843465805054, + 3.2417105436325073, + 3.22912859916687, + 3.3266366720199585, + 3.295850157737732, + 3.2666921615600586, + 3.277311325073242, + 3.3413619995117188, + 3.2371039390563965, + 3.203627109527588, + 3.3280574083328247, + 3.313723564147949, + 3.435691475868225, + 3.412056803703308, + 3.311555862426758, + 3.148516058921814, + 3.33504319190979, + 3.2134326696395874, + 3.1975399255752563, + 3.2477771043777466, + 3.3223981857299805, + 3.253838896751404, + 3.299152970314026, + 3.2972733974456787, + 3.283071517944336, + 3.1766180992126465, + 3.177915930747986, + 3.2256044149398804, + 3.288369655609131, + 3.3517353534698486, + 3.3798916339874268, + 3.3104453086853027, + 3.228913187980652, + 3.387970447540283, + 3.319370150566101, + 3.337522029876709, + 3.2075852155685425, + 3.3619359731674194, + 3.367494821548462, + 3.274208426475525, + 3.1797221899032593, + 3.326419472694397, + 3.3539514541625977, + 3.1905088424682617, + 3.3682559728622437, + 3.258684754371643, + 3.165399432182312, + 3.379910469055176, + 3.2653053998947144, + 3.356666088104248, + 3.388769507408142, + 3.1964014768600464, + 3.26131808757782, + 3.344655394554138, + 3.2384095191955566, + 3.3142788410186768, + 3.2726497650146484, + 3.3553473949432373, + 3.2864450216293335, + 3.3981635570526123, + 3.310488224029541, + 3.405124068260193, + 3.1474404335021973, + 3.341571807861328, + 3.555699110031128, + 3.351510167121887, + 3.356718897819519, + 3.314704418182373, + 3.2768964767456055, + 3.314183831214905 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 117.55270862986598, + 73.1884096392179, + 58.097423203948445, + 51.21322975444702, + 46.7210432594688, + 43.867047029636254, + 41.162847400076465, + 39.289508737255055, + 37.5643366964298, + 36.25412887668644, + 34.94861984360324, + 34.03733483279951, + 33.35289969809013, + 32.62285262127078, + 31.897323688198203, + 31.614064264442234, + 31.28316212145717, + 31.135725424474956, + 31.044674412911494, + 31.032339174116004 + ], + "best_ppl": 31.032339174116004, + "train_time_s": 21301.16504049301 +} \ No newline at end of file diff --git a/063_sparse_dispatch/balanced_deltanet_1plus3of6_best.pt b/063_sparse_dispatch/balanced_deltanet_1plus3of6_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..c46eb208d5a1a929a978a6537245195b64b4e679 --- /dev/null +++ b/063_sparse_dispatch/balanced_deltanet_1plus3of6_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95507dc5a22142205638d65c06768df7792e0ece2681d887ca3c920af9cf60a8 +size 394730538 diff --git a/063_sparse_dispatch/balanced_deltanet_1plus3of6_results.json b/063_sparse_dispatch/balanced_deltanet_1plus3of6_results.json new file mode 100644 index 0000000000000000000000000000000000000000..a934d1fdc426d3db4f637f2728f78914307d840d --- /dev/null +++ b/063_sparse_dispatch/balanced_deltanet_1plus3of6_results.json @@ -0,0 +1,2059 @@ +{ + "label": "balanced_deltanet_1plus3of6", + "seed": 42, + "balanced_shared_weight": true, + "num_thin_blocks": 6, + "num_shared_blocks": 1, + "top_k_blocks": 3, + "total_params": 98587456, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.085857391357422, + 15.7960844039917, + 8.084715366363525, + 7.269772052764893, + 7.115804433822632, + 6.976958274841309, + 6.865390300750732, + 6.6973748207092285, + 6.563857555389404, + 6.58165717124939, + 6.318305492401123, + 6.207465171813965, + 6.198413848876953, + 6.188330888748169, + 6.045083045959473, + 6.025580406188965, + 5.8372368812561035, + 5.885495662689209, + 5.801267623901367, + 5.750070571899414, + 5.699856758117676, + 5.6602373123168945, + 5.514317035675049, + 5.534551620483398, + 5.4814934730529785, + 5.4945971965789795, + 5.333246231079102, + 5.173483371734619, + 5.3304078578948975, + 5.2302350997924805, + 5.252413749694824, + 5.3107452392578125, + 5.0876123905181885, + 5.149800539016724, + 5.170955181121826, + 5.039217472076416, + 5.055363416671753, + 4.9825310707092285, + 4.904711008071899, + 5.0719873905181885, + 4.985686540603638, + 4.929452180862427, + 5.030965566635132, + 5.011810302734375, + 4.930705308914185, + 4.984079599380493, + 4.79124903678894, + 4.902130842208862, + 4.830636739730835, + 4.87602162361145, + 4.773613214492798, + 4.840571880340576, + 4.622952461242676, + 4.6917760372161865, + 4.764496088027954, + 4.803630352020264, + 4.753455638885498, + 4.811884164810181, + 4.749538898468018, + 4.615954399108887, + 4.670522689819336, + 4.693573474884033, + 4.623382568359375, + 4.518625974655151, + 4.6535563468933105, + 4.626264333724976, + 4.518850803375244, + 4.6091461181640625, + 4.594249963760376, + 4.507962942123413, + 4.6594178676605225, + 4.651342153549194, + 4.544722080230713, + 4.487107038497925, + 4.542503356933594, + 4.562257766723633, + 4.472853660583496, + 4.536550045013428, + 4.410857200622559, + 4.5833234786987305, + 4.450330972671509, + 4.410064697265625, + 4.382476568222046, + 4.396435976028442, + 4.462592363357544, + 4.418374538421631, + 4.43113899230957, + 4.355134963989258, + 4.321986198425293, + 4.448178291320801, + 4.345733165740967, + 4.341181039810181, + 4.4604761600494385, + 4.47143292427063, + 4.358581781387329, + 4.421976089477539, + 4.33013653755188, + 4.305842399597168, + 4.313527345657349, + 4.309006452560425, + 4.305314779281616, + 4.453691482543945, + 4.357759714126587, + 4.361327171325684, + 4.257774114608765, + 4.228090524673462, + 4.321726560592651, + 4.24001407623291, + 4.336049318313599, + 4.356085300445557, + 4.330629110336304, + 4.272109270095825, + 4.371903896331787, + 4.248117208480835, + 4.355980396270752, + 4.234881639480591, + 4.196161508560181, + 4.204831600189209, + 4.206858396530151, + 4.217545032501221, + 4.250429630279541, + 4.188518762588501, + 4.223635673522949, + 4.141183376312256, + 4.196440696716309, + 4.120326042175293, + 4.074208498001099, + 4.2417426109313965, + 3.9905829429626465, + 4.160809278488159, + 4.04470431804657, + 4.05075740814209, + 4.070109128952026, + 4.225004434585571, + 4.150050163269043, + 4.224951267242432, + 4.1129865646362305, + 4.072397232055664, + 4.089522361755371, + 4.160640716552734, + 4.217164039611816, + 4.158113241195679, + 4.189948558807373, + 4.1697916984558105, + 4.060506820678711, + 4.060059666633606, + 4.120606422424316, + 4.095697641372681, + 4.082784414291382, + 4.02613377571106, + 4.134916067123413, + 4.064754009246826, + 4.09190559387207, + 4.191615581512451, + 4.138452768325806, + 3.97579026222229, + 4.084965944290161, + 4.144678354263306, + 4.117613315582275, + 4.076606750488281, + 3.9879218339920044, + 4.03940224647522, + 4.020867586135864, + 3.9884461164474487, + 4.0302348136901855, + 4.181478977203369, + 4.017704367637634, + 4.0111870765686035, + 3.933433175086975, + 3.9776909351348877, + 4.173984050750732, + 4.022980332374573, + 4.146968603134155, + 4.028210639953613, + 4.044329881668091, + 3.97452974319458, + 4.091890335083008, + 3.952360153198242, + 3.847607731819153, + 4.089722990989685, + 3.83358895778656, + 3.990049123764038, + 4.147211074829102, + 3.8798019886016846, + 3.9097964763641357, + 3.9258095026016235, + 3.982540011405945, + 4.066657781600952, + 4.123251438140869, + 3.995290756225586, + 4.0668206214904785, + 4.014410495758057, + 4.0054895877838135, + 4.129716157913208, + 3.9705721139907837, + 3.8613511323928833, + 3.897835612297058, + 3.956338405609131, + 4.065925598144531, + 4.075861215591431, + 4.117193937301636, + 4.032204985618591, + 3.9518481492996216, + 3.896615147590637, + 3.8115952014923096, + 3.904595971107483, + 3.891157627105713, + 3.967212677001953, + 4.0295268297195435, + 4.127885103225708, + 3.862181305885315, + 3.8781094551086426, + 3.780071973800659, + 4.063954830169678, + 3.9423820972442627, + 3.938888907432556, + 4.054575443267822, + 3.955027222633362, + 3.7468942403793335, + 3.970065951347351, + 3.874788761138916, + 3.867268204689026, + 3.8324806690216064, + 4.058453559875488, + 3.8589102029800415, + 3.9618762731552124, + 3.8000218868255615, + 4.056195974349976, + 3.9364267587661743, + 3.8989239931106567, + 3.925264596939087, + 4.011020302772522, + 3.8175920248031616, + 3.993189811706543, + 3.819059133529663, + 3.8678847551345825, + 3.9834537506103516, + 3.823626399040222, + 3.840659976005554, + 3.8917731046676636, + 3.9119069576263428, + 3.9247580766677856, + 3.866446852684021, + 3.832081437110901, + 3.98179292678833, + 3.8467177152633667, + 3.920008420944214, + 3.9017884731292725, + 3.947686791419983, + 3.825057625770569, + 3.8873369693756104, + 3.8500802516937256, + 3.9028533697128296, + 3.8486249446868896, + 3.8373324871063232, + 3.821506977081299, + 3.7795110940933228, + 3.776155114173889, + 3.8671339750289917, + 3.7546600103378296, + 3.7924067974090576, + 3.7508617639541626, + 3.8053420782089233, + 3.7924829721450806, + 3.8839621543884277, + 3.6306872367858887, + 3.728341221809387, + 3.835069179534912, + 3.770739197731018, + 3.7929775714874268, + 3.671121835708618, + 3.84882128238678, + 3.9592552185058594, + 3.812801241874695, + 3.86094868183136, + 3.804375171661377, + 3.7795807123184204, + 3.8340845108032227, + 3.837369441986084, + 3.6737221479415894, + 3.7676498889923096, + 3.8291335105895996, + 3.7829973697662354, + 3.729866623878479, + 3.900453567504883, + 3.7708632946014404, + 3.7612842321395874, + 3.8208075761795044, + 3.79190194606781, + 3.754246473312378, + 3.871347188949585, + 3.7299524545669556, + 3.913514733314514, + 3.885175585746765, + 3.8195910453796387, + 3.7941527366638184, + 3.8150583505630493, + 3.7282506227493286, + 3.7921836376190186, + 3.8470691442489624, + 3.7571548223495483, + 3.751710057258606, + 3.7542353868484497, + 3.7352092266082764, + 3.898082971572876, + 3.9275354146957397, + 3.6714115142822266, + 3.6646543741226196, + 3.759265422821045, + 3.7709131240844727, + 3.826076030731201, + 3.8097962141036987, + 3.8380662202835083, + 3.9318339824676514, + 3.7341413497924805, + 3.817577600479126, + 3.648656487464905, + 3.636190414428711, + 3.7715903520584106, + 3.628665804862976, + 3.657242178916931, + 3.8720637559890747, + 3.68780517578125, + 3.776196599006653, + 3.7790063619613647, + 3.755399465560913, + 3.671147584915161, + 3.873558521270752, + 3.717485189437866, + 3.729549288749695, + 3.7322709560394287, + 3.6620726585388184, + 3.746409773826599, + 3.735427498817444, + 3.6962414979934692, + 3.7033727169036865, + 3.8094534873962402, + 3.8385342359542847, + 3.700697183609009, + 3.763546943664551, + 3.656964063644409, + 3.8059802055358887, + 3.9088499546051025, + 3.6325840950012207, + 3.775164484977722, + 3.7689954042434692, + 3.664253830909729, + 3.63487708568573, + 3.7680680751800537, + 3.8414621353149414, + 3.7448129653930664, + 3.7109265327453613, + 3.7842360734939575, + 3.809434652328491, + 3.8124940395355225, + 3.706286072731018, + 3.688620924949646, + 3.700234889984131, + 3.66012966632843, + 3.739850878715515, + 3.6360907554626465, + 3.744483709335327, + 3.7161561250686646, + 3.652345657348633, + 3.797444224357605, + 3.638648748397827, + 3.7121026515960693, + 3.7890695333480835, + 3.7891123294830322, + 3.772064447402954, + 3.7260196208953857, + 3.7972004413604736, + 3.760788679122925, + 3.828640937805176, + 3.8144237995147705, + 3.6433029174804688, + 3.6449689865112305, + 3.5788238048553467, + 3.6491811275482178, + 3.512993812561035, + 3.6338398456573486, + 3.5789244174957275, + 3.715512275695801, + 3.5672240257263184, + 3.604828953742981, + 3.5196787118911743, + 3.670942187309265, + 3.7305082082748413, + 3.627218008041382, + 3.7021149396896362, + 3.5059242248535156, + 3.6233015060424805, + 3.526971697807312, + 3.6856112480163574, + 3.6086466312408447, + 3.5168014764785767, + 3.4511674642562866, + 3.6543385982513428, + 3.5486161708831787, + 3.758297562599182, + 3.6573034524917603, + 3.678022027015686, + 3.6008599996566772, + 3.6652969121932983, + 3.5909301042556763, + 3.6087968349456787, + 3.585546374320984, + 3.607954978942871, + 3.6132577657699585, + 3.594838857650757, + 3.5985394716262817, + 3.7380220890045166, + 3.6330913305282593, + 3.5789486169815063, + 3.6749989986419678, + 3.716678738594055, + 3.697543978691101, + 3.6230239868164062, + 3.5652209520339966, + 3.7170544862747192, + 3.5330251455307007, + 3.810490846633911, + 3.724609136581421, + 3.561073899269104, + 3.6718064546585083, + 3.5068799257278442, + 3.5932323932647705, + 3.551817297935486, + 3.4983668327331543, + 3.6497461795806885, + 3.5796427726745605, + 3.6000269651412964, + 3.6933008432388306, + 3.5949782133102417, + 3.6968377828598022, + 3.6005741357803345, + 3.612046957015991, + 3.482120394706726, + 3.607231616973877, + 3.6498392820358276, + 3.5115609169006348, + 3.6608314514160156, + 3.5216907262802124, + 3.5909489393234253, + 3.6114935874938965, + 3.5884337425231934, + 3.647085428237915, + 3.524978518486023, + 3.651800513267517, + 3.702857494354248, + 3.529950737953186, + 3.5442434549331665, + 3.7024816274642944, + 3.5780550241470337, + 3.7128835916519165, + 3.658521056175232, + 3.5938894748687744, + 3.5769840478897095, + 3.561738133430481, + 3.553110361099243, + 3.5183814764022827, + 3.588633418083191, + 3.6672381162643433, + 3.5232666730880737, + 3.5956904888153076, + 3.724709987640381, + 3.450002908706665, + 3.632975697517395, + 3.673710346221924, + 3.544259190559387, + 3.5972477197647095, + 3.5284199714660645, + 3.5061110258102417, + 3.5235201120376587, + 3.4450191259384155, + 3.5785884857177734, + 3.6303504705429077, + 3.5324342250823975, + 3.6026575565338135, + 3.5649322271347046, + 3.6760066747665405, + 3.626176953315735, + 3.5918142795562744, + 3.605644702911377, + 3.6085108518600464, + 3.4657047986984253, + 3.5043137073516846, + 3.5343143939971924, + 3.525006890296936, + 3.7086288928985596, + 3.498537302017212, + 3.454050302505493, + 3.541843295097351, + 3.585397243499756, + 3.6103166341781616, + 3.6568347215652466, + 3.5047181844711304, + 3.615755558013916, + 3.536681294441223, + 3.503383755683899, + 3.5559369325637817, + 3.6793911457061768, + 3.6192731857299805, + 3.571473479270935, + 3.6235891580581665, + 3.6395152807235718, + 3.516170620918274, + 3.3950165510177612, + 3.472417950630188, + 3.4915359020233154, + 3.5036309957504272, + 3.4992101192474365, + 3.344006896018982, + 3.493019938468933, + 3.424659490585327, + 3.3315166234970093, + 3.5526100397109985, + 3.521783709526062, + 3.4770662784576416, + 3.5973376035690308, + 3.383613705635071, + 3.4992047548294067, + 3.460150957107544, + 3.353829860687256, + 3.5032438039779663, + 3.4499207735061646, + 3.454202651977539, + 3.5920486450195312, + 3.378092646598816, + 3.4688901901245117, + 3.5313223600387573, + 3.4810580015182495, + 3.542093873023987, + 3.446246385574341, + 3.456005334854126, + 3.4142926931381226, + 3.500682234764099, + 3.5142959356307983, + 3.4448786973953247, + 3.499859094619751, + 3.4596431255340576, + 3.336891531944275, + 3.608299970626831, + 3.5169790983200073, + 3.538418173789978, + 3.5857752561569214, + 3.3397337198257446, + 3.5124783515930176, + 3.39776611328125, + 3.444884419441223, + 3.447464346885681, + 3.4829622507095337, + 3.5616607666015625, + 3.469622850418091, + 3.446899652481079, + 3.5091177225112915, + 3.480888247489929, + 3.459283947944641, + 3.454621911048889, + 3.5811493396759033, + 3.4386292695999146, + 3.477880358695984, + 3.562601923942566, + 3.6020805835723877, + 3.4274277687072754, + 3.440282702445984, + 3.500128149986267, + 3.4171189069747925, + 3.49467396736145, + 3.5112897157669067, + 3.5143450498580933, + 3.5718328952789307, + 3.5022870302200317, + 3.4477137327194214, + 3.485353708267212, + 3.532519221305847, + 3.3993546962738037, + 3.4768253564834595, + 3.4573798179626465, + 3.44390869140625, + 3.4403228759765625, + 3.3378442525863647, + 3.442158818244934, + 3.4197582006454468, + 3.3369029760360718, + 3.493538737297058, + 3.405121684074402, + 3.5316795110702515, + 3.351660132408142, + 3.6473180055618286, + 3.582255244255066, + 3.5548737049102783, + 3.5083612203598022, + 3.4057867527008057, + 3.5874361991882324, + 3.4648126363754272, + 3.4087129831314087, + 3.4903299808502197, + 3.5154075622558594, + 3.2878735065460205, + 3.4980380535125732, + 3.4918292760849, + 3.4273247718811035, + 3.4311107397079468, + 3.515590190887451, + 3.3391276597976685, + 3.3868513107299805, + 3.438092589378357, + 3.503023147583008, + 3.6643340587615967, + 3.367851495742798, + 3.340087890625, + 3.355377435684204, + 3.4220372438430786, + 3.426239013671875, + 3.4224541187286377, + 3.4547271728515625, + 3.466562867164612, + 3.4756988286972046, + 3.5525639057159424, + 3.4243243932724, + 3.477448344230652, + 3.4515609741210938, + 3.4300626516342163, + 3.456786274909973, + 3.5966724157333374, + 3.4866397380828857, + 3.485027551651001, + 3.41692054271698, + 3.56682813167572, + 3.599721908569336, + 3.369882583618164, + 3.4150465726852417, + 3.4105299711227417, + 3.3339741230010986, + 3.1999436616897583, + 3.4655182361602783, + 3.3986356258392334, + 3.3295185565948486, + 3.1879571676254272, + 3.433794617652893, + 3.298597574234009, + 3.402157187461853, + 3.337030291557312, + 3.35670268535614, + 3.3611106872558594, + 3.38663911819458, + 3.3719347715377808, + 3.3851362466812134, + 3.3840616941452026, + 3.2845702171325684, + 3.45470929145813, + 3.3889806270599365, + 3.4267090559005737, + 3.422686219215393, + 3.3859745264053345, + 3.3026578426361084, + 3.2775062322616577, + 3.473412275314331, + 3.429982900619507, + 3.206347703933716, + 3.4222558736801147, + 3.3337236642837524, + 3.369503974914551, + 3.340955376625061, + 3.374082088470459, + 3.372135877609253, + 3.2888067960739136, + 3.5637019872665405, + 3.4388052225112915, + 3.3350309133529663, + 3.33974289894104, + 3.389807105064392, + 3.237383723258972, + 3.3529118299484253, + 3.295095920562744, + 3.284854292869568, + 3.329591751098633, + 3.3868976831436157, + 3.4105677604675293, + 3.2640366554260254, + 3.252860426902771, + 3.2688406705856323, + 3.461190104484558, + 3.4517080783843994, + 3.2577351331710815, + 3.4646196365356445, + 3.394858956336975, + 3.406968116760254, + 3.3131109476089478, + 3.446077585220337, + 3.195185422897339, + 3.3014795780181885, + 3.4019370079040527, + 3.413231134414673, + 3.3209002017974854, + 3.374460220336914, + 3.3490008115768433, + 3.4238635301589966, + 3.3837730884552, + 3.323177933692932, + 3.425782561302185, + 3.353366255760193, + 3.3581619262695312, + 3.495099902153015, + 3.373226284980774, + 3.401865601539612, + 3.2753530740737915, + 3.4046237468719482, + 3.3222811222076416, + 3.3649609088897705, + 3.3352245092391968, + 3.3922334909439087, + 3.3864704370498657, + 3.482372283935547, + 3.36634361743927, + 3.380824089050293, + 3.2058820724487305, + 3.3978769779205322, + 3.3923075199127197, + 3.20743727684021, + 3.4003732204437256, + 3.4032814502716064, + 3.350336194038391, + 3.4476383924484253, + 3.451524257659912, + 3.379690647125244, + 3.4468414783477783, + 3.4049272537231445, + 3.4368592500686646, + 3.377269744873047, + 3.288966178894043, + 3.332480788230896, + 3.336626410484314, + 3.3442524671554565, + 3.3138710260391235, + 3.438999652862549, + 3.3852123022079468, + 3.3402732610702515, + 3.3911805152893066, + 3.242873191833496, + 3.425474762916565, + 3.3345041275024414, + 3.399553418159485, + 3.2937171459198, + 3.4808183908462524, + 3.4035600423812866, + 3.5013880729675293, + 3.2875025272369385, + 3.287014603614807, + 3.366511583328247, + 3.38117253780365, + 3.4117945432662964, + 3.2939692735671997, + 3.3262219429016113, + 3.3819996118545532, + 3.5263839960098267, + 3.340938687324524, + 3.30381977558136, + 3.2495651245117188, + 3.284385085105896, + 3.2649508714675903, + 3.262583374977112, + 3.2296403646469116, + 3.3558555841445923, + 3.185515284538269, + 3.191487193107605, + 3.2069011926651, + 3.243308186531067, + 3.292126178741455, + 3.2940019369125366, + 3.314452052116394, + 3.2622735500335693, + 3.21907901763916, + 3.2744439840316772, + 3.3281853199005127, + 3.2323484420776367, + 3.3102301359176636, + 3.2682148218154907, + 3.2207179069519043, + 3.3059366941452026, + 3.252349853515625, + 3.3307684659957886, + 3.2921818494796753, + 3.258306384086609, + 3.332956075668335, + 3.3948779106140137, + 3.334708333015442, + 3.3566380739212036, + 3.2159441709518433, + 3.2470524311065674, + 3.2658475637435913, + 3.2807955741882324, + 3.251396417617798, + 3.2178642749786377, + 3.2878329753875732, + 3.3256510496139526, + 3.187905430793762, + 3.3372998237609863, + 3.290140986442566, + 3.408114194869995, + 3.245687246322632, + 3.2002724409103394, + 3.29876708984375, + 3.184470534324646, + 3.267530679702759, + 3.255461096763611, + 3.2179064750671387, + 3.288639187812805, + 3.245124101638794, + 3.350965142250061, + 3.2256935834884644, + 3.446868896484375, + 3.3231178522109985, + 3.3332284688949585, + 3.3148633241653442, + 3.3068264722824097, + 3.2827329635620117, + 3.2794876098632812, + 3.3340755701065063, + 3.226125121116638, + 3.239516496658325, + 3.142952561378479, + 3.3697479963302612, + 3.3119823932647705, + 3.3023563623428345, + 3.202345848083496, + 3.351705312728882, + 3.3490021228790283, + 3.304727554321289, + 3.2613227367401123, + 3.2374162673950195, + 3.2715418338775635, + 3.2585225105285645, + 3.272093892097473, + 3.416548252105713, + 3.3174030780792236, + 3.3170140981674194, + 3.3160669803619385, + 3.163527727127075, + 3.3944647312164307, + 3.2353742122650146, + 3.2248432636260986, + 3.1159344911575317, + 3.247612953186035, + 3.3042858839035034, + 3.2133820056915283, + 3.1403732299804688, + 3.327017068862915, + 3.287666440010071, + 3.3129639625549316, + 3.3391966819763184, + 3.2068463563919067, + 3.358134627342224, + 3.3544737100601196, + 3.2979568243026733, + 3.349106788635254, + 3.3001397848129272, + 3.3463938236236572, + 3.409606099128723, + 3.2405701875686646, + 3.2447482347488403, + 3.2978515625, + 3.2806835174560547, + 3.187174081802368, + 3.237376570701599, + 3.277395725250244, + 3.384771227836609, + 3.2115978002548218, + 3.1984527111053467, + 3.324797511100769, + 3.3195316791534424, + 3.233686089515686, + 3.214277148246765, + 3.344047427177429, + 3.310545802116394, + 3.3962483406066895, + 3.323320150375366, + 3.3497334718704224, + 3.2894092798233032, + 3.222906231880188, + 3.4471241235733032, + 3.2608468532562256, + 3.3410913944244385, + 3.2313953638076782, + 3.2172255516052246, + 3.2376296520233154, + 3.1811130046844482, + 3.21339213848114, + 3.3748810291290283, + 3.2225656509399414, + 3.2753278017044067, + 3.165448784828186, + 3.295535683631897, + 3.285962224006653, + 3.258027672767639, + 3.2383294105529785, + 3.229519486427307, + 3.250970721244812, + 3.2738810777664185, + 3.2363189458847046, + 3.221648931503296, + 3.253149390220642, + 3.2753700017929077, + 3.2453447580337524, + 3.160874366760254, + 3.183371663093567, + 3.3464635610580444, + 3.2561975717544556, + 3.2486921548843384, + 3.2117499113082886, + 3.216084599494934, + 3.3073320388793945, + 3.2254937887191772, + 3.2006741762161255, + 3.1220595836639404, + 3.274412512779236, + 3.226749539375305, + 3.2818663120269775, + 3.317286729812622, + 3.1840239763259888, + 3.1953521966934204, + 3.266696572303772, + 3.0718109607696533, + 3.2025623321533203, + 3.205581307411194, + 3.2689671516418457, + 3.2053871154785156, + 3.340722918510437, + 3.2831268310546875, + 3.2625967264175415, + 3.1032480001449585, + 3.308484673500061, + 3.225854516029358, + 3.169452667236328, + 3.371433138847351, + 3.2346768379211426, + 3.20757520198822, + 3.280930280685425, + 3.241533398628235, + 3.1621766090393066, + 3.2237380743026733, + 3.281359076499939, + 3.190453052520752, + 3.1945393085479736, + 3.2740641832351685, + 3.145572066307068, + 3.3015536069869995, + 3.2158403396606445, + 3.234758496284485, + 3.3235827684402466, + 3.300554394721985, + 3.256203532218933, + 3.0477343797683716, + 3.3287699222564697, + 3.3098702430725098, + 3.2743613719940186, + 3.1995779275894165, + 3.235758662223816, + 3.1713035106658936, + 3.308602213859558, + 3.1312272548675537, + 3.250993013381958, + 3.2157912254333496, + 3.222941756248474, + 3.2375353574752808, + 3.280011296272278, + 3.3173301219940186, + 3.3541042804718018, + 3.245886445045471, + 3.2437922954559326, + 3.231110453605652, + 3.3107895851135254, + 3.148490786552429, + 3.1939122676849365, + 3.1939380168914795, + 3.2030200958251953, + 3.3315013647079468, + 3.262391209602356, + 3.283397912979126, + 3.191396474838257, + 3.2470624446868896, + 3.2937779426574707, + 3.2864906787872314, + 3.4136247634887695, + 3.3652180433273315, + 3.3136682510375977, + 3.0997599363327026, + 3.1714054346084595, + 3.119111180305481, + 3.217866539955139, + 3.1822811365127563, + 3.212621569633484, + 3.1786141395568848, + 3.2456339597702026, + 3.2373738288879395, + 3.1844124794006348, + 3.2028744220733643, + 3.208802580833435, + 3.2397903203964233 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 112.61341631211903, + 69.62313474938047, + 55.80111351505194, + 49.574230919848915, + 45.03983984517833, + 41.862030791063106, + 39.80897956313427, + 38.079905403098145, + 36.7626309107031, + 35.0493389291677, + 34.13996323594311, + 33.1322106920102, + 32.464854631522776, + 31.839985501404286, + 31.08520446947901, + 30.920169225166067, + 30.550881496646515, + 30.436750427614857, + 30.35986848946529, + 30.33643866358181 + ], + "best_ppl": 30.33643866358181, + "train_time_s": 27790.683158159256 +} \ No newline at end of file diff --git a/063_sparse_dispatch/balanced_softmax_1plus2of5_best.pt b/063_sparse_dispatch/balanced_softmax_1plus2of5_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..708cf5d1fd44c8360c5e6872c3b9274a424388ce --- /dev/null +++ b/063_sparse_dispatch/balanced_softmax_1plus2of5_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bfeba44285941120e62717615ce4ebaaf37d9c52d5aac0ee2e4c27fffe00c75 +size 341413089 diff --git a/063_sparse_dispatch/balanced_softmax_1plus2of5_results.json b/063_sparse_dispatch/balanced_softmax_1plus2of5_results.json new file mode 100644 index 0000000000000000000000000000000000000000..36780ccebd7e01e5f31098e018b5540167642293 --- /dev/null +++ b/063_sparse_dispatch/balanced_softmax_1plus2of5_results.json @@ -0,0 +1,2056 @@ +{ + "label": "balanced_softmax_1plus2of5", + "seed": 42, + "balanced_shared_weight": true, + "total_params": 85293056, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.003496170043945, + 14.166847705841064, + 7.88243293762207, + 7.266923904418945, + 7.172660827636719, + 6.9758501052856445, + 6.840214252471924, + 6.704789161682129, + 6.6965131759643555, + 6.645236015319824, + 6.465904951095581, + 6.439613342285156, + 6.348862886428833, + 6.258130073547363, + 6.21268630027771, + 6.110261678695679, + 6.091821908950806, + 5.949145317077637, + 5.888932704925537, + 5.882192134857178, + 5.813299894332886, + 5.760247468948364, + 5.631585121154785, + 5.503554105758667, + 5.430958986282349, + 5.501707315444946, + 5.441454887390137, + 5.475198030471802, + 5.460506200790405, + 5.375519275665283, + 5.3609938621521, + 5.453240871429443, + 5.120209455490112, + 5.370307683944702, + 5.235085487365723, + 5.270686626434326, + 5.212702035903931, + 5.197176218032837, + 5.241120338439941, + 5.202385425567627, + 5.179021835327148, + 5.142108201980591, + 5.149211883544922, + 4.946668863296509, + 5.0246195793151855, + 5.046720027923584, + 5.090641021728516, + 5.184459924697876, + 4.922495603561401, + 4.970280647277832, + 4.9199512004852295, + 5.037330150604248, + 4.932050466537476, + 4.883550405502319, + 4.841316223144531, + 4.971362352371216, + 4.924697637557983, + 4.81965184211731, + 4.863598108291626, + 4.816957235336304, + 4.842488050460815, + 4.715836048126221, + 4.816137075424194, + 4.786607265472412, + 4.869043827056885, + 4.67882513999939, + 4.766928195953369, + 4.776235342025757, + 4.657484769821167, + 4.705994129180908, + 4.833116054534912, + 4.567715644836426, + 4.645092010498047, + 4.744460821151733, + 4.658832788467407, + 4.643839359283447, + 4.5918354988098145, + 4.753476858139038, + 4.754580974578857, + 4.668309450149536, + 4.50773811340332, + 4.65565824508667, + 4.619933128356934, + 4.746325731277466, + 4.551788091659546, + 4.531439542770386, + 4.401928186416626, + 4.663261413574219, + 4.564005613327026, + 4.591930866241455, + 4.33915376663208, + 4.476330518722534, + 4.472464084625244, + 4.565536737442017, + 4.53745436668396, + 4.602298021316528, + 4.472759485244751, + 4.517643451690674, + 4.460171222686768, + 4.407193422317505, + 4.409074783325195, + 4.58929705619812, + 4.518643140792847, + 4.419883966445923, + 4.547415018081665, + 4.471724271774292, + 4.512354373931885, + 4.406387567520142, + 4.542852163314819, + 4.407515525817871, + 4.311256408691406, + 4.50041127204895, + 4.443995714187622, + 4.47555947303772, + 4.4434287548065186, + 4.474149942398071, + 4.379198789596558, + 4.278962850570679, + 4.356207609176636, + 4.370499610900879, + 4.442741632461548, + 4.4381818771362305, + 4.318247079849243, + 4.500631809234619, + 4.423088550567627, + 4.417073011398315, + 4.40184211730957, + 4.315424919128418, + 4.303927183151245, + 4.379015207290649, + 4.199099540710449, + 4.370241403579712, + 4.4106810092926025, + 4.478646278381348, + 4.34366774559021, + 4.3012535572052, + 4.319376230239868, + 4.32060432434082, + 4.206883430480957, + 4.256082534790039, + 4.492998123168945, + 4.207131862640381, + 4.242155075073242, + 4.2074785232543945, + 4.356115102767944, + 4.402629613876343, + 4.148167848587036, + 4.2729408740997314, + 4.370410919189453, + 4.3023247718811035, + 4.277918100357056, + 4.193343877792358, + 4.208033323287964, + 4.228591442108154, + 4.253261566162109, + 4.126150369644165, + 4.179292798042297, + 4.1768646240234375, + 4.173889636993408, + 4.184645652770996, + 4.278455495834351, + 4.242055177688599, + 4.316927909851074, + 4.265687465667725, + 4.290700912475586, + 4.097663879394531, + 4.12557315826416, + 4.167634725570679, + 4.180511951446533, + 4.151561737060547, + 4.130006313323975, + 4.267870187759399, + 4.084503412246704, + 4.2071428298950195, + 4.277455806732178, + 4.306017160415649, + 4.151503801345825, + 4.169181823730469, + 4.182326316833496, + 4.165094375610352, + 4.246058702468872, + 4.12325918674469, + 4.236901044845581, + 4.152280330657959, + 4.206755638122559, + 4.0776753425598145, + 4.223975896835327, + 4.103104829788208, + 4.174492359161377, + 4.179409027099609, + 4.210444211959839, + 4.11644721031189, + 4.166521072387695, + 4.078541278839111, + 4.210438251495361, + 4.16995096206665, + 4.157559633255005, + 4.220724105834961, + 4.099176406860352, + 4.081532001495361, + 4.133792161941528, + 3.998894453048706, + 4.001245141029358, + 4.085070371627808, + 4.1440911293029785, + 4.047072649002075, + 3.9513182640075684, + 4.021229982376099, + 4.174240350723267, + 4.106481552124023, + 4.131633758544922, + 4.068526387214661, + 4.018041253089905, + 3.9739469289779663, + 4.039098501205444, + 3.838638663291931, + 4.069210767745972, + 4.037926197052002, + 3.9134066104888916, + 4.161969423294067, + 4.028848648071289, + 4.1215434074401855, + 4.172611474990845, + 4.007876396179199, + 4.16370415687561, + 4.033615350723267, + 4.02206289768219, + 4.023914098739624, + 4.089343309402466, + 3.975265860557556, + 4.081206798553467, + 4.156308889389038, + 3.9849976301193237, + 4.0361082553863525, + 3.953459858894348, + 4.053573131561279, + 4.0199501514434814, + 3.9986472129821777, + 4.029232740402222, + 4.038950800895691, + 4.119354248046875, + 3.981931209564209, + 4.0414345264434814, + 4.013686418533325, + 3.80029559135437, + 4.148109674453735, + 3.9464951753616333, + 4.002118825912476, + 3.9593957662582397, + 4.033435344696045, + 4.006327271461487, + 3.9663279056549072, + 3.9178009033203125, + 3.97066330909729, + 3.880553364753723, + 3.9014742374420166, + 3.9869340658187866, + 4.0253249406814575, + 3.8573269844055176, + 4.003081321716309, + 4.0022382736206055, + 3.9525771141052246, + 3.9264460802078247, + 3.9595009088516235, + 3.94770348072052, + 4.037136793136597, + 3.9167940616607666, + 3.842689037322998, + 3.760077953338623, + 3.9741228818893433, + 3.956446647644043, + 3.9221901893615723, + 4.00365686416626, + 3.9001611471176147, + 4.002708673477173, + 4.04225754737854, + 3.9372406005859375, + 3.9848052263259888, + 3.9357097148895264, + 4.0321433544158936, + 3.8863577842712402, + 3.8780020475387573, + 3.950118064880371, + 3.8589898347854614, + 3.8682639598846436, + 3.866114020347595, + 3.861698627471924, + 3.881505012512207, + 3.912028193473816, + 3.8595197200775146, + 3.9069257974624634, + 4.047906994819641, + 3.8292418718338013, + 3.9920616149902344, + 3.8416764736175537, + 3.9298750162124634, + 4.028677463531494, + 3.954358458518982, + 3.9422580003738403, + 4.047253370285034, + 3.975367546081543, + 3.928210735321045, + 3.8941597938537598, + 3.92824649810791, + 3.8992336988449097, + 3.9531452655792236, + 3.8464267253875732, + 3.929287314414978, + 3.8930609226226807, + 3.876767158508301, + 3.899361491203308, + 3.863723039627075, + 3.8354835510253906, + 3.831938147544861, + 4.014564871788025, + 3.9641538858413696, + 3.808051109313965, + 3.9814794063568115, + 3.886781930923462, + 3.7726110219955444, + 4.010114908218384, + 3.8651323318481445, + 4.0049638748168945, + 3.8921691179275513, + 3.9478747844696045, + 3.949902296066284, + 3.8996262550354004, + 4.004582047462463, + 3.8922585248947144, + 3.9569824934005737, + 3.832594871520996, + 3.7919121980667114, + 3.8709518909454346, + 3.810159206390381, + 3.876504898071289, + 3.9495118856430054, + 3.919263005256653, + 3.952997326850891, + 3.9810681343078613, + 3.8737324476242065, + 3.8112844228744507, + 4.001265645027161, + 3.683832287788391, + 3.9652223587036133, + 3.7805354595184326, + 3.8203035593032837, + 3.791541576385498, + 3.8846248388290405, + 3.8008241653442383, + 3.7709295749664307, + 3.838513970375061, + 3.924919009208679, + 3.9250903129577637, + 3.794939160346985, + 3.946478009223938, + 3.7589980363845825, + 3.923753023147583, + 3.8053935766220093, + 3.835451364517212, + 3.8826276063919067, + 3.9116854667663574, + 3.7537238597869873, + 3.839272141456604, + 3.8773553371429443, + 3.8719006776809692, + 3.8724958896636963, + 3.8871755599975586, + 3.901718497276306, + 3.871931552886963, + 3.88154137134552, + 3.9441572427749634, + 3.854166269302368, + 3.909159541130066, + 3.9249861240386963, + 3.9103513956069946, + 3.8828970193862915, + 3.862390398979187, + 3.91011118888855, + 3.8837331533432007, + 3.7313159704208374, + 3.8178985118865967, + 3.753087878227234, + 3.800571084022522, + 3.90202260017395, + 3.627692937850952, + 3.6159908771514893, + 3.8572752475738525, + 3.70142662525177, + 3.727233052253723, + 3.8103013038635254, + 3.7627055644989014, + 3.6907254457473755, + 3.7282944917678833, + 3.7050588130950928, + 3.731243848800659, + 3.7645559310913086, + 3.7567286491394043, + 3.756168007850647, + 3.8725125789642334, + 3.8042815923690796, + 3.718317151069641, + 3.7463464736938477, + 3.8013412952423096, + 3.8504233360290527, + 3.5721654891967773, + 3.8830082416534424, + 3.6771295070648193, + 3.733334183692932, + 3.7111470699310303, + 3.739341139793396, + 3.6451252698898315, + 3.8705583810806274, + 3.7368885278701782, + 3.688987135887146, + 3.7667641639709473, + 3.789336681365967, + 3.623471736907959, + 3.8257346153259277, + 3.8694300651550293, + 3.736946225166321, + 3.6780089139938354, + 3.8413796424865723, + 3.712794542312622, + 3.7489415407180786, + 3.8049575090408325, + 3.725799322128296, + 3.7670358419418335, + 3.7419283390045166, + 3.7135192155838013, + 3.7975170612335205, + 3.740403890609741, + 3.6495413780212402, + 3.6553674936294556, + 3.632391333580017, + 3.8244224786758423, + 3.6821409463882446, + 3.7675318717956543, + 3.881359338760376, + 3.7816351652145386, + 3.830204129219055, + 3.7067097425460815, + 3.7333778142929077, + 3.8495757579803467, + 3.639036774635315, + 3.754597544670105, + 3.7170149087905884, + 3.7345083951950073, + 3.793953061103821, + 3.7450764179229736, + 3.7622846364974976, + 3.8628040552139282, + 3.8117698431015015, + 3.812092900276184, + 3.748150944709778, + 3.7776143550872803, + 3.728775143623352, + 3.8863197565078735, + 3.7705260515213013, + 3.6901984214782715, + 3.714751124382019, + 3.7024868726730347, + 3.7988672256469727, + 3.661667823791504, + 3.801621913909912, + 3.688439726829529, + 3.636989116668701, + 3.7969117164611816, + 3.6962698698043823, + 3.7174112796783447, + 3.7395213842391968, + 3.732521414756775, + 3.788118600845337, + 3.801681876182556, + 3.6298816204071045, + 3.705265998840332, + 3.7526358366012573, + 3.7465529441833496, + 3.6223713159561157, + 3.7063316106796265, + 3.7153677940368652, + 3.6722687482833862, + 3.8147857189178467, + 3.6401809453964233, + 3.6987727880477905, + 3.7599090337753296, + 3.709645986557007, + 3.7283411026000977, + 3.716724157333374, + 3.7018110752105713, + 3.6726468801498413, + 3.7563791275024414, + 3.8351924419403076, + 3.7154301404953003, + 3.6917611360549927, + 3.8154876232147217, + 3.7537120580673218, + 3.708288073539734, + 3.6474361419677734, + 3.7847273349761963, + 3.74926495552063, + 3.716661214828491, + 3.6885043382644653, + 3.73895800113678, + 3.6867412328720093, + 3.7706079483032227, + 3.7503209114074707, + 3.5658140182495117, + 3.5105291604995728, + 3.5763814449310303, + 3.657581329345703, + 3.5840561389923096, + 3.6563161611557007, + 3.708793878555298, + 3.64100444316864, + 3.6628806591033936, + 3.6464911699295044, + 3.6405177116394043, + 3.666326403617859, + 3.7255587577819824, + 3.570039987564087, + 3.6447603702545166, + 3.5501734018325806, + 3.7739548683166504, + 3.624989151954651, + 3.587247848510742, + 3.5506750345230103, + 3.7268335819244385, + 3.5539071559906006, + 3.656806468963623, + 3.657961845397949, + 3.5990341901779175, + 3.4913419485092163, + 3.6625068187713623, + 3.6027008295059204, + 3.700054168701172, + 3.683324694633484, + 3.6768205165863037, + 3.6782220602035522, + 3.568033218383789, + 3.711235523223877, + 3.6058762073516846, + 3.6227976083755493, + 3.4115577936172485, + 3.6886295080184937, + 3.5863754749298096, + 3.647623300552368, + 3.7721697092056274, + 3.604224443435669, + 3.5800392627716064, + 3.6409977674484253, + 3.6314462423324585, + 3.581998109817505, + 3.537149429321289, + 3.709749698638916, + 3.49999737739563, + 3.52048921585083, + 3.6649537086486816, + 3.6424981355667114, + 3.681482195854187, + 3.6772751808166504, + 3.6748690605163574, + 3.6298121213912964, + 3.602896213531494, + 3.7416787147521973, + 3.558492422103882, + 3.556758999824524, + 3.689899206161499, + 3.458741307258606, + 3.6540446281433105, + 3.6286641359329224, + 3.6724411249160767, + 3.6476269960403442, + 3.6699676513671875, + 3.6657660007476807, + 3.620723605155945, + 3.450077772140503, + 3.4767708778381348, + 3.755632162094116, + 3.49981951713562, + 3.699729800224304, + 3.4744688272476196, + 3.6901313066482544, + 3.657062292098999, + 3.600037455558777, + 3.509733557701111, + 3.6003963947296143, + 3.483408212661743, + 3.6753145456314087, + 3.6231155395507812, + 3.661076068878174, + 3.595944046974182, + 3.605983853340149, + 3.617904305458069, + 3.634984016418457, + 3.6373705863952637, + 3.6426724195480347, + 3.607192039489746, + 3.5985406637191772, + 3.6584768295288086, + 3.6164783239364624, + 3.5671815872192383, + 3.5892248153686523, + 3.4531384706497192, + 3.6237716674804688, + 3.586065173149109, + 3.6649004220962524, + 3.502446174621582, + 3.503793954849243, + 3.6621134281158447, + 3.306742787361145, + 3.492893695831299, + 3.678791046142578, + 3.617958903312683, + 3.5662142038345337, + 3.715579867362976, + 3.615195870399475, + 3.658796191215515, + 3.7655553817749023, + 3.6078981161117554, + 3.626962900161743, + 3.6431586742401123, + 3.4879757165908813, + 3.671836018562317, + 3.648162364959717, + 3.674996256828308, + 3.634280800819397, + 3.6118255853652954, + 3.5110177993774414, + 3.5596373081207275, + 3.5963674783706665, + 3.5939457416534424, + 3.50605309009552, + 3.607396364212036, + 3.265288472175598, + 3.640688419342041, + 3.457732319831848, + 3.5898244380950928, + 3.5632688999176025, + 3.5311933755874634, + 3.409720778465271, + 3.542514443397522, + 3.6073344945907593, + 3.537306070327759, + 3.528507709503174, + 3.4739418029785156, + 3.519754409790039, + 3.4846800565719604, + 3.557201623916626, + 3.490255832672119, + 3.5005778074264526, + 3.5805130004882812, + 3.4474825859069824, + 3.559253215789795, + 3.5705093145370483, + 3.3926122188568115, + 3.6204291582107544, + 3.429872751235962, + 3.5972421169281006, + 3.469539761543274, + 3.467741370201111, + 3.5479238033294678, + 3.5508052110671997, + 3.5216920375823975, + 3.689570188522339, + 3.4204368591308594, + 3.470265030860901, + 3.4960931539535522, + 3.5246394872665405, + 3.5499589443206787, + 3.6338753700256348, + 3.5309377908706665, + 3.3673490285873413, + 3.572025418281555, + 3.6261065006256104, + 3.4873080253601074, + 3.6680057048797607, + 3.5417271852493286, + 3.596471905708313, + 3.5357303619384766, + 3.5540125370025635, + 3.59277880191803, + 3.610098123550415, + 3.548797130584717, + 3.516157865524292, + 3.414189577102661, + 3.480779767036438, + 3.4734532833099365, + 3.441915273666382, + 3.5326048135757446, + 3.397330403327942, + 3.5132617950439453, + 3.578986406326294, + 3.5726274251937866, + 3.5296164751052856, + 3.6379629373550415, + 3.5492446422576904, + 3.571678638458252, + 3.5112240314483643, + 3.4913249015808105, + 3.525425672531128, + 3.4511457681655884, + 3.593566417694092, + 3.4528214931488037, + 3.5859628915786743, + 3.590737223625183, + 3.50448215007782, + 3.4424346685409546, + 3.581613779067993, + 3.5927069187164307, + 3.494994640350342, + 3.5293734073638916, + 3.5299179553985596, + 3.574876070022583, + 3.557406783103943, + 3.6174814701080322, + 3.531668543815613, + 3.54925537109375, + 3.610634684562683, + 3.519826650619507, + 3.5695310831069946, + 3.526548743247986, + 3.617580771446228, + 3.4512909650802612, + 3.604657292366028, + 3.404452681541443, + 3.4920769929885864, + 3.4873162508010864, + 3.503808617591858, + 3.495560884475708, + 3.576878070831299, + 3.4986188411712646, + 3.5636024475097656, + 3.3328932523727417, + 3.684823989868164, + 3.5538171529769897, + 3.4567534923553467, + 3.545827269554138, + 3.454236149787903, + 3.464126944541931, + 3.557331085205078, + 3.4011529684066772, + 3.4259246587753296, + 3.4983516931533813, + 3.5381988286972046, + 3.485522150993347, + 3.5295501947402954, + 3.4964585304260254, + 3.557851791381836, + 3.590960741043091, + 3.574462056159973, + 3.521818518638611, + 3.5668728351593018, + 3.4310495853424072, + 3.4197232723236084, + 3.4687232971191406, + 3.5024839639663696, + 3.552349090576172, + 3.450604796409607, + 3.565437078475952, + 3.3737070560455322, + 3.495754837989807, + 3.46041202545166, + 3.3667439222335815, + 3.4482988119125366, + 3.472194790840149, + 3.385809898376465, + 3.4077470302581787, + 3.564046859741211, + 3.423558235168457, + 3.485205888748169, + 3.480081796646118, + 3.5443164110183716, + 3.5817357301712036, + 3.519318699836731, + 3.468448758125305, + 3.438006281852722, + 3.6379162073135376, + 3.3355950117111206, + 3.4428482055664062, + 3.376372218132019, + 3.4581209421157837, + 3.492584228515625, + 3.5239278078079224, + 3.4178539514541626, + 3.428782105445862, + 3.3555115461349487, + 3.3320682048797607, + 3.3781144618988037, + 3.503637194633484, + 3.3743181228637695, + 3.4295473098754883, + 3.391198515892029, + 3.4989105463027954, + 3.4542492628097534, + 3.483052372932434, + 3.3985960483551025, + 3.4459304809570312, + 3.468077301979065, + 3.2409812211990356, + 3.4338470697402954, + 3.482508659362793, + 3.601600766181946, + 3.4861010313034058, + 3.346140503883362, + 3.4192830324172974, + 3.421652317047119, + 3.3352324962615967, + 3.541792154312134, + 3.3344866037368774, + 3.505992293357849, + 3.4716747999191284, + 3.363161087036133, + 3.509081482887268, + 3.522454619407654, + 3.368965744972229, + 3.492730498313904, + 3.4766249656677246, + 3.5104199647903442, + 3.6364229917526245, + 3.4154046773910522, + 3.239776134490967, + 3.3760263919830322, + 3.526918411254883, + 3.432717800140381, + 3.4491909742355347, + 3.4198614358901978, + 3.475834608078003, + 3.418826460838318, + 3.4074782133102417, + 3.463393449783325, + 3.3870012760162354, + 3.4373713731765747, + 3.4045023918151855, + 3.507096767425537, + 3.4559595584869385, + 3.4361770153045654, + 3.5411664247512817, + 3.3738235235214233, + 3.4387553930282593, + 3.4846279621124268, + 3.496446132659912, + 3.405899167060852, + 3.4516549110412598, + 3.573423385620117, + 3.4886802434921265, + 3.532185673713684, + 3.6126149892807007, + 3.424368977546692, + 3.4338427782058716, + 3.5659672021865845, + 3.4892842769622803, + 3.4609375, + 3.513348698616028, + 3.383841872215271, + 3.4895737171173096, + 3.5384992361068726, + 3.469638705253601, + 3.3664650917053223, + 3.3126879930496216, + 3.417705774307251, + 3.304863214492798, + 3.3543163537979126, + 3.464027762413025, + 3.42568039894104, + 3.4631141424179077, + 3.555903673171997, + 3.5400571823120117, + 3.461397647857666, + 3.496762275695801, + 3.449206233024597, + 3.347755789756775, + 3.3756309747695923, + 3.513399124145508, + 3.5189214944839478, + 3.55666446685791, + 3.387305974960327, + 3.4060380458831787, + 3.4594327211380005, + 3.3789647817611694, + 3.4564353227615356, + 3.5202490091323853, + 3.372259497642517, + 3.5458552837371826, + 3.520488977432251, + 3.3470983505249023, + 3.3074498176574707, + 3.5191287994384766, + 3.487565755844116, + 3.4441237449645996, + 3.4632694721221924, + 3.4605648517608643, + 3.3645142316818237, + 3.344731569290161, + 3.5416605472564697, + 3.4130699634552, + 3.3245604038238525, + 3.5233373641967773, + 3.356065273284912, + 3.435474991798401, + 3.4296822547912598, + 3.4267700910568237, + 3.30202853679657, + 3.455188512802124, + 3.3936468362808228, + 3.3645185232162476, + 3.4740490913391113, + 3.4329806566238403, + 3.3520278930664062, + 3.471078395843506, + 3.3744380474090576, + 3.39512836933136, + 3.463034510612488, + 3.397561192512512, + 3.448794960975647, + 3.4487465620040894, + 3.471897840499878, + 3.4635508060455322, + 3.4522974491119385, + 3.305882215499878, + 3.4317935705184937, + 3.41694974899292, + 3.4611191749572754, + 3.32020902633667, + 3.3419657945632935, + 3.389171838760376, + 3.341762661933899, + 3.504210352897644, + 3.4612826108932495, + 3.379102110862732, + 3.510753035545349, + 3.428085207939148, + 3.4657868146896362, + 3.440286636352539, + 3.4818111658096313, + 3.4916958808898926, + 3.4575140476226807, + 3.4788553714752197, + 3.4320470094680786, + 3.408298373222351, + 3.4481221437454224, + 3.3860143423080444, + 3.3784842491149902, + 3.4207818508148193, + 3.3120553493499756, + 3.407226324081421, + 3.350026249885559, + 3.379809617996216, + 3.3924111127853394, + 3.3257055282592773, + 3.297686815261841, + 3.492753267288208, + 3.327759265899658, + 3.3603144884109497, + 3.402890205383301, + 3.39008104801178, + 3.3150984048843384, + 3.3606677055358887, + 3.4505865573883057, + 3.4383832216262817, + 3.457379698753357, + 3.398754358291626, + 3.4356777667999268, + 3.5466309785842896, + 3.309173822402954, + 3.3663567304611206, + 3.4380797147750854, + 3.3457316160202026, + 3.3307905197143555, + 3.249606132507324, + 3.4630117416381836, + 3.3174028396606445, + 3.438191771507263, + 3.4029186964035034, + 3.41845440864563, + 3.4075595140457153, + 3.4226038455963135, + 3.4245940446853638, + 3.3140642642974854, + 3.4679365158081055, + 3.4192752838134766, + 3.4094818830490112, + 3.386574149131775, + 3.5745065212249756, + 3.4771015644073486, + 3.320472240447998, + 3.4526439905166626, + 3.439746618270874, + 3.4454751014709473, + 3.3111438751220703, + 3.4742432832717896, + 3.3002140522003174, + 3.449667811393738, + 3.4319989681243896, + 3.4822548627853394, + 3.587804913520813, + 3.492327570915222, + 3.4259952306747437, + 3.548733711242676, + 3.4201899766921997, + 3.431657552719116 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 128.02676729069574, + 80.64404987535173, + 62.98342636882259, + 55.089174507466524, + 49.72350024262939, + 46.49195923213775, + 43.59797406683537, + 41.70351087995402, + 39.69026242412459, + 38.03267571913264, + 37.13190440726017, + 35.766060328765604, + 34.93583472576851, + 34.07142402931105, + 33.396101740676755, + 33.00819648756424, + 32.60378132814218, + 32.41589997100834, + 32.29595773525427, + 32.282018974945395 + ], + "best_ppl": 32.282018974945395, + "train_time_s": 20888.069296360016 +} \ No newline at end of file diff --git a/063_sparse_dispatch/deltanet_3of15_best.pt b/063_sparse_dispatch/deltanet_3of15_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..5afd5d4b6d1d03308160fe3e6fc814b624584af7 --- /dev/null +++ b/063_sparse_dispatch/deltanet_3of15_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56a9d8aa3bccea6765d9f433dc179c8e28a68fb68d7632f4ad6e1e21486ad924 +size 795875344 diff --git a/063_sparse_dispatch/deltanet_3of15_results.json b/063_sparse_dispatch/deltanet_3of15_results.json new file mode 100644 index 0000000000000000000000000000000000000000..d3635f6094b3401fd69277922ac444295b49910c --- /dev/null +++ b/063_sparse_dispatch/deltanet_3of15_results.json @@ -0,0 +1,2059 @@ +{ + "label": "deltanet_3of15", + "seed": 42, + "attention_type": "deltanet", + "num_thin_blocks": 15, + "num_shared_blocks": 0, + "top_k_blocks": 3, + "total_params": 198728128, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 18.953080654144287, + 15.959983110427856, + 8.29456877708435, + 7.2259567975997925, + 7.084102630615234, + 6.920876502990723, + 6.828663468360901, + 6.653005361557007, + 6.562945604324341, + 6.472415447235107, + 6.49503219127655, + 6.375160217285156, + 6.193181872367859, + 6.114583492279053, + 6.10823667049408, + 6.0928508043289185, + 5.92290461063385, + 5.950495600700378, + 5.815910458564758, + 5.820511817932129, + 5.657215595245361, + 5.79017972946167, + 5.754097104072571, + 5.5260783433914185, + 5.562936186790466, + 5.476665019989014, + 5.410043716430664, + 5.440510272979736, + 5.481915593147278, + 5.434414744377136, + 5.343634128570557, + 5.307816743850708, + 5.206463694572449, + 5.3008341789245605, + 5.228801131248474, + 5.214430332183838, + 5.046915650367737, + 5.106627106666565, + 5.055347561836243, + 5.0962220430374146, + 5.0419358015060425, + 4.996540784835815, + 5.073680400848389, + 4.898923873901367, + 5.055986642837524, + 5.02150571346283, + 4.958790063858032, + 5.0382795333862305, + 4.995933532714844, + 4.939785480499268, + 4.906861901283264, + 4.802058458328247, + 5.0391682386398315, + 4.854848623275757, + 4.86903703212738, + 4.807073473930359, + 4.782943606376648, + 4.757477641105652, + 4.854511857032776, + 4.693225264549255, + 4.734682679176331, + 4.7526233196258545, + 4.795349597930908, + 4.753857612609863, + 4.6908111572265625, + 4.825307846069336, + 4.800307273864746, + 4.674037218093872, + 4.742711186408997, + 4.649279356002808, + 4.690407752990723, + 4.446491360664368, + 4.807615399360657, + 4.590352535247803, + 4.5696223974227905, + 4.743587136268616, + 4.582673788070679, + 4.482483744621277, + 4.528334140777588, + 4.438618421554565, + 4.491156697273254, + 4.580667972564697, + 4.455562114715576, + 4.518878102302551, + 4.580722808837891, + 4.693718314170837, + 4.570234894752502, + 4.4415106773376465, + 4.548556804656982, + 4.528740763664246, + 4.4739543199539185, + 4.523414611816406, + 4.42387592792511, + 4.487061381340027, + 4.49917471408844, + 4.398936986923218, + 4.403285503387451, + 4.4296183586120605, + 4.486264705657959, + 4.3300312757492065, + 4.442367315292358, + 4.328650593757629, + 4.30688214302063, + 4.531997084617615, + 4.442355632781982, + 4.313724398612976, + 4.456803202629089, + 4.326036095619202, + 4.404494762420654, + 4.424278378486633, + 4.411131381988525, + 4.4622015953063965, + 4.325833439826965, + 4.409428954124451, + 4.307608008384705, + 4.39563512802124, + 4.264089941978455, + 4.4309916496276855, + 4.317338347434998, + 4.322222828865051, + 4.193993926048279, + 4.251686096191406, + 4.352545261383057, + 4.254119038581848, + 4.401811957359314, + 4.208304524421692, + 4.297709107398987, + 4.240981817245483, + 4.122017502784729, + 4.213692665100098, + 4.161878943443298, + 4.22728967666626, + 4.319410562515259, + 4.3004865646362305, + 4.3571871519088745, + 4.165244460105896, + 4.156403303146362, + 4.282574772834778, + 4.104346752166748, + 4.316551446914673, + 4.112705290317535, + 4.178110599517822, + 4.241247653961182, + 4.340431571006775, + 4.29747748374939, + 3.988399624824524, + 4.1485098004341125, + 4.215573310852051, + 4.1762025356292725, + 4.251799523830414, + 4.0520713329315186, + 4.025966286659241, + 4.214555859565735, + 4.212492346763611, + 4.192029237747192, + 4.1953805685043335, + 4.206493496894836, + 4.203598260879517, + 4.081656336784363, + 4.179298281669617, + 4.132426977157593, + 4.12423437833786, + 4.219385981559753, + 4.103232026100159, + 4.158239364624023, + 4.032073199748993, + 4.0941683650016785, + 4.106715440750122, + 4.09070098400116, + 4.083575367927551, + 4.165720582008362, + 4.18952739238739, + 4.1233203411102295, + 4.120022118091583, + 4.135205626487732, + 4.200562834739685, + 4.167716801166534, + 4.135865569114685, + 4.079260349273682, + 3.986903727054596, + 4.154362678527832, + 3.9527488350868225, + 4.074076235294342, + 4.121735572814941, + 4.051144599914551, + 4.078164339065552, + 4.199310660362244, + 3.985551655292511, + 4.091814994812012, + 4.1099637150764465, + 4.149719774723053, + 4.16594672203064, + 4.050686597824097, + 4.035383224487305, + 4.170611262321472, + 4.095017075538635, + 4.000961720943451, + 4.0421977043151855, + 4.012477874755859, + 4.052110016345978, + 4.006635785102844, + 4.07734489440918, + 4.182418465614319, + 4.031980872154236, + 3.9384633898735046, + 4.10716837644577, + 3.9476444721221924, + 4.014200031757355, + 4.056074798107147, + 3.962430477142334, + 3.993980824947357, + 4.064448893070221, + 4.068450570106506, + 4.035200476646423, + 4.125722527503967, + 3.982472062110901, + 3.9470152854919434, + 4.037098467350006, + 4.0633867383003235, + 3.972408890724182, + 4.058974862098694, + 4.154379367828369, + 4.119762182235718, + 4.01295930147171, + 3.919037163257599, + 3.926827311515808, + 4.089035511016846, + 3.938905894756317, + 3.9040045738220215, + 3.976610839366913, + 3.8052220940589905, + 4.100192964076996, + 4.041523098945618, + 3.9480522871017456, + 4.048669397830963, + 3.9671950936317444, + 4.157432019710541, + 4.020190179347992, + 3.926209270954132, + 4.039573073387146, + 4.139577627182007, + 4.056957006454468, + 4.06357741355896, + 4.006599605083466, + 4.035916328430176, + 3.9626957178115845, + 4.109623968601227, + 3.9566677808761597, + 3.923959493637085, + 3.878528118133545, + 3.9362140893936157, + 3.946616768836975, + 3.9821234345436096, + 3.8178048729896545, + 3.847028434276581, + 3.8626312017440796, + 3.9315600991249084, + 3.8896178603172302, + 3.908646523952484, + 3.9422285556793213, + 3.8984087109565735, + 3.8750551342964172, + 3.9742186069488525, + 3.8892889618873596, + 3.789942681789398, + 3.8109180331230164, + 3.808179795742035, + 3.808871328830719, + 3.8845539689064026, + 3.841197669506073, + 3.8368334770202637, + 3.954772412776947, + 3.8866323828697205, + 3.909105956554413, + 3.7691203951835632, + 3.9422662258148193, + 3.822555661201477, + 3.7678354382514954, + 3.785534679889679, + 4.049133718013763, + 3.8939995765686035, + 3.7996917366981506, + 3.859772801399231, + 4.003019869327545, + 3.9128443002700806, + 3.8269848823547363, + 3.8889142870903015, + 3.755826413631439, + 3.873141586780548, + 3.69486665725708, + 3.87457537651062, + 3.8454381227493286, + 3.9113078117370605, + 3.7558470368385315, + 3.8707064986228943, + 3.9972482323646545, + 3.7527304887771606, + 3.814167618751526, + 3.826689839363098, + 3.87725430727005, + 3.8865336775779724, + 3.801650822162628, + 3.836603105068207, + 3.8022965788841248, + 4.021027565002441, + 3.7210699319839478, + 4.028819262981415, + 3.909795045852661, + 3.8383787870407104, + 3.9293293356895447, + 3.933139741420746, + 3.646544337272644, + 3.8374019861221313, + 3.860632061958313, + 3.8764231204986572, + 3.853683888912201, + 3.8830080032348633, + 3.8770431876182556, + 3.900580585002899, + 3.933069348335266, + 3.8121694326400757, + 4.0103952288627625, + 3.8611910343170166, + 4.063252687454224, + 3.830340564250946, + 3.7811226844787598, + 3.8764195442199707, + 3.873583495616913, + 3.6861196160316467, + 3.926840901374817, + 3.8445897102355957, + 3.7925488352775574, + 3.8849103450775146, + 3.7834157943725586, + 3.892871081829071, + 3.7571728825569153, + 3.8844470977783203, + 3.8418497443199158, + 3.690878450870514, + 3.849420487880707, + 3.711365044116974, + 3.8178272247314453, + 3.9013646245002747, + 3.759451925754547, + 3.7091481685638428, + 3.9111629128456116, + 3.6992788314819336, + 3.847055673599243, + 3.8670178055763245, + 3.8993095755577087, + 3.799022912979126, + 3.892583906650543, + 3.8573980927467346, + 3.739738702774048, + 3.8394317030906677, + 3.66049587726593, + 3.773229718208313, + 3.7573529481887817, + 3.8468732237815857, + 3.749920070171356, + 3.6403629779815674, + 3.8892393112182617, + 3.741956412792206, + 3.7325371503829956, + 3.744070291519165, + 3.7938790917396545, + 3.7902987599372864, + 3.6902425289154053, + 3.8012797236442566, + 3.7749626636505127, + 3.693065583705902, + 3.8129884004592896, + 3.7882151007652283, + 3.6456069350242615, + 3.8572720289230347, + 3.8608577251434326, + 3.8433912992477417, + 3.7778209447860718, + 3.626061737537384, + 3.6881784200668335, + 3.6843372583389282, + 3.6934499740600586, + 3.5777719616889954, + 3.6475815773010254, + 3.7499354481697083, + 3.711522936820984, + 3.7519057393074036, + 3.828204870223999, + 3.6868902444839478, + 3.698460102081299, + 3.7534674406051636, + 3.611026704311371, + 3.6388851404190063, + 3.611998438835144, + 3.6331703066825867, + 3.761093556880951, + 3.6522327065467834, + 3.773572266101837, + 3.7080304622650146, + 3.645772635936737, + 3.557802736759186, + 3.6376054286956787, + 3.643476128578186, + 3.72699636220932, + 3.7539819478988647, + 3.862342655658722, + 3.6864930987358093, + 3.767989456653595, + 3.62359619140625, + 3.704304039478302, + 3.7294185757637024, + 3.801201105117798, + 3.6821661591529846, + 3.667479157447815, + 3.691497266292572, + 3.6929412484169006, + 3.703488767147064, + 3.6267287135124207, + 3.6017042994499207, + 3.748997688293457, + 3.644925355911255, + 3.5005279779434204, + 3.5279242992401123, + 3.6312599182128906, + 3.5903208255767822, + 3.5090290904045105, + 3.6136701703071594, + 3.6388036608695984, + 3.7725391387939453, + 3.6772698760032654, + 3.5451143383979797, + 3.6448978781700134, + 3.6340051889419556, + 3.640730381011963, + 3.783643960952759, + 3.711569368839264, + 3.724964439868927, + 3.638049602508545, + 3.791511297225952, + 3.542078971862793, + 3.705964684486389, + 3.6397237181663513, + 3.6420556902885437, + 3.5760682821273804, + 3.6457292437553406, + 3.6555559635162354, + 3.6524031162261963, + 3.6659326553344727, + 3.5028952956199646, + 3.607978582382202, + 3.583121657371521, + 3.7734246850013733, + 3.6142609119415283, + 3.73714542388916, + 3.7316982746124268, + 3.7112680077552795, + 3.6311649084091187, + 3.671261429786682, + 3.6759312748908997, + 3.6823930740356445, + 3.6932790875434875, + 3.7261385321617126, + 3.693838059902191, + 3.5776743292808533, + 3.6134549975395203, + 3.6012414693832397, + 3.6661933064460754, + 3.7292933464050293, + 3.6056785583496094, + 3.7002965211868286, + 3.62629234790802, + 3.544073522090912, + 3.7703563570976257, + 3.711623251438141, + 3.78527569770813, + 3.646434247493744, + 3.747985064983368, + 3.6344525814056396, + 3.6249324679374695, + 3.796490490436554, + 3.6793823838233948, + 3.6335939168930054, + 3.7155290842056274, + 3.7162431478500366, + 3.7316508889198303, + 3.663343071937561, + 3.683036983013153, + 3.641298532485962, + 3.8198089599609375, + 3.6524817943573, + 3.668393135070801, + 3.610043227672577, + 3.7605130672454834, + 3.7219399213790894, + 3.60166996717453, + 3.7476730942726135, + 3.62516325712204, + 3.722872018814087, + 3.6625848412513733, + 3.702233850955963, + 3.6870571970939636, + 3.7054402828216553, + 3.696231424808502, + 3.556131422519684, + 3.619132101535797, + 3.7500232458114624, + 3.630948007106781, + 3.540121614933014, + 3.6246184706687927, + 3.4368461966514587, + 3.529137909412384, + 3.5999176502227783, + 3.4133660197257996, + 3.5786798000335693, + 3.5887950658798218, + 3.508126199245453, + 3.602358877658844, + 3.734651744365692, + 3.5007349252700806, + 3.616150379180908, + 3.564991295337677, + 3.623863637447357, + 3.5146694779396057, + 3.489324986934662, + 3.5976467728614807, + 3.5551225543022156, + 3.6198169589042664, + 3.5029646158218384, + 3.459853231906891, + 3.4043856263160706, + 3.497139275074005, + 3.554706573486328, + 3.5867748260498047, + 3.5349740982055664, + 3.5989134311676025, + 3.5428449511528015, + 3.66421377658844, + 3.5625290274620056, + 3.5014071464538574, + 3.524748921394348, + 3.436324954032898, + 3.587723195552826, + 3.5129461884498596, + 3.6314201951026917, + 3.6877312660217285, + 3.5132423639297485, + 3.521744728088379, + 3.5027453303337097, + 3.43914794921875, + 3.5743351578712463, + 3.5305062532424927, + 3.420307993888855, + 3.602595031261444, + 3.476470410823822, + 3.5349716544151306, + 3.5207645297050476, + 3.46427845954895, + 3.4023467898368835, + 3.626731276512146, + 3.5649150013923645, + 3.477690577507019, + 3.531040847301483, + 3.60806542634964, + 3.5251739025115967, + 3.506921648979187, + 3.4979716539382935, + 3.6278706789016724, + 3.434842884540558, + 3.536092162132263, + 3.524833858013153, + 3.528550446033478, + 3.540393590927124, + 3.57684463262558, + 3.484550714492798, + 3.3712841272354126, + 3.628976881504059, + 3.4295572638511658, + 3.571745216846466, + 3.5214218497276306, + 3.5350394248962402, + 3.662238299846649, + 3.5204588770866394, + 3.541739821434021, + 3.4336551427841187, + 3.4588209986686707, + 3.4846100211143494, + 3.4413461685180664, + 3.490779459476471, + 3.5641881823539734, + 3.464996576309204, + 3.512079358100891, + 3.562376618385315, + 3.457877993583679, + 3.400874674320221, + 3.4133344888687134, + 3.4792513847351074, + 3.5485385060310364, + 3.4972627758979797, + 3.555534839630127, + 3.5645068287849426, + 3.692218601703644, + 3.502536118030548, + 3.573233425617218, + 3.4936848878860474, + 3.602371335029602, + 3.6078333854675293, + 3.3998392820358276, + 3.5780243277549744, + 3.666493058204651, + 3.537034571170807, + 3.6003249287605286, + 3.5485777258872986, + 3.5950734615325928, + 3.5995643734931946, + 3.5762577652931213, + 3.5146058201789856, + 3.467808485031128, + 3.5660253763198853, + 3.588400900363922, + 3.586184561252594, + 3.5324148535728455, + 3.654058873653412, + 3.4306570291519165, + 3.4727783203125, + 3.586322247982025, + 3.463130831718445, + 3.5909026861190796, + 3.56242835521698, + 3.434516966342926, + 3.5524529814720154, + 3.4569884538650513, + 3.501182198524475, + 3.396462619304657, + 3.4236398339271545, + 3.2672118544578552, + 3.327546536922455, + 3.5250179171562195, + 3.4542577862739563, + 3.3633089661598206, + 3.507577419281006, + 3.4204404950141907, + 3.38828843832016, + 3.378567934036255, + 3.4861780405044556, + 3.3483241200447083, + 3.402093529701233, + 3.4781301617622375, + 3.3964664340019226, + 3.3333851099014282, + 3.4308539032936096, + 3.42066890001297, + 3.42600417137146, + 3.4307650923728943, + 3.4661170840263367, + 3.399843692779541, + 3.3527462482452393, + 3.490359842777252, + 3.506344199180603, + 3.5379488468170166, + 3.383552312850952, + 3.3308772444725037, + 3.477473497390747, + 3.4379029870033264, + 3.5073152780532837, + 3.4577269554138184, + 3.54326856136322, + 3.4971607327461243, + 3.3914299607276917, + 3.3722370266914368, + 3.4328927993774414, + 3.463484466075897, + 3.3655380606651306, + 3.432557463645935, + 3.3877270221710205, + 3.396670699119568, + 3.409777581691742, + 3.361781418323517, + 3.3121633529663086, + 3.2461191415786743, + 3.464390277862549, + 3.475393772125244, + 3.2445876002311707, + 3.5033138394355774, + 3.308802306652069, + 3.477163255214691, + 3.2032665014266968, + 3.4460721611976624, + 3.54925137758255, + 3.4214342832565308, + 3.432910978794098, + 3.3943267464637756, + 3.4330523014068604, + 3.3553810715675354, + 3.4259912967681885, + 3.42621248960495, + 3.4189966320991516, + 3.3796462416648865, + 3.4862729907035828, + 3.33855539560318, + 3.4960511326789856, + 3.298312485218048, + 3.4537100195884705, + 3.400036871433258, + 3.33080130815506, + 3.483996629714966, + 3.4331719875335693, + 3.363630771636963, + 3.483964204788208, + 3.432455062866211, + 3.3694428205490112, + 3.554670572280884, + 3.445753335952759, + 3.451356828212738, + 3.5079495906829834, + 3.4933385848999023, + 3.4490312933921814, + 3.4014424681663513, + 3.400210201740265, + 3.4038368463516235, + 3.4243218898773193, + 3.418941080570221, + 3.5147798657417297, + 3.2134979367256165, + 3.370076298713684, + 3.4453486800193787, + 3.5405357480049133, + 3.192415952682495, + 3.4645209908485413, + 3.529001533985138, + 3.420100450515747, + 3.4184786677360535, + 3.4388354420661926, + 3.336589992046356, + 3.455218195915222, + 3.5209746956825256, + 3.273221790790558, + 3.484278619289398, + 3.420933783054352, + 3.434686541557312, + 3.4057669043540955, + 3.3705421686172485, + 3.380951404571533, + 3.400159537792206, + 3.417746901512146, + 3.3899790048599243, + 3.4133172631263733, + 3.4296754002571106, + 3.4233394861221313, + 3.301844537258148, + 3.399325728416443, + 3.435671091079712, + 3.462408185005188, + 3.3946186900138855, + 3.441523253917694, + 3.361421227455139, + 3.4730252623558044, + 3.409518241882324, + 3.3286635279655457, + 3.303480386734009, + 3.2587783336639404, + 3.3148505091667175, + 3.3774203658103943, + 3.2653778195381165, + 3.3572139739990234, + 3.3299408555030823, + 3.283914625644684, + 3.282969295978546, + 3.408460259437561, + 3.3540229201316833, + 3.3655835390090942, + 3.3898459672927856, + 3.342075526714325, + 3.4020936489105225, + 3.3278976678848267, + 3.4494601488113403, + 3.4060439467430115, + 3.403628170490265, + 3.285578489303589, + 3.2125869393348694, + 3.291562795639038, + 3.273035228252411, + 3.243423104286194, + 3.4100534915924072, + 3.3431678414344788, + 3.3788686394691467, + 3.330682098865509, + 3.28775155544281, + 3.356779932975769, + 3.384898364543915, + 3.461090624332428, + 3.3022100925445557, + 3.249232232570648, + 3.33537095785141, + 3.416721224784851, + 3.4330617785453796, + 3.13403856754303, + 3.217413067817688, + 3.3806774020195007, + 3.3240582942962646, + 3.218754827976227, + 3.3626375794410706, + 3.4359424710273743, + 3.2865039706230164, + 3.442403793334961, + 3.475831627845764, + 3.3382480144500732, + 3.384296953678131, + 3.3608424067497253, + 3.3796932101249695, + 3.2696738243103027, + 3.368241012096405, + 3.3460896015167236, + 3.1522589921951294, + 3.2953246235847473, + 3.430953323841095, + 3.3429972529411316, + 3.303293764591217, + 3.3876196146011353, + 3.442706882953644, + 3.3194857835769653, + 3.325086534023285, + 3.3467671871185303, + 3.376068353652954, + 3.2235442399978638, + 3.3517661690711975, + 3.2105907797813416, + 3.3950769901275635, + 3.4555907249450684, + 3.474072813987732, + 3.3386871218681335, + 3.3216801285743713, + 3.420534133911133, + 3.2957770228385925, + 3.1870580911636353, + 3.422091245651245, + 3.3015376329421997, + 3.315343737602234, + 3.2388855814933777, + 3.2787256240844727, + 3.3521222472190857, + 3.426806151866913, + 3.2567718625068665, + 3.2822291254997253, + 3.439281940460205, + 3.2952444553375244, + 3.240922749042511, + 3.21829617023468, + 3.4306180477142334, + 3.271361529827118, + 3.376651167869568, + 3.25654399394989, + 3.3875429034233093, + 3.4548083543777466, + 3.496591091156006, + 3.386153757572174, + 3.2390307188034058, + 3.2689948081970215, + 3.31302273273468, + 3.3087443709373474, + 3.3264594078063965, + 3.393727123737335, + 3.298044800758362, + 3.2216902375221252, + 3.303098678588867, + 3.409157872200012, + 3.349583685398102, + 3.283361077308655, + 3.34101003408432, + 3.2771376967430115, + 3.345164179801941, + 3.228361189365387, + 3.3350759744644165, + 3.424533247947693, + 3.1399511098861694, + 3.3658031821250916, + 3.2928852438926697, + 3.1890416741371155, + 3.266346752643585, + 3.232190728187561, + 3.2841057777404785, + 3.3610634803771973, + 3.234183609485626, + 3.268258571624756, + 3.4341105818748474, + 3.510606586933136, + 3.1877516508102417, + 3.2842445373535156, + 3.216649830341339, + 3.2499881982803345, + 3.2592726945877075, + 3.2129249572753906, + 3.127463400363922, + 3.2979597449302673, + 3.1188806295394897, + 3.293554902076721, + 3.167905032634735, + 3.2561066150665283, + 3.381335973739624, + 3.137845754623413, + 3.218725264072418, + 3.32383394241333, + 3.3204296231269836, + 3.151120364665985, + 3.1904155611991882, + 3.2562682032585144, + 3.2824215292930603, + 3.259353458881378, + 3.3223695755004883, + 3.162794351577759, + 3.2149439454078674, + 3.2667206525802612, + 3.225191831588745, + 3.224725902080536, + 3.2580551505088806, + 3.3023780584335327, + 3.2901018261909485, + 3.401239514350891, + 3.239872455596924, + 3.3347232937812805, + 3.246800720691681, + 3.2770928740501404, + 3.306493580341339, + 3.232381522655487, + 3.29657119512558, + 3.322908878326416, + 3.375291585922241, + 3.1650946736335754, + 3.287213683128357, + 3.1405972242355347, + 3.34095299243927, + 3.2779799103736877, + 3.331801414489746, + 3.3547781705856323, + 3.2921340465545654, + 3.3238492012023926, + 3.2402207255363464, + 3.2789494395256042, + 3.2915478944778442, + 3.3342060446739197, + 3.2609516382217407, + 3.175330102443695, + 3.336585581302643, + 3.278520405292511, + 3.3146109580993652, + 3.1898273229599, + 3.294302523136139, + 3.263831079006195, + 3.234450936317444, + 3.334159731864929, + 3.2458688020706177, + 3.2770785689353943, + 3.2702428698539734, + 3.2883559465408325, + 3.3925697803497314, + 3.1549238562583923, + 3.206856429576874, + 3.2705695033073425, + 3.2864332795143127, + 3.2480862736701965, + 3.2497132420539856, + 3.2163694500923157, + 3.3345996141433716, + 3.3333210945129395, + 3.2964975237846375, + 3.295778214931488, + 3.3559020161628723, + 3.172161102294922, + 3.207295000553131, + 3.1836447715759277, + 3.177534759044647, + 3.1532793045043945, + 3.1993756890296936, + 3.2142946124076843, + 3.3520936965942383, + 3.3082063794136047, + 3.303859531879425, + 3.17156320810318, + 3.2267478704452515, + 3.282149076461792, + 3.3000842332839966, + 3.2701364755630493, + 3.2481160759925842, + 3.3324596285820007, + 3.2560782432556152, + 3.304301142692566, + 3.224777340888977, + 3.173732817173004, + 3.2053903937339783, + 3.2742655873298645, + 3.266732931137085, + 3.198939263820648, + 3.146627187728882, + 3.282462954521179, + 3.1993277072906494, + 3.352869689464569, + 3.271382987499237, + 3.256039083003998, + 3.1594260931015015, + 3.238038122653961, + 3.2231443524360657 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 124.40624376361691, + 76.90308119469324, + 60.83070151823607, + 53.25296313010046, + 48.16818215578708, + 45.11690183771508, + 42.436310256615734, + 41.03938608553992, + 39.04235838269652, + 37.42512559118413, + 36.226709095688165, + 34.91472346619415, + 33.962290845737755, + 33.204334533682996, + 32.36510114367384, + 32.14213219174525, + 31.81922304044747, + 31.621077111687907, + 31.52723210467981, + 31.528721945436523 + ], + "best_ppl": 31.52723210467981, + "train_time_s": 96439.73704695702 +} \ No newline at end of file diff --git a/063_sparse_dispatch/deltanet_3of5_best.pt b/063_sparse_dispatch/deltanet_3of5_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..17b671b1c8d484c2ce6b1366e0d5ac4285e02d60 --- /dev/null +++ b/063_sparse_dispatch/deltanet_3of5_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be23dcff46d5b494cc75b6c6c9524197dc263cb78f2bc4405173b271e82a5fcf +size 352680348 diff --git a/063_sparse_dispatch/deltanet_3of5_results.json b/063_sparse_dispatch/deltanet_3of5_results.json new file mode 100644 index 0000000000000000000000000000000000000000..b1b3c4fbab73f5ac6058945b5936483ed0dcbe65 --- /dev/null +++ b/063_sparse_dispatch/deltanet_3of5_results.json @@ -0,0 +1,1651 @@ +{ + "label": "deltanet_3of5", + "seed": 42, + "attention_type": "deltanet", + "num_thin_blocks": 5, + "num_shared_blocks": 0, + "top_k_blocks": 3, + "total_params": 88089408, + "steps": [ + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 3.9422590732574463, + 3.9675341844558716, + 3.969909191131592, + 3.986530065536499, + 3.894091844558716, + 3.967142701148987, + 3.826001524925232, + 3.978479504585266, + 3.7735103368759155, + 4.019956469535828, + 3.8946967124938965, + 3.8403897285461426, + 3.9393192529678345, + 3.9308416843414307, + 3.9205490350723267, + 3.898720383644104, + 3.7886803150177, + 3.909398674964905, + 3.888807535171509, + 3.8362454175949097, + 3.967948794364929, + 3.854230523109436, + 3.9127461910247803, + 3.7240031957626343, + 3.872744083404541, + 3.8324148654937744, + 3.802101969718933, + 3.879868268966675, + 3.905840516090393, + 3.8532440662384033, + 3.9098923206329346, + 3.9315567016601562, + 3.8616952896118164, + 3.8369433879852295, + 4.017995119094849, + 3.8836933374404907, + 3.856881618499756, + 4.02899956703186, + 3.8234628438949585, + 3.732765555381775, + 3.9119592905044556, + 3.8639378547668457, + 3.7937062978744507, + 3.8437016010284424, + 3.7759222984313965, + 3.916173219680786, + 3.824639081954956, + 3.773738980293274, + 3.839208245277405, + 3.8480374813079834, + 3.8626492023468018, + 3.9014750719070435, + 3.8580936193466187, + 3.892083764076233, + 3.7693110704421997, + 3.685591220855713, + 3.760485291481018, + 3.8214054107666016, + 3.756196141242981, + 4.017513394355774, + 3.6432502269744873, + 3.834251642227173, + 3.812460780143738, + 3.849121570587158, + 3.7963337898254395, + 3.8443888425827026, + 3.904997229576111, + 4.007732629776001, + 3.7928191423416138, + 3.827404022216797, + 3.833053231239319, + 3.9116472005844116, + 3.759111166000366, + 3.9644559621810913, + 3.8585463762283325, + 3.8310422897338867, + 3.839364171028137, + 3.8903664350509644, + 3.80182683467865, + 3.950187563896179, + 3.8543978929519653, + 3.851966142654419, + 3.8118748664855957, + 3.7696712017059326, + 3.752600908279419, + 3.8333213329315186, + 3.9473586082458496, + 3.898249626159668, + 3.812654495239258, + 3.7768393754959106, + 3.785224437713623, + 3.815900444984436, + 3.804279088973999, + 3.841876745223999, + 3.854727864265442, + 3.8843908309936523, + 3.6833990812301636, + 3.789442539215088, + 3.735448479652405, + 3.6960169076919556, + 3.8275281190872192, + 3.617362856864929, + 3.809678077697754, + 3.8487255573272705, + 3.6976828575134277, + 3.823567032814026, + 3.750401258468628, + 3.6558783054351807, + 3.7468477487564087, + 3.8169015645980835, + 3.829082489013672, + 3.895378828048706, + 3.7676955461502075, + 3.766987442970276, + 3.624611020088196, + 3.6239418983459473, + 3.713942527770996, + 3.8072025775909424, + 3.673784613609314, + 3.8002976179122925, + 3.7837865352630615, + 3.815678834915161, + 3.7529594898223877, + 3.904994249343872, + 3.786298990249634, + 3.7572144269943237, + 3.7090280055999756, + 3.6134849786758423, + 3.76148784160614, + 3.6832817792892456, + 3.631274461746216, + 3.770827054977417, + 3.5525213479995728, + 3.8108198642730713, + 3.7326629161834717, + 3.659224271774292, + 3.549646258354187, + 3.72822105884552, + 3.8014737367630005, + 3.6859967708587646, + 3.8036078214645386, + 3.730479598045349, + 3.7066802978515625, + 3.5937583446502686, + 3.695152759552002, + 3.5937055349349976, + 3.648066997528076, + 3.648657441139221, + 3.6221375465393066, + 3.760121464729309, + 3.6855770349502563, + 3.695491313934326, + 3.517857789993286, + 3.768725872039795, + 3.5655362606048584, + 3.729286313056946, + 3.5958412885665894, + 3.769040107727051, + 3.6535375118255615, + 3.6561224460601807, + 3.764535665512085, + 3.6582051515579224, + 3.624191164970398, + 3.7013293504714966, + 3.602477550506592, + 3.7001848220825195, + 3.7279624938964844, + 3.5379250049591064, + 3.651950478553772, + 3.6962965726852417, + 3.665989637374878, + 3.645952820777893, + 3.599920392036438, + 3.745449185371399, + 3.6010559797286987, + 3.669199228286743, + 3.5075148344039917, + 3.682714581489563, + 3.722628951072693, + 3.741955041885376, + 3.668637990951538, + 3.6052446365356445, + 3.7074326276779175, + 3.5189043283462524, + 3.7691667079925537, + 3.7742456197738647, + 3.657196283340454, + 3.5973533391952515, + 3.526582956314087, + 3.6749496459960938, + 3.7009451389312744, + 3.7491897344589233, + 3.6722724437713623, + 3.5730977058410645, + 3.720415234565735, + 3.6775221824645996, + 3.7024412155151367, + 3.5285797119140625, + 3.6969616413116455, + 3.585243582725525, + 3.7533873319625854, + 3.640601396560669, + 3.825839877128601, + 3.662623643875122, + 3.6402244567871094, + 3.5792789459228516, + 3.8462162017822266, + 3.7177218198776245, + 3.74960994720459, + 3.6168289184570312, + 3.690210461616516, + 3.5733730792999268, + 3.7042239904403687, + 3.649417996406555, + 3.71832811832428, + 3.5901294946670532, + 3.59306263923645, + 3.7133891582489014, + 3.6026731729507446, + 3.692668318748474, + 3.6684179306030273, + 3.6596405506134033, + 3.6120049953460693, + 3.7183111906051636, + 3.763471841812134, + 3.6573251485824585, + 3.7735605239868164, + 3.6639820337295532, + 3.5975884199142456, + 3.5677661895751953, + 3.7238250970840454, + 3.6574490070343018, + 3.6618930101394653, + 3.6275440454483032, + 3.712257146835327, + 3.647499680519104, + 3.70621919631958, + 3.7409918308258057, + 3.7085877656936646, + 3.6303406953811646, + 3.548994302749634, + 3.68615984916687, + 3.7111862897872925, + 3.7511481046676636, + 3.695488691329956, + 3.6881033182144165, + 3.6464017629623413, + 3.736749529838562, + 3.656655430793762, + 3.676147699356079, + 3.591293692588806, + 3.7446463108062744, + 3.646834373474121, + 3.5508028268814087, + 3.5766814947128296, + 3.5592929124832153, + 3.467690944671631, + 3.6417057514190674, + 3.50041663646698, + 3.54603111743927, + 3.6440773010253906, + 3.534401774406433, + 3.5421353578567505, + 3.4720191955566406, + 3.60621976852417, + 3.5888983011245728, + 3.578012228012085, + 3.525696635246277, + 3.3340394496917725, + 3.483004927635193, + 3.6256158351898193, + 3.4882240295410156, + 3.6087483167648315, + 3.5086028575897217, + 3.5299421548843384, + 3.520854949951172, + 3.606602668762207, + 3.64292311668396, + 3.632864475250244, + 3.509312868118286, + 3.4476133584976196, + 3.581142783164978, + 3.642890453338623, + 3.6268866062164307, + 3.5708590745925903, + 3.4664385318756104, + 3.5696297883987427, + 3.4971083402633667, + 3.630854845046997, + 3.432034134864807, + 3.602981925010681, + 3.7037625312805176, + 3.6058636903762817, + 3.5236200094223022, + 3.6128132343292236, + 3.5277265310287476, + 3.4807896614074707, + 3.5149961709976196, + 3.570640444755554, + 3.694519877433777, + 3.5581181049346924, + 3.601982593536377, + 3.651274561882019, + 3.5569260120391846, + 3.614356756210327, + 3.517993927001953, + 3.6431084871292114, + 3.4266932010650635, + 3.495204448699951, + 3.4548802375793457, + 3.4636203050613403, + 3.4273135662078857, + 3.479853630065918, + 3.6504483222961426, + 3.593104600906372, + 3.563412666320801, + 3.490407943725586, + 3.5189536809921265, + 3.559446930885315, + 3.51863694190979, + 3.6092920303344727, + 3.7556262016296387, + 3.445136785507202, + 3.5067996978759766, + 3.5626081228256226, + 3.3791438341140747, + 3.6216678619384766, + 3.4458500146865845, + 3.5285218954086304, + 3.590374231338501, + 3.5420126914978027, + 3.49111545085907, + 3.504103660583496, + 3.621264100074768, + 3.565430998802185, + 3.5062150955200195, + 3.474846363067627, + 3.4620277881622314, + 3.4575339555740356, + 3.615398645401001, + 3.579739212989807, + 3.5626097917556763, + 3.5136446952819824, + 3.3954750299453735, + 3.5427197217941284, + 3.4782681465148926, + 3.522294282913208, + 3.4136736392974854, + 3.5054389238357544, + 3.6602749824523926, + 3.696934938430786, + 3.5041515827178955, + 3.5224207639694214, + 3.519915819168091, + 3.5644112825393677, + 3.7024521827697754, + 3.5781434774398804, + 3.531313180923462, + 3.6312203407287598, + 3.5300493240356445, + 3.5348429679870605, + 3.4284965991973877, + 3.6054787635803223, + 3.565656304359436, + 3.5558637380599976, + 3.43314528465271, + 3.6026946306228638, + 3.4788646697998047, + 3.569953680038452, + 3.526946544647217, + 3.4594181776046753, + 3.4901938438415527, + 3.626563787460327, + 3.62404727935791, + 3.5452332496643066, + 3.5175927877426147, + 3.429434895515442, + 3.5847058296203613, + 3.5852307081222534, + 3.33981454372406, + 3.281216025352478, + 3.296889305114746, + 3.4752609729766846, + 3.4219762086868286, + 3.5027856826782227, + 3.347859263420105, + 3.500957727432251, + 3.4465314149856567, + 3.469826340675354, + 3.3652082681655884, + 3.4284249544143677, + 3.2977298498153687, + 3.364906430244446, + 3.1721774339675903, + 3.3362828493118286, + 3.4969452619552612, + 3.4734467267990112, + 3.3740365505218506, + 3.3837101459503174, + 3.3867887258529663, + 3.4336090087890625, + 3.4871973991394043, + 3.355883479118347, + 3.3895760774612427, + 3.3635209798812866, + 3.482885956764221, + 3.5132782459259033, + 3.2953922748565674, + 3.4393563270568848, + 3.4625028371810913, + 3.4524550437927246, + 3.3952152729034424, + 3.545716404914856, + 3.3892147541046143, + 3.378274440765381, + 3.4402672052383423, + 3.3945013284683228, + 3.4813753366470337, + 3.332290530204773, + 3.2530195713043213, + 3.369800090789795, + 3.3800268173217773, + 3.4666972160339355, + 3.348142623901367, + 3.446080207824707, + 3.387197971343994, + 3.4798085689544678, + 3.354780077934265, + 3.449241518974304, + 3.4275952577590942, + 3.4434287548065186, + 3.3392422199249268, + 3.2976244688034058, + 3.3423954248428345, + 3.371030569076538, + 3.372397303581238, + 3.396714210510254, + 3.5533173084259033, + 3.4994003772735596, + 3.565462827682495, + 3.4668582677841187, + 3.2828704118728638, + 3.468946099281311, + 3.4230916500091553, + 3.388060688972473, + 3.360906481742859, + 3.456939935684204, + 3.368908405303955, + 3.2324116230010986, + 3.4888696670532227, + 3.339056134223938, + 3.4275182485580444, + 3.538208484649658, + 3.5251755714416504, + 3.29793119430542, + 3.4582396745681763, + 3.3693671226501465, + 3.3442596197128296, + 3.39595103263855, + 3.577646851539612, + 3.5269216299057007, + 3.425654411315918, + 3.266364336013794, + 3.516979455947876, + 3.397485375404358, + 3.5715014934539795, + 3.5305222272872925, + 3.427865505218506, + 3.5364394187927246, + 3.4728275537490845, + 3.494882345199585, + 3.4111764430999756, + 3.467007279396057, + 3.39540433883667, + 3.416795492172241, + 3.2955344915390015, + 3.2974389791488647, + 3.4086055755615234, + 3.438058376312256, + 3.4193516969680786, + 3.4327131509780884, + 3.307589530944824, + 3.448094367980957, + 3.44210147857666, + 3.4893150329589844, + 3.4933615922927856, + 3.4524909257888794, + 3.459634780883789, + 3.421342968940735, + 3.411379814147949, + 3.4059102535247803, + 3.4296629428863525, + 3.5330662727355957, + 3.4602311849594116, + 3.4538938999176025, + 3.4145915508270264, + 3.459244132041931, + 3.4809043407440186, + 3.389139175415039, + 3.463180184364319, + 3.3012173175811768, + 3.429402232170105, + 3.502185344696045, + 3.450729489326477, + 3.3740051984786987, + 3.4445215463638306, + 3.267192006111145, + 3.33810293674469, + 3.314465045928955, + 3.373964309692383, + 3.2668060064315796, + 3.331259250640869, + 3.325383424758911, + 3.2540130615234375, + 3.214161515235901, + 3.4293302297592163, + 3.3490166664123535, + 3.2063746452331543, + 3.312957525253296, + 3.1869125366210938, + 3.367389440536499, + 3.2464828491210938, + 3.305304169654846, + 3.2839983701705933, + 3.39917254447937, + 3.5032787322998047, + 3.279444456100464, + 3.496985077857971, + 3.2956135272979736, + 3.303612232208252, + 3.396046280860901, + 3.297341227531433, + 3.3784810304641724, + 3.2210150957107544, + 3.270094394683838, + 3.348914384841919, + 3.2434751987457275, + 3.2287710905075073, + 3.4093645811080933, + 3.4030641317367554, + 3.3671854734420776, + 3.3418612480163574, + 3.336822986602783, + 3.3791269063949585, + 3.3458346128463745, + 3.4019991159439087, + 3.3753000497817993, + 3.3428308963775635, + 3.390660285949707, + 3.2816919088363647, + 3.4062252044677734, + 3.3420424461364746, + 3.3140512704849243, + 3.3647780418395996, + 3.3843555450439453, + 3.1973037719726562, + 3.418972373008728, + 3.22514271736145, + 3.2530179023742676, + 3.3090097904205322, + 3.29462730884552, + 3.3664796352386475, + 3.4338061809539795, + 3.422312617301941, + 3.4235492944717407, + 3.3677589893341064, + 3.2682565450668335, + 3.401677370071411, + 3.2762956619262695, + 3.3934556245803833, + 3.3420032262802124, + 3.204176187515259, + 3.40524160861969, + 3.27634859085083, + 3.305304527282715, + 3.4631316661834717, + 3.4278427362442017, + 3.4441497325897217, + 3.3101035356521606, + 3.326542615890503, + 3.281483292579651, + 3.4448007345199585, + 3.3318185806274414, + 3.376061201095581, + 3.4129222631454468, + 3.3908233642578125, + 3.3589481115341187, + 3.2686681747436523, + 3.3479907512664795, + 3.387356758117676, + 3.251608371734619, + 3.253799557685852, + 3.370947241783142, + 3.3995941877365112, + 3.298783302307129, + 3.315213441848755, + 3.275313973426819, + 3.3138316869735718, + 3.305180311203003, + 3.4001424312591553, + 3.4326841831207275, + 3.41601026058197, + 3.3562939167022705, + 3.270445704460144, + 3.262215256690979, + 3.3633029460906982, + 3.4302953481674194, + 3.4540958404541016, + 3.224974751472473, + 3.3786033391952515, + 3.3239892721176147, + 3.332621216773987, + 3.2833878993988037, + 3.25635302066803, + 3.212698817253113, + 3.2872384786605835, + 3.3716201782226562, + 3.406069755554199, + 3.340477705001831, + 3.4440338611602783, + 3.2967453002929688, + 3.4175559282302856, + 3.2987966537475586, + 3.3412762880325317, + 3.4355558156967163, + 3.2913763523101807, + 3.280189037322998, + 3.2916125059127808, + 3.3274433612823486, + 3.292117953300476, + 3.233557105064392, + 3.4018101692199707, + 3.355584979057312, + 3.3719562292099, + 3.2447357177734375, + 3.4895808696746826, + 3.2453973293304443, + 3.319196581840515, + 3.37776780128479, + 3.172890782356262, + 3.2085325717926025, + 3.3962783813476562, + 3.325406074523926, + 3.2754740715026855, + 3.2461233139038086, + 3.2916144132614136, + 3.355416417121887, + 3.12799870967865, + 3.2906076908111572, + 3.2514429092407227, + 3.2040375471115112, + 3.3569111824035645, + 3.2246304750442505, + 3.327357292175293, + 3.2106059789657593, + 3.3088161945343018, + 3.2628333568573, + 3.302847146987915, + 3.344430446624756, + 3.2852734327316284, + 3.287777066230774, + 3.1909550428390503, + 3.2628021240234375, + 3.2822649478912354, + 3.3118672370910645, + 3.3257551193237305, + 3.2112631797790527, + 3.308869242668152, + 3.322316884994507, + 3.3773937225341797, + 3.341936707496643, + 3.2621136903762817, + 3.2874186038970947, + 3.272802948951721, + 3.2948917150497437, + 3.2399665117263794, + 3.293473482131958, + 3.177647590637207, + 3.249691963195801, + 3.3727242946624756, + 3.2286360263824463, + 3.2144891023635864, + 3.199497699737549, + 3.318456292152405, + 3.298787236213684, + 3.3024438619613647, + 3.298857808113098, + 3.3676928281784058, + 3.3430347442626953, + 3.410310387611389, + 3.352942943572998, + 3.2689995765686035, + 3.116025924682617, + 3.2484391927719116, + 3.294577479362488, + 3.2892550230026245, + 3.2310630083084106, + 3.226002335548401, + 3.2988439798355103, + 3.2817916870117188, + 3.303650736808777, + 3.3122719526290894, + 3.1548914909362793, + 3.23011577129364, + 3.299717426300049, + 3.233570694923401, + 3.3290226459503174, + 3.3175867795944214, + 3.2865006923675537, + 3.30028760433197, + 3.2315430641174316, + 3.2820358276367188, + 3.2932660579681396, + 3.1365010738372803, + 3.1543169021606445, + 3.4064141511917114, + 3.3630988597869873, + 3.234227418899536, + 3.2244280576705933, + 3.209546446800232, + 3.310997724533081, + 3.217120885848999, + 3.287865996360779, + 3.3889979124069214, + 3.426220417022705, + 3.2478832006454468, + 3.2598406076431274, + 3.2705971002578735, + 3.355074644088745, + 3.2716152667999268, + 3.3413801193237305, + 3.1578933000564575, + 3.309425473213196, + 3.2267216444015503, + 3.2278612852096558, + 3.3013041019439697, + 3.225550651550293, + 3.165761947631836, + 3.1824545860290527, + 3.3192089796066284, + 3.291691780090332, + 3.238741159439087, + 3.2459800243377686, + 3.247408151626587, + 3.320467948913574, + 3.2648922204971313, + 3.2719779014587402, + 3.278268575668335, + 3.147145390510559, + 3.2359156608581543, + 3.4030542373657227, + 3.2276389598846436, + 3.398088216781616, + 3.213351249694824, + 3.301742672920227, + 3.1197842359542847, + 3.175839900970459, + 3.288431167602539, + 3.1280263662338257, + 3.2405800819396973, + 3.3225146532058716, + 3.253536343574524, + 3.322104811668396, + 3.2418230772018433, + 3.213525414466858, + 3.293125867843628, + 3.347917079925537, + 3.2471752166748047, + 3.308092474937439, + 3.1149944067001343, + 3.261310338973999, + 3.195342540740967, + 3.3427833318710327, + 3.2151224613189697, + 3.348935127258301, + 3.289778232574463, + 3.405527353286743, + 3.2730021476745605, + 3.2563849687576294, + 3.457221508026123, + 3.1079803705215454, + 3.2386845350265503, + 3.3160152435302734, + 3.207384467124939, + 3.283173680305481, + 3.249492645263672, + 3.185503125190735, + 3.289707899093628, + 3.226640224456787, + 3.3483564853668213, + 3.1975655555725098, + 3.2126752138137817, + 3.259338617324829, + 3.3017678260803223, + 3.281827211380005, + 3.249472141265869, + 3.269622325897217, + 3.3633811473846436, + 3.2217801809310913, + 3.3714914321899414 + ], + "val_steps": [ + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 45.46563948268351, + 42.156447983527364, + 40.604824819901616, + 38.70401258056204, + 36.76944640996809, + 35.7441481101651, + 34.51506616425472, + 33.73328822858661, + 32.78745842912967, + 32.028009891050345, + 31.813266542201422, + 31.2511754987786, + 31.096732810005825, + 30.89015460290493, + 30.765135436673603, + 30.786072515340948 + ], + "best_ppl": 30.765135436673603, + "train_time_s": 18727.689113378525 +} \ No newline at end of file diff --git a/063_sparse_dispatch/deltanet_3of5_s137_best.pt b/063_sparse_dispatch/deltanet_3of5_s137_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b8af4e13a7480f944026b0539fb4e46e00b3f2b --- /dev/null +++ b/063_sparse_dispatch/deltanet_3of5_s137_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b9f68ca7dbead6ebc5a934477db19c69c994fb8f3437ecb0085e25d79427fee +size 352684297 diff --git a/063_sparse_dispatch/deltanet_3of5_s137_results.json b/063_sparse_dispatch/deltanet_3of5_s137_results.json new file mode 100644 index 0000000000000000000000000000000000000000..595ee700715f7e22f9563c22fff00f9b5339d4c6 --- /dev/null +++ b/063_sparse_dispatch/deltanet_3of5_s137_results.json @@ -0,0 +1,1651 @@ +{ + "label": "deltanet_3of5_s137", + "seed": 137, + "attention_type": "deltanet", + "num_thin_blocks": 5, + "num_shared_blocks": 0, + "top_k_blocks": 3, + "total_params": 88089408, + "steps": [ + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 3.9089035987854004, + 3.912316918373108, + 3.979505777359009, + 3.919536828994751, + 3.8753925561904907, + 3.910730004310608, + 3.8488703966140747, + 3.999323010444641, + 3.932865023612976, + 3.9643144607543945, + 4.08697783946991, + 3.9318798780441284, + 3.9344598054885864, + 3.9815160036087036, + 3.8200124502182007, + 3.853782057762146, + 3.9787161350250244, + 3.7452200651168823, + 3.7950788736343384, + 4.010727524757385, + 3.9644999504089355, + 3.9282702207565308, + 4.010738253593445, + 3.856671929359436, + 3.9857746362686157, + 3.8583669662475586, + 3.918078660964966, + 3.8029736280441284, + 3.947762131690979, + 3.897342324256897, + 3.6650744676589966, + 3.8164052963256836, + 3.87934410572052, + 3.9757977724075317, + 3.8318426609039307, + 3.8376011848449707, + 3.9940531253814697, + 3.946517586708069, + 3.8789472579956055, + 3.9775339365005493, + 3.762147068977356, + 4.014288067817688, + 3.9460864067077637, + 3.89154589176178, + 3.989864468574524, + 3.901667833328247, + 3.761192560195923, + 3.8546879291534424, + 3.800935745239258, + 3.880099058151245, + 3.8383203744888306, + 3.77161967754364, + 3.9040629863739014, + 3.81939435005188, + 3.8604787588119507, + 3.8387908935546875, + 3.948624014854431, + 3.9622802734375, + 3.941304326057434, + 3.8236385583877563, + 3.804457664489746, + 3.7722153663635254, + 3.8310617208480835, + 3.7502760887145996, + 3.7878522872924805, + 4.007832646369934, + 3.9672021865844727, + 3.8090577125549316, + 3.8612921237945557, + 3.6694165468215942, + 3.808257579803467, + 3.860294818878174, + 3.779584765434265, + 3.8809237480163574, + 3.8694852590560913, + 3.796665906906128, + 3.8092323541641235, + 3.8289250135421753, + 3.796328902244568, + 3.8334211111068726, + 3.8704737424850464, + 3.7843356132507324, + 3.794908046722412, + 3.832832455635071, + 3.794481039047241, + 3.8144237995147705, + 4.003569483757019, + 3.9890915155410767, + 3.930446147918701, + 3.786261796951294, + 3.8236236572265625, + 3.8212826251983643, + 3.7657824754714966, + 3.8040817975997925, + 3.7374210357666016, + 3.702545166015625, + 3.845770239830017, + 3.795036554336548, + 3.837907910346985, + 3.742283582687378, + 3.7010163068771362, + 3.7032231092453003, + 3.7113503217697144, + 3.764298439025879, + 3.790497303009033, + 3.811331629753113, + 3.7951860427856445, + 3.812446713447571, + 3.737309217453003, + 3.945034146308899, + 3.695108413696289, + 3.8050246238708496, + 3.7770907878875732, + 3.7345420122146606, + 3.735442638397217, + 3.6341341733932495, + 3.7655380964279175, + 3.6501829624176025, + 3.809074282646179, + 3.7919254302978516, + 3.7689473628997803, + 3.6743011474609375, + 3.8838021755218506, + 3.821621298789978, + 3.8720293045043945, + 3.816161274909973, + 3.5947201251983643, + 3.617642402648926, + 3.661855101585388, + 3.6064542531967163, + 3.7142823934555054, + 3.6602530479431152, + 3.6546090841293335, + 3.6938936710357666, + 3.612930655479431, + 3.6112377643585205, + 3.6357561349868774, + 3.808004140853882, + 3.7428550720214844, + 3.7763484716415405, + 3.7271385192871094, + 3.6864925622940063, + 3.746228337287903, + 3.652751922607422, + 3.6397465467453003, + 3.7370448112487793, + 3.7058883905410767, + 3.6785080432891846, + 3.638139486312866, + 3.6722038984298706, + 3.6315999031066895, + 3.7325046062469482, + 3.5854287147521973, + 3.7303191423416138, + 3.655068039894104, + 3.5843515396118164, + 3.7768187522888184, + 3.725579261779785, + 3.7227169275283813, + 3.6748863458633423, + 3.758960247039795, + 3.7754297256469727, + 3.589411497116089, + 3.6761633157730103, + 3.671493411064148, + 3.81754207611084, + 3.690736413002014, + 3.6481220722198486, + 3.6501415967941284, + 3.728173851966858, + 3.769550919532776, + 3.718774914741516, + 3.723101496696472, + 3.6770026683807373, + 3.484426259994507, + 3.6640132665634155, + 3.717418074607849, + 3.696651339530945, + 3.7359116077423096, + 3.5096789598464966, + 3.5753679275512695, + 3.6668357849121094, + 3.7530550956726074, + 3.6434266567230225, + 3.688285708427429, + 3.6757105588912964, + 3.5920662879943848, + 3.5508679151535034, + 3.8261016607284546, + 3.6947327852249146, + 3.672310948371887, + 3.723358631134033, + 3.812616467475891, + 3.6824480295181274, + 3.791775345802307, + 3.621644616127014, + 3.711298108100891, + 3.623636245727539, + 3.6451979875564575, + 3.6219366788864136, + 3.6261461973190308, + 3.7789386510849, + 3.6864125728607178, + 3.71444571018219, + 3.8457510471343994, + 3.931787371635437, + 3.6880247592926025, + 3.7900785207748413, + 3.6568561792373657, + 3.749119997024536, + 3.7776790857315063, + 3.766924262046814, + 3.7379637956619263, + 3.7777810096740723, + 3.8198790550231934, + 3.589619517326355, + 3.667588233947754, + 3.7186594009399414, + 3.5262959003448486, + 3.752069354057312, + 3.6222063302993774, + 3.6881439685821533, + 3.655896544456482, + 3.626279592514038, + 3.611320376396179, + 3.667009115219116, + 3.7877382040023804, + 3.649499773979187, + 3.6646289825439453, + 3.5721633434295654, + 3.596329927444458, + 3.6743810176849365, + 3.6592663526535034, + 3.6362011432647705, + 3.6985836029052734, + 3.6160197257995605, + 3.602977156639099, + 3.7226675748825073, + 3.7555627822875977, + 3.6479755640029907, + 3.7465312480926514, + 3.6229008436203003, + 3.6122909784317017, + 3.6679799556732178, + 3.617098808288574, + 3.849349856376648, + 3.787968158721924, + 3.693284511566162, + 3.6673372983932495, + 3.845181941986084, + 3.760926842689514, + 3.7085111141204834, + 3.670254707336426, + 3.5214446783065796, + 3.5900481939315796, + 3.5786426067352295, + 3.7144694328308105, + 3.7193119525909424, + 3.4640650749206543, + 3.5748209953308105, + 3.497527599334717, + 3.579824924468994, + 3.5977574586868286, + 3.5328328609466553, + 3.572326421737671, + 3.6199660301208496, + 3.625357508659363, + 3.5415265560150146, + 3.5117543935775757, + 3.5583263635635376, + 3.6064188480377197, + 3.6012827157974243, + 3.6223011016845703, + 3.5480713844299316, + 3.639050245285034, + 3.557071089744568, + 3.5182204246520996, + 3.5287147760391235, + 3.7127604484558105, + 3.5787575244903564, + 3.49848473072052, + 3.653630018234253, + 3.5886605978012085, + 3.5851316452026367, + 3.7048535346984863, + 3.569829225540161, + 3.5448668003082275, + 3.4580163955688477, + 3.627094030380249, + 3.478412628173828, + 3.5195523500442505, + 3.43704891204834, + 3.5333592891693115, + 3.5729340314865112, + 3.5243172645568848, + 3.6034483909606934, + 3.6389715671539307, + 3.594304084777832, + 3.469675898551941, + 3.545260787010193, + 3.513740658760071, + 3.5923136472702026, + 3.5646724700927734, + 3.624664545059204, + 3.5925676822662354, + 3.563307762145996, + 3.5807448625564575, + 3.6519778966903687, + 3.5918482542037964, + 3.5181221961975098, + 3.5733487606048584, + 3.421842932701111, + 3.368257522583008, + 3.635949969291687, + 3.5087122917175293, + 3.6295650005340576, + 3.5294018983840942, + 3.4608967304229736, + 3.5038628578186035, + 3.6038286685943604, + 3.620107054710388, + 3.4350838661193848, + 3.4637718200683594, + 3.480448603630066, + 3.5510400533676147, + 3.6017292737960815, + 3.60333251953125, + 3.498348116874695, + 3.5519357919692993, + 3.448713183403015, + 3.455313205718994, + 3.5360876321792603, + 3.57760488986969, + 3.605586290359497, + 3.555442452430725, + 3.5590693950653076, + 3.607561945915222, + 3.6400513648986816, + 3.606365442276001, + 3.557821035385132, + 3.5720903873443604, + 3.540840268135071, + 3.583042621612549, + 3.6220356225967407, + 3.747709035873413, + 3.554613471031189, + 3.695870041847229, + 3.552278995513916, + 3.4804673194885254, + 3.441484570503235, + 3.613515615463257, + 3.554478406906128, + 3.4694602489471436, + 3.5090866088867188, + 3.5676571130752563, + 3.5429996252059937, + 3.5961869955062866, + 3.5903797149658203, + 3.5464032888412476, + 3.497155785560608, + 3.600304961204529, + 3.5109105110168457, + 3.3358447551727295, + 3.611093044281006, + 3.46391761302948, + 3.551493287086487, + 3.552049160003662, + 3.431032657623291, + 3.6068170070648193, + 3.6401950120925903, + 3.6406986713409424, + 3.506571412086487, + 3.5674489736557007, + 3.4988582134246826, + 3.4655356407165527, + 3.596021056175232, + 3.4340215921401978, + 3.411217451095581, + 3.4674415588378906, + 3.4312984943389893, + 3.312800884246826, + 3.4597564935684204, + 3.538244366645813, + 3.5493627786636353, + 3.4741692543029785, + 3.4623398780822754, + 3.4567222595214844, + 3.4617390632629395, + 3.393087387084961, + 3.480006217956543, + 3.4868746995925903, + 3.516595482826233, + 3.5030540227890015, + 3.452601194381714, + 3.4076619148254395, + 3.4596184492111206, + 3.4603837728500366, + 3.4788318872451782, + 3.579012155532837, + 3.526969313621521, + 3.436540126800537, + 3.466479539871216, + 3.48294734954834, + 3.5165237188339233, + 3.4016977548599243, + 3.564794898033142, + 3.4288188219070435, + 3.445014238357544, + 3.399817705154419, + 3.460698962211609, + 3.437789559364319, + 3.4596054553985596, + 3.4310078620910645, + 3.508361339569092, + 3.6126989126205444, + 3.5021703243255615, + 3.4017746448516846, + 3.459948778152466, + 3.5128469467163086, + 3.4410799741744995, + 3.5158114433288574, + 3.504482626914978, + 3.482686161994934, + 3.510887384414673, + 3.4745339155197144, + 3.451247811317444, + 3.470324397087097, + 3.5408159494400024, + 3.3887360095977783, + 3.500615119934082, + 3.192864775657654, + 3.442151188850403, + 3.4266573190689087, + 3.4634087085723877, + 3.402265191078186, + 3.4621001482009888, + 3.576458215713501, + 3.392354965209961, + 3.4197336435317993, + 3.4679991006851196, + 3.433324456214905, + 3.4688820838928223, + 3.4657548666000366, + 3.451295018196106, + 3.4955523014068604, + 3.441329836845398, + 3.4899117946624756, + 3.400867462158203, + 3.2784945964813232, + 3.428376317024231, + 3.481157660484314, + 3.3255743980407715, + 3.4997751712799072, + 3.3896682262420654, + 3.515414595603943, + 3.4212067127227783, + 3.4159557819366455, + 3.4768974781036377, + 3.5360347032546997, + 3.453235387802124, + 3.4872450828552246, + 3.395163059234619, + 3.532423257827759, + 3.3238723278045654, + 3.523009777069092, + 3.3749167919158936, + 3.4112545251846313, + 3.450603723526001, + 3.4162641763687134, + 3.5229629278182983, + 3.466298460960388, + 3.482724189758301, + 3.4433934688568115, + 3.559742569923401, + 3.4535739421844482, + 3.43496572971344, + 3.523987650871277, + 3.4731944799423218, + 3.4615767002105713, + 3.390993356704712, + 3.477187991142273, + 3.4221296310424805, + 3.5085924863815308, + 3.40548574924469, + 3.419637441635132, + 3.537157654762268, + 3.5344053506851196, + 3.394649863243103, + 3.4132291078567505, + 3.404167056083679, + 3.4140273332595825, + 3.3149757385253906, + 3.560787796974182, + 3.406548500061035, + 3.486966609954834, + 3.351679563522339, + 3.4201276302337646, + 3.5702320337295532, + 3.5124181509017944, + 3.399451494216919, + 3.4175055027008057, + 3.439902901649475, + 3.5551947355270386, + 3.4482637643814087, + 3.4550920724868774, + 3.50724458694458, + 3.2951446771621704, + 3.4174275398254395, + 3.214722514152527, + 3.3063933849334717, + 3.517977476119995, + 3.2378852367401123, + 3.3190221786499023, + 3.414399266242981, + 3.3783817291259766, + 3.1972824335098267, + 3.3960189819335938, + 3.345292568206787, + 3.439259171485901, + 3.4011459350585938, + 3.285227656364441, + 3.2781063318252563, + 3.296401858329773, + 3.2828913927078247, + 3.2927740812301636, + 3.388163685798645, + 3.421255946159363, + 3.4020700454711914, + 3.2966034412384033, + 3.3462212085723877, + 3.3803539276123047, + 3.347473621368408, + 3.4609529972076416, + 3.4001002311706543, + 3.213384747505188, + 3.420904278755188, + 3.345331907272339, + 3.325158476829529, + 3.3912957906723022, + 3.4087412357330322, + 3.3656524419784546, + 3.43449068069458, + 3.275039315223694, + 3.372856020927429, + 3.3496352434158325, + 3.4335628747940063, + 3.4131500720977783, + 3.345498204231262, + 3.411146640777588, + 3.424409508705139, + 3.325145721435547, + 3.3307007551193237, + 3.3651204109191895, + 3.283310055732727, + 3.4591199159622192, + 3.3968974351882935, + 3.4026490449905396, + 3.421285629272461, + 3.3892639875411987, + 3.351272463798523, + 3.443820595741272, + 3.306688666343689, + 3.4405022859573364, + 3.39013671875, + 3.431983232498169, + 3.398680567741394, + 3.425460934638977, + 3.5373419523239136, + 3.2568447589874268, + 3.4015791416168213, + 3.4046348333358765, + 3.442814588546753, + 3.3323612213134766, + 3.34354567527771, + 3.433657169342041, + 3.336389422416687, + 3.3829838037490845, + 3.399419903755188, + 3.289039731025696, + 3.476996421813965, + 3.126626968383789, + 3.340777277946472, + 3.271410346031189, + 3.2355384826660156, + 3.4065558910369873, + 3.289699912071228, + 3.418025493621826, + 3.2683353424072266, + 3.314147472381592, + 3.3282371759414673, + 3.31701123714447, + 3.3513094186782837, + 3.3781977891921997, + 3.2238473892211914, + 3.3378785848617554, + 3.296387553215027, + 3.197161316871643, + 3.4256097078323364, + 3.191287398338318, + 3.4135440587997437, + 3.4600733518600464, + 3.4198964834213257, + 3.356545329093933, + 3.303293228149414, + 3.355406403541565, + 3.307757019996643, + 3.4133846759796143, + 3.4156241416931152, + 3.2607580423355103, + 3.265904188156128, + 3.3927674293518066, + 3.3511962890625, + 3.2725298404693604, + 3.3560173511505127, + 3.3922712802886963, + 3.33391010761261, + 3.2652426958084106, + 3.324928402900696, + 3.4613338708877563, + 3.364407539367676, + 3.2970482110977173, + 3.4327027797698975, + 3.4069676399230957, + 3.3848828077316284, + 3.2450140714645386, + 3.404258370399475, + 3.360080599784851, + 3.4195178747177124, + 3.290656805038452, + 3.29909348487854, + 3.356588125228882, + 3.2459524869918823, + 3.3495463132858276, + 3.3244848251342773, + 3.345605969429016, + 3.1781922578811646, + 3.261506199836731, + 3.2560923099517822, + 3.3868188858032227, + 3.3718267679214478, + 3.2665140628814697, + 3.3116462230682373, + 3.288416028022766, + 3.3895689249038696, + 3.3591389656066895, + 3.231000304222107, + 3.308146834373474, + 3.260487675666809, + 3.298035502433777, + 3.378728747367859, + 3.314690113067627, + 3.2133564949035645, + 3.3024415969848633, + 3.2530248165130615, + 3.206122398376465, + 3.328423023223877, + 3.0981907844543457, + 3.2063636779785156, + 3.2129443883895874, + 3.17179274559021, + 3.369922637939453, + 3.349414825439453, + 3.3187928199768066, + 3.1874985694885254, + 3.1658644676208496, + 3.3331995010375977, + 3.2580782175064087, + 3.253458023071289, + 3.2638275623321533, + 3.356052875518799, + 3.278388023376465, + 3.3689141273498535, + 3.3256229162216187, + 3.341530442237854, + 3.3472641706466675, + 3.290623188018799, + 3.3313791751861572, + 3.4005786180496216, + 3.460574984550476, + 3.239790201187134, + 3.201432704925537, + 3.324458956718445, + 3.097664952278137, + 3.3372347354888916, + 3.357335090637207, + 3.3454110622406006, + 3.2886637449264526, + 3.240367293357849, + 3.392470598220825, + 3.353692054748535, + 3.2974199056625366, + 3.2584253549575806, + 3.1843897104263306, + 3.327470541000366, + 3.2012134790420532, + 3.336639642715454, + 3.2822433710098267, + 3.288234829902649, + 3.33419668674469, + 3.287899613380432, + 3.364524245262146, + 3.3155202865600586, + 3.181489109992981, + 3.3746906518936157, + 3.4640052318573, + 3.1970587968826294, + 3.406738758087158, + 3.346125364303589, + 3.4156895875930786, + 3.282729148864746, + 3.3205496072769165, + 3.278996229171753, + 3.2986334562301636, + 3.3094966411590576, + 3.2692532539367676, + 3.359157085418701, + 3.2868423461914062, + 3.3578561544418335, + 3.2370052337646484, + 3.320604681968689, + 3.2344794273376465, + 3.425639271736145, + 3.2112265825271606, + 3.215544581413269, + 3.2627166509628296, + 3.2035000324249268, + 3.3474828004837036, + 3.357385754585266, + 3.304925799369812, + 3.3303428888320923, + 3.311642050743103, + 3.260830521583557, + 3.1617023944854736, + 3.263885021209717, + 3.3446240425109863, + 3.2803558111190796, + 3.35726797580719, + 3.4432055950164795, + 3.2735755443573, + 3.3234658241271973, + 3.2751026153564453, + 3.275083065032959, + 3.2666813135147095, + 3.2985039949417114, + 3.2627298831939697, + 3.485125780105591, + 3.313646674156189, + 3.218798279762268, + 3.3913867473602295, + 3.3297617435455322, + 3.1924670934677124, + 3.3578373193740845, + 3.2921864986419678, + 3.3722418546676636, + 3.4272983074188232, + 3.326393723487854, + 3.1079403162002563, + 3.265078544616699, + 3.1619433164596558, + 3.309643507003784, + 3.2295830249786377, + 3.280146360397339, + 3.314220666885376, + 3.297829270362854, + 3.270655393600464, + 3.289483070373535, + 3.221555709838867, + 3.3408987522125244, + 3.191118359565735, + 3.2882678508758545, + 3.2318984270095825, + 3.202000856399536, + 3.361063241958618, + 3.26404345035553, + 3.3042796850204468, + 3.2971198558807373, + 3.3717986345291138, + 3.299193024635315, + 3.3393207788467407, + 3.3039129972457886, + 3.2867391109466553, + 3.24187171459198, + 3.3042566776275635, + 3.288931727409363, + 3.238686203956604, + 3.204067349433899, + 3.1773980855941772, + 3.3050734996795654, + 3.2020047903060913, + 3.2609682083129883, + 3.2088342905044556, + 3.27059805393219, + 3.3004499673843384, + 3.216372489929199, + 3.2722889184951782, + 3.1636345386505127, + 3.3585517406463623, + 3.27128529548645, + 3.0946543216705322, + 3.346296787261963 + ], + "val_steps": [ + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 45.439581522284186, + 42.677102793299724, + 40.85063683676074, + 39.13335219988097, + 37.519488112189215, + 36.476250348398345, + 35.230484710697816, + 34.42218682576831, + 33.4244322338798, + 32.517697388928845, + 32.280951726131704, + 31.83214049550759, + 31.597061716665316, + 31.430463229839436, + 31.28550555536559, + 31.310236260119844 + ], + "best_ppl": 31.28550555536559, + "train_time_s": 19479.595780849457 +} \ No newline at end of file diff --git a/063_sparse_dispatch/deltanet_3of5_s256_best.pt b/063_sparse_dispatch/deltanet_3of5_s256_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa863df509b4ce6c2445e1c4d587bf41e9ed96c7 --- /dev/null +++ b/063_sparse_dispatch/deltanet_3of5_s256_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2acb784bfb9ee731932c5f06040fc72b1302ddc9dac5f4f3c22c7ea04050799d +size 352684297 diff --git a/063_sparse_dispatch/deltanet_3of5_s256_results.json b/063_sparse_dispatch/deltanet_3of5_s256_results.json new file mode 100644 index 0000000000000000000000000000000000000000..e0d12033f2f7e643d20083c138dc8c5b238ac5dc --- /dev/null +++ b/063_sparse_dispatch/deltanet_3of5_s256_results.json @@ -0,0 +1,2059 @@ +{ + "label": "deltanet_3of5_s256", + "seed": 256, + "attention_type": "deltanet", + "num_thin_blocks": 5, + "num_shared_blocks": 0, + "top_k_blocks": 3, + "total_params": 88089408, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.070488929748535, + 16.197874069213867, + 8.383760929107666, + 7.285171031951904, + 7.245490550994873, + 7.076878786087036, + 6.9669599533081055, + 6.752969264984131, + 6.640660524368286, + 6.418508052825928, + 6.461064577102661, + 6.317113399505615, + 6.216515779495239, + 6.108285188674927, + 6.049522161483765, + 5.9669084548950195, + 5.903541803359985, + 5.871365547180176, + 5.776335954666138, + 5.771231651306152, + 5.789379358291626, + 5.67598295211792, + 5.639819383621216, + 5.467181444168091, + 5.503422021865845, + 5.343157052993774, + 5.3956427574157715, + 5.3421125411987305, + 5.25864315032959, + 5.129066228866577, + 5.267850160598755, + 5.264739513397217, + 5.053381681442261, + 5.201064586639404, + 5.15140962600708, + 5.007229804992676, + 5.046075344085693, + 4.981792211532593, + 5.0182061195373535, + 5.016430377960205, + 5.0664613246917725, + 5.064194440841675, + 4.85503363609314, + 4.994741916656494, + 4.923568248748779, + 4.901627063751221, + 4.893970489501953, + 4.770350694656372, + 4.8790295124053955, + 4.803880214691162, + 4.584028005599976, + 4.811565160751343, + 4.749274969100952, + 4.765066385269165, + 4.867171049118042, + 4.731847286224365, + 4.705142974853516, + 4.73606014251709, + 4.574234247207642, + 4.689075469970703, + 4.675455808639526, + 4.5540008544921875, + 4.583674669265747, + 4.647869110107422, + 4.610080718994141, + 4.54615592956543, + 4.504095554351807, + 4.622476577758789, + 4.616280555725098, + 4.605635643005371, + 4.667905330657959, + 4.541551828384399, + 4.4984941482543945, + 4.5452094078063965, + 4.429370641708374, + 4.492565393447876, + 4.475898265838623, + 4.577865362167358, + 4.350993871688843, + 4.502145528793335, + 4.546029567718506, + 4.450871467590332, + 4.365506887435913, + 4.441570281982422, + 4.299476146697998, + 4.340014219284058, + 4.4471752643585205, + 4.432770013809204, + 4.420640230178833, + 4.366755723953247, + 4.376517295837402, + 4.338333606719971, + 4.402202129364014, + 4.308491945266724, + 4.339515686035156, + 4.35455846786499, + 4.35521674156189, + 4.3662965297698975, + 4.10907769203186, + 4.352291107177734, + 4.315929412841797, + 4.233549118041992, + 4.349750280380249, + 4.175265312194824, + 4.46806001663208, + 4.28252911567688, + 4.2831175327301025, + 4.275463104248047, + 4.349050283432007, + 4.279146909713745, + 4.2604594230651855, + 4.27384352684021, + 4.344712495803833, + 4.307395935058594, + 4.226639032363892, + 4.204847812652588, + 4.268575191497803, + 4.23178505897522, + 4.278383255004883, + 4.262828826904297, + 4.124202489852905, + 4.182528495788574, + 4.091794013977051, + 4.191782236099243, + 4.306813478469849, + 4.179495811462402, + 4.125201225280762, + 4.056956768035889, + 3.976455330848694, + 4.135244846343994, + 4.1546807289123535, + 4.07195782661438, + 4.1974077224731445, + 4.179563999176025, + 4.096607685089111, + 4.094249844551086, + 4.169165372848511, + 4.05410099029541, + 4.1548895835876465, + 4.078669786453247, + 3.982259750366211, + 4.1079161167144775, + 4.074643850326538, + 4.014505505561829, + 4.131140470504761, + 4.066274404525757, + 4.087463021278381, + 4.132949113845825, + 3.9621230363845825, + 4.097449541091919, + 4.128808498382568, + 4.132335186004639, + 4.200136661529541, + 3.9783642292022705, + 4.0163856744766235, + 3.989811658859253, + 4.157412767410278, + 4.095746994018555, + 3.8641222715377808, + 4.035767078399658, + 4.128614187240601, + 4.193820238113403, + 3.963891863822937, + 3.823011040687561, + 4.129372835159302, + 4.07855749130249, + 4.038941144943237, + 4.046957015991211, + 3.9952704906463623, + 3.991152763366699, + 4.036816358566284, + 4.116659879684448, + 3.9683709144592285, + 4.125774621963501, + 3.9810744524002075, + 4.107434272766113, + 4.125850677490234, + 3.921272039413452, + 3.986000418663025, + 3.931007504463196, + 4.038409948348999, + 4.087373733520508, + 3.910524606704712, + 3.9727402925491333, + 4.128594875335693, + 3.9144684076309204, + 3.994239330291748, + 4.037198424339294, + 4.020049095153809, + 3.993860125541687, + 3.9839394092559814, + 4.015908718109131, + 3.9163988828659058, + 3.998085856437683, + 3.8768482208251953, + 4.054077386856079, + 3.969451069831848, + 3.8453396558761597, + 3.934442400932312, + 4.073631048202515, + 3.9756500720977783, + 4.037248730659485, + 3.929473638534546, + 3.9318026304244995, + 3.989142417907715, + 4.093991756439209, + 3.9503061771392822, + 3.8932143449783325, + 4.0218788385391235, + 4.045052766799927, + 3.9110440015792847, + 3.907576322555542, + 3.864227771759033, + 3.907152533531189, + 3.9526829719543457, + 3.9615570306777954, + 3.9201109409332275, + 3.931860566139221, + 3.8923858404159546, + 3.916226625442505, + 4.074283719062805, + 3.9223625659942627, + 3.936587929725647, + 3.8760796785354614, + 3.806334137916565, + 3.8842989206314087, + 3.890386462211609, + 3.8341647386550903, + 3.969871163368225, + 3.9380706548690796, + 3.9133214950561523, + 3.8339357376098633, + 3.997219443321228, + 3.8251919746398926, + 3.9545156955718994, + 3.9651997089385986, + 3.9192649126052856, + 3.965295433998108, + 3.8798389434814453, + 3.871444344520569, + 3.9117531776428223, + 3.8287066221237183, + 3.7705976963043213, + 3.83825421333313, + 3.8113001585006714, + 3.7782063484191895, + 3.784410238265991, + 3.8722920417785645, + 3.809239983558655, + 3.936600089073181, + 3.8499927520751953, + 3.8972198963165283, + 3.900777578353882, + 3.739751935005188, + 3.7594629526138306, + 3.7101285457611084, + 3.884038805961609, + 3.815091848373413, + 3.872146487236023, + 3.787809729576111, + 3.647059202194214, + 3.6666473150253296, + 3.7953460216522217, + 3.750231981277466, + 3.827580213546753, + 3.7961214780807495, + 3.689079761505127, + 3.7242785692214966, + 3.7751914262771606, + 3.723623752593994, + 3.784478545188904, + 3.7829378843307495, + 3.8218876123428345, + 3.7502089738845825, + 3.7622896432876587, + 3.769646644592285, + 3.83472216129303, + 3.7823925018310547, + 3.762119174003601, + 3.7317538261413574, + 3.774401307106018, + 3.8924931287765503, + 3.83337938785553, + 3.732750415802002, + 3.6735700368881226, + 3.670088052749634, + 3.8160005807876587, + 3.808514714241028, + 3.7415300607681274, + 3.802563190460205, + 3.772758722305298, + 3.8148345947265625, + 3.7762908935546875, + 3.689322829246521, + 3.8077633380889893, + 3.8400877714157104, + 3.772662401199341, + 3.7047170400619507, + 3.8461050987243652, + 3.8054476976394653, + 3.713488221168518, + 3.6996461153030396, + 3.7003774642944336, + 3.641313672065735, + 3.6950595378875732, + 3.8537336587905884, + 3.7644237279891968, + 3.6971877813339233, + 3.884180188179016, + 3.722131609916687, + 3.763922333717346, + 3.8206136226654053, + 3.765533685684204, + 3.7470602989196777, + 3.7350692749023438, + 3.902188777923584, + 3.8492226600646973, + 3.811231017112732, + 3.821281313896179, + 3.8714747428894043, + 3.8279805183410645, + 3.677379846572876, + 3.75596022605896, + 3.870705485343933, + 3.7094240188598633, + 3.83516001701355, + 3.711748480796814, + 3.9460361003875732, + 3.875521421432495, + 3.7680081129074097, + 3.5498554706573486, + 3.8333137035369873, + 3.689704656600952, + 3.6277530193328857, + 3.7673258781433105, + 3.781683087348938, + 3.694435238838196, + 3.7314648628234863, + 3.7434829473495483, + 3.692585587501526, + 3.8364336490631104, + 3.7726006507873535, + 3.760796070098877, + 3.6638962030410767, + 3.592347502708435, + 3.8293161392211914, + 3.6995514631271362, + 3.7511982917785645, + 3.7694520950317383, + 3.7348815202713013, + 3.7738953828811646, + 3.6726702451705933, + 3.719177007675171, + 3.6727895736694336, + 3.7569245100021362, + 3.719396710395813, + 3.535669684410095, + 3.822334885597229, + 3.7597776651382446, + 3.632578134536743, + 3.632954478263855, + 3.7452704906463623, + 3.7728339433670044, + 3.6666877269744873, + 3.7158987522125244, + 3.8746774196624756, + 3.7388473749160767, + 3.819368004798889, + 3.68988299369812, + 3.7283358573913574, + 3.5898385047912598, + 3.7166759967803955, + 3.7400418519973755, + 3.680847644805908, + 3.7334307432174683, + 3.66620934009552, + 3.584657073020935, + 3.7550095319747925, + 3.7946081161499023, + 3.474698543548584, + 3.50434410572052, + 3.6562135219573975, + 3.510784387588501, + 3.749587655067444, + 3.536706566810608, + 3.6290805339813232, + 3.6518824100494385, + 3.669892430305481, + 3.7450441122055054, + 3.585960865020752, + 3.6596343517303467, + 3.614559531211853, + 3.5443427562713623, + 3.6504454612731934, + 3.636035203933716, + 3.630205512046814, + 3.6112334728240967, + 3.6190887689590454, + 3.5989354848861694, + 3.4565680027008057, + 3.5489693880081177, + 3.6928107738494873, + 3.42383074760437, + 3.627388119697571, + 3.5651371479034424, + 3.587159514427185, + 3.5281691551208496, + 3.6887401342391968, + 3.6756311655044556, + 3.60009503364563, + 3.651123046875, + 3.752449631690979, + 3.62577486038208, + 3.7131032943725586, + 3.6597282886505127, + 3.673322081565857, + 3.661511182785034, + 3.586289167404175, + 3.599435329437256, + 3.6308850049972534, + 3.6559202671051025, + 3.58467960357666, + 3.6606816053390503, + 3.602426528930664, + 3.763862729072571, + 3.594746947288513, + 3.603905200958252, + 3.7271907329559326, + 3.6065064668655396, + 3.6542649269104004, + 3.643715262413025, + 3.6184111833572388, + 3.5325552225112915, + 3.7353515625, + 3.4744468927383423, + 3.5646543502807617, + 3.642720937728882, + 3.68129301071167, + 3.6455708742141724, + 3.5693178176879883, + 3.5548548698425293, + 3.6624614000320435, + 3.756271481513977, + 3.6088438034057617, + 3.4807010889053345, + 3.551961898803711, + 3.6269015073776245, + 3.5709950923919678, + 3.615769386291504, + 3.5815807580947876, + 3.572413682937622, + 3.7168378829956055, + 3.489501476287842, + 3.650404214859009, + 3.6149818897247314, + 3.672079920768738, + 3.6312445402145386, + 3.5721296072006226, + 3.4901145696640015, + 3.655793786048889, + 3.744305968284607, + 3.5408071279525757, + 3.594594359397888, + 3.609003186225891, + 3.5922346115112305, + 3.578027129173279, + 3.552927613258362, + 3.587171196937561, + 3.6066004037857056, + 3.595477342605591, + 3.6985135078430176, + 3.578312873840332, + 3.558108925819397, + 3.662604331970215, + 3.613031506538391, + 3.5581109523773193, + 3.6744308471679688, + 3.559236764907837, + 3.6939783096313477, + 3.567178726196289, + 3.695218563079834, + 3.5501201152801514, + 3.4418193101882935, + 3.541283369064331, + 3.578117847442627, + 3.56337833404541, + 3.585649847984314, + 3.493496894836426, + 3.5707181692123413, + 3.5810118913650513, + 3.6352959871292114, + 3.57122266292572, + 3.6019866466522217, + 3.596324920654297, + 3.6784350872039795, + 3.5380364656448364, + 3.5683515071868896, + 3.6007111072540283, + 3.538956642150879, + 3.662197709083557, + 3.651126265525818, + 3.6386884450912476, + 3.6511093378067017, + 3.603049397468567, + 3.7285237312316895, + 3.658076763153076, + 3.423938751220703, + 3.474487066268921, + 3.434968948364258, + 3.4691120386123657, + 3.506085157394409, + 3.3879865407943726, + 3.3446273803710938, + 3.48216450214386, + 3.4988009929656982, + 3.491677403450012, + 3.4721795320510864, + 3.4096227884292603, + 3.513189673423767, + 3.5115933418273926, + 3.4046525955200195, + 3.4491126537323, + 3.3891180753707886, + 3.4926265478134155, + 3.3761074542999268, + 3.633254647254944, + 3.5508228540420532, + 3.5796533823013306, + 3.5583380460739136, + 3.456549048423767, + 3.4640259742736816, + 3.510334849357605, + 3.5150198936462402, + 3.606967568397522, + 3.475376009941101, + 3.480979561805725, + 3.37176513671875, + 3.5133237838745117, + 3.4113324880599976, + 3.522747278213501, + 3.5091217756271362, + 3.4498064517974854, + 3.4420629739761353, + 3.467380166053772, + 3.460078477859497, + 3.6345385313034058, + 3.4722307920455933, + 3.469461679458618, + 3.6074180603027344, + 3.5428173542022705, + 3.469629645347595, + 3.5247998237609863, + 3.474055528640747, + 3.423362374305725, + 3.557559013366699, + 3.5446282625198364, + 3.4439340829849243, + 3.3868794441223145, + 3.566153407096863, + 3.470290780067444, + 3.5258913040161133, + 3.4391329288482666, + 3.543471574783325, + 3.5919898748397827, + 3.4176465272903442, + 3.5273886919021606, + 3.4026941061019897, + 3.4644426107406616, + 3.512492299079895, + 3.5242680311203003, + 3.6156429052352905, + 3.4838417768478394, + 3.472557306289673, + 3.4693514108657837, + 3.376757860183716, + 3.4685546159744263, + 3.5765957832336426, + 3.5068031549453735, + 3.514246344566345, + 3.6369489431381226, + 3.6038061380386353, + 3.429994583129883, + 3.4705963134765625, + 3.5466439723968506, + 3.523122549057007, + 3.51065731048584, + 3.5267117023468018, + 3.3690080642700195, + 3.5973565578460693, + 3.3905445337295532, + 3.25199818611145, + 3.37168550491333, + 3.481680393218994, + 3.5819584131240845, + 3.4867924451828003, + 3.543713688850403, + 3.4204131364822388, + 3.5069916248321533, + 3.4834916591644287, + 3.5175665616989136, + 3.423655152320862, + 3.517916202545166, + 3.471355438232422, + 3.5022042989730835, + 3.4719271659851074, + 3.4695308208465576, + 3.5566974878311157, + 3.437148690223694, + 3.4284987449645996, + 3.4536877870559692, + 3.389127016067505, + 3.5386933088302612, + 3.5679993629455566, + 3.4033936262130737, + 3.4123553037643433, + 3.5803287029266357, + 3.450579524040222, + 3.5394479036331177, + 3.435714840888977, + 3.433990001678467, + 3.437276244163513, + 3.490794062614441, + 3.531827211380005, + 3.4424593448638916, + 3.3888492584228516, + 3.618859648704529, + 3.3675308227539062, + 3.378718614578247, + 3.4012887477874756, + 3.5157657861709595, + 3.5943044424057007, + 3.43987238407135, + 3.445648431777954, + 3.410320520401001, + 3.3708317279815674, + 3.259817957878113, + 3.50541889667511, + 3.3630833625793457, + 3.269199013710022, + 3.4470221996307373, + 3.306577205657959, + 3.430361270904541, + 3.2607016563415527, + 3.349104881286621, + 3.3004400730133057, + 3.386443257331848, + 3.395419120788574, + 3.3269256353378296, + 3.3866920471191406, + 3.410520911216736, + 3.5156713724136353, + 3.2855430841445923, + 3.5299848318099976, + 3.419633984565735, + 3.5047407150268555, + 3.4116909503936768, + 3.5141561031341553, + 3.4305837154388428, + 3.465202212333679, + 3.375963807106018, + 3.4021328687667847, + 3.4171804189682007, + 3.30346143245697, + 3.4100282192230225, + 3.399503469467163, + 3.317202091217041, + 3.418726086616516, + 3.344210624694824, + 3.476285457611084, + 3.4761550426483154, + 3.4022902250289917, + 3.410346269607544, + 3.387152075767517, + 3.428146719932556, + 3.3606059551239014, + 3.4351619482040405, + 3.393306255340576, + 3.4369829893112183, + 3.3445050716400146, + 3.4889103174209595, + 3.3933494091033936, + 3.323814272880554, + 3.311417818069458, + 3.255605697631836, + 3.276581048965454, + 3.325914740562439, + 3.4668641090393066, + 3.452326536178589, + 3.388930082321167, + 3.49515962600708, + 3.472582459449768, + 3.282320261001587, + 3.405840516090393, + 3.4068987369537354, + 3.401857376098633, + 3.5023611783981323, + 3.406246066093445, + 3.376749038696289, + 3.358017921447754, + 3.380479574203491, + 3.4316776990890503, + 3.5199054479599, + 3.209040403366089, + 3.4427603483200073, + 3.4955270290374756, + 3.407845139503479, + 3.457017183303833, + 3.4100472927093506, + 3.3563657999038696, + 3.4017953872680664, + 3.259079337120056, + 3.4322317838668823, + 3.3761597871780396, + 3.449344277381897, + 3.3884332180023193, + 3.389167547225952, + 3.4186341762542725, + 3.458642601966858, + 3.3012794256210327, + 3.3114179372787476, + 3.3477284908294678, + 3.4215394258499146, + 3.4627753496170044, + 3.3577523231506348, + 3.37129545211792, + 3.510759115219116, + 3.4772053956985474, + 3.463297963142395, + 3.353865146636963, + 3.279980421066284, + 3.250464916229248, + 3.31849467754364, + 3.392053723335266, + 3.4981924295425415, + 3.411875605583191, + 3.3106569051742554, + 3.4185550212860107, + 3.404945969581604, + 3.455335259437561, + 3.2266844511032104, + 3.3432159423828125, + 3.413447618484497, + 3.5238611698150635, + 3.347116231918335, + 3.3838837146759033, + 3.317181944847107, + 3.3351333141326904, + 3.264041781425476, + 3.345522403717041, + 3.4317526817321777, + 3.316465973854065, + 3.42656409740448, + 3.3652454614639282, + 3.3210006952285767, + 3.325934648513794, + 3.3169318437576294, + 3.4546103477478027, + 3.419620394706726, + 3.3407405614852905, + 3.360917568206787, + 3.1783515214920044, + 3.2427867650985718, + 3.304649829864502, + 3.244579553604126, + 3.258529782295227, + 3.3433209657669067, + 3.1883453130722046, + 3.1863397359848022, + 3.3297908306121826, + 3.332509994506836, + 3.331916332244873, + 3.3073885440826416, + 3.1920011043548584, + 3.3132028579711914, + 3.127058982849121, + 3.20893657207489, + 3.302273988723755, + 3.2962846755981445, + 3.2150511741638184, + 3.306158661842346, + 3.1604148149490356, + 3.2710092067718506, + 3.270421862602234, + 3.4721704721450806, + 3.37467098236084, + 3.4077669382095337, + 3.2863752841949463, + 3.1396586894989014, + 3.2833712100982666, + 3.2641429901123047, + 3.4535189867019653, + 3.2886388301849365, + 3.2747195959091187, + 3.3055588006973267, + 3.4001543521881104, + 3.4310219287872314, + 3.3141942024230957, + 3.2714775800704956, + 3.282470464706421, + 3.2765690088272095, + 3.304901123046875, + 3.287634015083313, + 3.334359049797058, + 3.2705577611923218, + 3.262386918067932, + 3.2214690446853638, + 3.2717336416244507, + 3.283029794692993, + 3.24992835521698, + 3.222260594367981, + 3.488537073135376, + 3.3568177223205566, + 3.338338613510132, + 3.3572922945022583, + 3.321294069290161, + 3.256870150566101, + 3.226378083229065, + 3.376546621322632, + 3.1842352151870728, + 3.281364917755127, + 3.246510863304138, + 3.1979280710220337, + 3.3181270360946655, + 3.2966448068618774, + 3.2025973796844482, + 3.1477246284484863, + 3.3676960468292236, + 3.349453330039978, + 3.4548405408859253, + 3.5529810190200806, + 3.3672103881835938, + 3.3941314220428467, + 3.1477043628692627, + 3.3216389417648315, + 3.3397538661956787, + 3.377160668373108, + 3.280761480331421, + 3.2479032278060913, + 3.361589789390564, + 3.178216576576233, + 3.286510705947876, + 3.3356043100357056, + 3.3804118633270264, + 3.416625142097473, + 3.316919445991516, + 3.295617938041687, + 3.30878484249115, + 3.3210724592208862, + 3.2610878944396973, + 3.4287840127944946, + 3.3268343210220337, + 3.3888102769851685, + 3.306909680366516, + 3.3515454530715942, + 3.352355122566223, + 3.30206835269928, + 3.3729195594787598, + 3.3180547952651978, + 3.2342370748519897, + 3.2476900815963745, + 3.2640998363494873, + 3.3137176036834717, + 3.256004571914673, + 3.32066011428833, + 3.2589999437332153, + 3.32186222076416, + 3.274621605873108, + 3.2431788444519043, + 3.300722599029541, + 3.400347590446472, + 3.333345651626587, + 3.3149449825286865, + 3.2858846187591553, + 3.3291205167770386, + 3.176385283470154, + 3.376481771469116, + 3.346725344657898, + 3.350654363632202, + 3.3290337324142456, + 3.2482688426971436, + 3.32470440864563, + 3.28462815284729, + 3.2426886558532715, + 3.4340453147888184, + 3.3747920989990234, + 3.2850961685180664, + 3.2278337478637695, + 3.386189818382263, + 3.3492006063461304, + 3.2119829654693604, + 3.4368377923965454, + 3.1939116716384888, + 3.2746381759643555, + 3.2858341932296753, + 3.185531497001648, + 3.2561964988708496, + 3.129492998123169, + 3.28571879863739, + 3.314669370651245, + 3.339518189430237, + 3.097015857696533, + 3.243474245071411, + 3.223570704460144, + 3.2755619287490845, + 3.2581193447113037, + 3.3598098754882812, + 3.2813245058059692, + 3.23192298412323, + 3.0820637941360474, + 3.3512046337127686, + 3.145097255706787, + 3.1687538623809814, + 3.229086756706238, + 3.1587719917297363, + 3.377452850341797, + 3.364911675453186, + 3.1682153940200806, + 3.2020076513290405, + 3.2446696758270264, + 3.1447218656539917, + 3.3115551471710205, + 3.2251129150390625, + 3.1847890615463257, + 3.1153584718704224, + 3.2005481719970703, + 3.2250016927719116, + 3.3471964597702026, + 3.4869260787963867, + 3.3283767700195312, + 3.27461576461792, + 3.3212212324142456, + 3.2397589683532715, + 3.306952476501465, + 3.2339625358581543, + 3.2852327823638916, + 3.273952603340149, + 3.2060569524765015, + 3.109536647796631, + 3.299519181251526, + 3.234908938407898, + 3.219856858253479, + 3.163646697998047, + 3.1696579456329346, + 3.2413289546966553, + 3.1072869300842285, + 3.353673577308655, + 3.259766459465027, + 3.2586097717285156, + 3.3076491355895996, + 3.1673911809921265, + 3.1798830032348633, + 3.281601309776306, + 3.260176181793213, + 3.197658896446228, + 3.3073015213012695, + 3.251469850540161, + 3.326173782348633, + 3.179431438446045, + 3.2092628479003906, + 3.293675661087036, + 3.176617383956909, + 3.2594127655029297, + 3.263038992881775, + 3.391505479812622, + 3.2358338832855225, + 3.300184965133667, + 3.285034656524658, + 3.3801103830337524, + 3.2389183044433594, + 3.1704585552215576, + 3.222731590270996, + 3.257646679878235, + 3.3075900077819824, + 3.256842851638794, + 3.2259494066238403, + 3.232935905456543, + 3.215555429458618, + 3.279793620109558, + 3.2217772006988525, + 3.2622461318969727, + 3.124892830848694, + 3.414165496826172, + 3.33297336101532, + 3.175740599632263, + 3.329282283782959, + 3.3415517807006836, + 3.1305129528045654, + 3.279018521308899, + 3.2997188568115234, + 3.297519326210022, + 3.3397656679153442, + 3.1025798320770264, + 3.145137667655945, + 3.2996450662612915, + 3.1993175745010376, + 3.234130859375, + 3.3215787410736084, + 3.2952723503112793, + 3.1628369092941284, + 3.26202929019928 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 113.74194565190241, + 68.28079784632587, + 55.32987621625877, + 48.75364446628124, + 44.607366243442826, + 42.16888126672351, + 39.94429798756659, + 38.3007750982554, + 36.75759988107576, + 35.362297382770286, + 34.64590187163361, + 33.403635764629854, + 32.77797037422077, + 32.11540015907494, + 31.420715559790995, + 31.237302592357608, + 30.814845865015787, + 30.729747683573258, + 30.636905975063762, + 30.609614803186656 + ], + "best_ppl": 30.609614803186656, + "train_time_s": 22177.277688503265 +} \ No newline at end of file diff --git a/063_sparse_dispatch/shared_1plus2of14_best.pt b/063_sparse_dispatch/shared_1plus2of14_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..51290ce8e1d56494039af30bce5b3ba3344eab8c --- /dev/null +++ b/063_sparse_dispatch/shared_1plus2of14_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99e0185fd01d7a965167c91a20202e7ce25d2ebb23a7f5244fe3f8d1c7f28724 +size 762119211 diff --git a/063_sparse_dispatch/shared_1plus2of14_results.json b/063_sparse_dispatch/shared_1plus2of14_results.json new file mode 100644 index 0000000000000000000000000000000000000000..cc608223776f9131f8e88f481dce8d42c7d341ee --- /dev/null +++ b/063_sparse_dispatch/shared_1plus2of14_results.json @@ -0,0 +1,508 @@ +{ + "label": "shared_1plus2of14", + "attention_mode": "sparse", + "top_k_blocks": 2, + "num_thin_blocks": 15, + "num_shared_blocks": 1, + "total_params": 190355456, + "steps": [ + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 2.9015581011772156, + 3.0877742767333984, + 3.0060471892356873, + 2.979893147945404, + 2.9070188403129578, + 2.9616722464561462, + 2.9606329798698425, + 2.952632486820221, + 2.985970377922058, + 2.9885979294776917, + 2.959640681743622, + 2.9275310039520264, + 2.9644022583961487, + 3.028023064136505, + 2.8567745685577393, + 2.981859505176544, + 2.9161601066589355, + 2.9018890261650085, + 3.0053383111953735, + 2.981068193912506, + 3.2714539766311646, + 3.200727105140686, + 3.2772529125213623, + 3.3006343245506287, + 3.224383592605591, + 3.2366743087768555, + 3.328200042247772, + 3.2752211689949036, + 3.126257836818695, + 3.2665703892707825, + 3.2506250739097595, + 3.256326138973236, + 3.1124512553215027, + 3.2942296266555786, + 3.3704161047935486, + 3.3830237984657288, + 3.131134808063507, + 3.3166852593421936, + 3.1025359630584717, + 3.339309334754944, + 3.389427065849304, + 3.252554178237915, + 3.2779111862182617, + 3.233321487903595, + 3.213325083255768, + 3.257583796977997, + 3.3224533200263977, + 3.3595325350761414, + 3.446103036403656, + 3.2393237948417664, + 3.5256888270378113, + 3.399535596370697, + 3.315065622329712, + 3.27653169631958, + 3.354659676551819, + 3.330633521080017, + 3.5394102334976196, + 3.2856783270835876, + 3.437723755836487, + 3.318567931652069, + 3.766273021697998, + 3.7216588258743286, + 3.8072307109832764, + 3.7736505270004272, + 3.7691721320152283, + 3.7887516021728516, + 3.856812059879303, + 3.874578297138214, + 3.9427862763404846, + 3.8634148240089417, + 3.7983070611953735, + 3.8670894503593445, + 3.7252750396728516, + 3.648626923561096, + 3.8420876264572144, + 3.5890191197395325, + 3.779814302921295, + 3.776054799556732, + 3.775964617729187, + 3.7213558554649353, + 3.7100287675857544, + 3.831614077091217, + 3.8754929900169373, + 3.6624096035957336, + 3.70512455701828, + 3.7648224234580994, + 3.869477331638336, + 3.666087567806244, + 3.692387044429779, + 3.963961660861969, + 3.7574403882026672, + 3.737488627433777, + 3.754002809524536, + 3.7285738587379456, + 3.8595767617225647, + 3.792065441608429, + 3.710012435913086, + 3.826682209968567, + 3.630021393299103, + 3.835118293762207, + 3.877008378505707, + 3.6724842190742493, + 3.7637985944747925, + 3.683578908443451, + 3.7392765879631042, + 3.7953012585639954, + 3.6229681372642517, + 3.693288505077362, + 3.711137890815735, + 3.9235337376594543, + 3.803937792778015, + 3.875222861766815, + 3.749752104282379, + 3.715513229370117, + 3.757683515548706, + 3.6391186714172363, + 3.665630578994751, + 3.688731014728546, + 3.713706314563751, + 3.8073756098747253, + 3.796328842639923, + 3.9164799451828003, + 3.63570237159729, + 3.673487901687622, + 3.698325753211975, + 3.8453012108802795, + 3.5700666308403015, + 3.454531729221344, + 3.605205476284027, + 3.550259292125702, + 3.4428544640541077, + 3.4148746132850647, + 3.527815878391266, + 3.3878287076950073, + 3.4765297770500183, + 3.3848544359207153, + 3.427438199520111, + 3.446206271648407, + 3.379789888858795, + 3.4588151574134827, + 3.2954816818237305, + 3.289965569972992, + 3.3505231738090515, + 3.345255970954895, + 3.5421721935272217, + 3.556142032146454, + 3.5374680161476135, + 3.5906589031219482, + 3.245023012161255, + 3.255285382270813, + 3.434578001499176, + 3.4980711936950684, + 3.553148567676544, + 3.4331920742988586, + 3.470349073410034, + 3.5473925471305847, + 3.5164272785186768, + 3.517852544784546, + 3.354642331600189, + 3.4197571873664856, + 3.483809173107147, + 3.5518792867660522, + 3.577494263648987, + 3.4167940616607666, + 3.563300132751465, + 3.4134522676467896, + 3.3110046982765198, + 3.5615519285202026, + 3.446166515350342, + 3.465098202228546, + 3.454281270503998, + 3.3804754614830017, + 3.394438147544861, + 3.4428005814552307, + 3.551225006580353, + 3.452974796295166, + 3.44801664352417, + 3.3881312012672424, + 3.3247726559638977, + 3.4634634256362915, + 3.395548164844513, + 3.51163512468338, + 3.349991202354431, + 3.4572901725769043, + 3.412857949733734, + 3.5346421599388123, + 3.4644089937210083, + 3.3988868594169617, + 3.3962671756744385, + 3.572585880756378, + 3.3299076557159424, + 3.4451955556869507, + 3.47091281414032, + 3.4972103238105774, + 3.436339259147644, + 3.435834586620331, + 3.512551724910736, + 3.4304845333099365, + 3.2975882291793823, + 3.5106979608535767, + 3.5004570484161377, + 3.4723839163780212, + 3.4300034642219543, + 3.5481141805648804, + 3.4319798946380615, + 3.4638107419013977, + 3.4913118481636047, + 3.442385256290436, + 3.5069888830184937, + 3.4873231053352356, + 3.3946642875671387, + 3.4911097288131714, + 3.5648680329322815, + 3.4298973083496094, + 3.405434548854828, + 3.4540992975234985, + 3.447507619857788, + 3.434307873249054, + 3.556748926639557, + 3.330422341823578, + 3.436522364616394, + 3.647747814655304, + 3.577707052230835, + 3.404127538204193, + 3.3136566281318665, + 3.3757554292678833, + 3.483103036880493, + 3.4589837789535522, + 3.517594873905182, + 3.4277899861335754, + 3.408407986164093, + 3.3644352555274963, + 3.3004584908485413, + 3.5263137221336365, + 3.5759807229042053, + 3.465166926383972, + 3.432847499847412, + 3.6107040643692017, + 3.471543550491333, + 3.3828750252723694 + ], + "val_steps": [ + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 41.25568802497702, + 37.060070068835074, + 36.62768732075425, + 36.51926911018504, + 36.50631941181755 + ], + "best_ppl": 36.50631941181755, + "train_time_s": 11454.23207616806 +} \ No newline at end of file diff --git a/063_sparse_dispatch/shared_deltanet_1plus2of14_best.pt b/063_sparse_dispatch/shared_deltanet_1plus2of14_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e63c4db7e9cf912fd266e663e03bf123839ff57 --- /dev/null +++ b/063_sparse_dispatch/shared_deltanet_1plus2of14_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:919074e8efde1afd5334666fe70e9bf5941a37963efb36ff04fb12bdebfca8a5 +size 793621220 diff --git a/063_sparse_dispatch/shared_deltanet_1plus2of14_results.json b/063_sparse_dispatch/shared_deltanet_1plus2of14_results.json new file mode 100644 index 0000000000000000000000000000000000000000..3729904a89f9c0c1d8892f75da74f3e757f41074 --- /dev/null +++ b/063_sparse_dispatch/shared_deltanet_1plus2of14_results.json @@ -0,0 +1,387 @@ +{ + "label": "shared_deltanet_1plus2of14", + "attention_mode": "sparse", + "routed_attn_type": "deltanet", + "top_k_blocks": 2, + "num_thin_blocks": 15, + "num_shared_blocks": 1, + "total_params": 198162304, + "steps": [ + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 3.1777361035346985, + 3.338351547718048, + 3.271555185317993, + 3.292502224445343, + 3.261455774307251, + 3.302325665950775, + 3.2950804233551025, + 3.4235018491744995, + 3.360715329647064, + 3.371263086795807, + 3.3345319032669067, + 3.193874418735504, + 3.2726223468780518, + 3.353284537792206, + 3.1914132237434387, + 3.2872437238693237, + 3.2940871715545654, + 3.1432998180389404, + 3.239644408226013, + 3.3787192702293396, + 3.3763453364372253, + 3.3380585312843323, + 3.2475173473358154, + 3.3093941807746887, + 3.2867513298988342, + 3.4278697967529297, + 3.205827832221985, + 3.236877143383026, + 3.2560861110687256, + 3.2242335081100464, + 3.406534790992737, + 3.2051538825035095, + 3.264488101005554, + 3.26824027299881, + 3.18367463350296, + 3.2046486735343933, + 3.197915494441986, + 3.2920270562171936, + 3.4991849064826965, + 3.3752328753471375, + 3.3344690799713135, + 3.252140700817108, + 3.3613393902778625, + 3.2992374897003174, + 3.3221980333328247, + 3.3080338835716248, + 3.2292964458465576, + 3.313994348049164, + 3.44545578956604, + 3.3007100224494934, + 3.194986641407013, + 3.3050803542137146, + 3.3802170157432556, + 3.3158013224601746, + 3.3206913471221924, + 3.2971912622451782, + 3.2400403022766113, + 3.403756022453308, + 3.353538990020752, + 3.206424593925476, + 3.2477879524230957, + 3.4066471457481384, + 3.3296971917152405, + 3.2992972135543823, + 3.3088077306747437, + 3.347862660884857, + 3.303692936897278, + 3.2345616221427917, + 3.3678589463233948, + 3.256811499595642, + 3.3792750239372253, + 3.2406296133995056, + 3.1156325340270996, + 3.179192304611206, + 3.3301971554756165, + 3.3071919679641724, + 3.2457733750343323, + 3.310214638710022, + 3.2419418692588806, + 3.320328414440155, + 3.3292152881622314, + 3.2412291765213013, + 3.41629821062088, + 3.3142210841178894, + 3.3168915510177612, + 3.3548924326896667, + 3.3303261399269104, + 3.26606285572052, + 3.450722813606262, + 3.3263164162635803, + 3.367241084575653, + 3.322207748889923, + 3.316314935684204, + 3.305887997150421, + 3.314009189605713, + 3.384882926940918, + 3.3008070588111877, + 3.347536087036133, + 3.3809441924095154, + 3.275594413280487, + 3.2994426488876343, + 3.3566797971725464, + 3.3669636845588684, + 3.2407862544059753, + 3.27314031124115, + 3.2802403569221497, + 3.2863895297050476, + 3.2457714080810547, + 3.1212738156318665, + 3.462621510028839, + 3.271378219127655, + 3.2616870403289795, + 3.287158191204071, + 3.199742376804352, + 3.2531277537345886, + 3.319532334804535, + 3.369244158267975, + 3.3616055846214294, + 3.258890926837921, + 3.3080557584762573, + 3.1819931268692017, + 3.195812940597534, + 3.381825625896454, + 3.4110931158065796, + 3.385704815387726, + 3.3306893706321716, + 3.1121129989624023, + 3.1711443066596985, + 3.2516523003578186, + 3.238161265850067, + 3.0427581667900085, + 3.133829176425934, + 3.206386685371399, + 3.2603593468666077, + 3.1881086826324463, + 3.190523087978363, + 3.18367075920105, + 3.222419321537018, + 3.295727014541626, + 3.2614758610725403, + 3.27193421125412, + 3.262248158454895, + 3.1637876629829407, + 3.2481150031089783, + 3.214955449104309, + 3.3274702429771423, + 3.36566299200058, + 3.2292956113815308, + 3.102342426776886, + 3.2513662576675415, + 3.252710461616516, + 3.2713785767555237, + 3.1305034160614014, + 3.3133273124694824, + 3.361756980419159, + 3.222153067588806, + 3.2048738598823547, + 3.241790533065796, + 3.267791450023651, + 3.199027180671692, + 3.3452389240264893, + 3.1349271535873413, + 3.15665864944458, + 3.189565122127533, + 3.245523452758789, + 3.1507402062416077, + 3.1808746457099915, + 3.3116108775138855, + 3.232621133327484, + 3.363927185535431, + 3.1451085805892944, + 3.206853747367859, + 3.195546865463257, + 3.1056870818138123, + 3.2210575938224792, + 3.237958312034607, + 3.1880670189857483, + 3.1151487827301025, + 3.2286102771759033, + 3.343379259109497 + ], + "val_steps": [ + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 33.06023241279871, + 32.804749635006544, + 32.74964090660532, + 32.749325928444215 + ], + "best_ppl": 32.749325928444215, + "train_time_s": 17265.204796552658 +} \ No newline at end of file diff --git a/063_sparse_dispatch/shared_deltanet_1plus2of5_best.pt b/063_sparse_dispatch/shared_deltanet_1plus2of5_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee73345a2e46382b04a6b9d9e2ca57433b96eff4 --- /dev/null +++ b/063_sparse_dispatch/shared_deltanet_1plus2of5_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:471537128042b7d7dc396eb20f9852dd3e2acc956debbc6ed3c71ae4ee7e665b +size 350408128 diff --git a/063_sparse_dispatch/shared_deltanet_1plus2of5_results.json b/063_sparse_dispatch/shared_deltanet_1plus2of5_results.json new file mode 100644 index 0000000000000000000000000000000000000000..6ffef4d9b417956c82d9039f1998834150027f75 --- /dev/null +++ b/063_sparse_dispatch/shared_deltanet_1plus2of5_results.json @@ -0,0 +1,1897 @@ +{ + "label": "shared_deltanet_1plus2of5", + "attention_mode": "sparse", + "routed_attn_type": "deltanet", + "top_k_blocks": 2, + "num_thin_blocks": 5, + "num_shared_blocks": 1, + "total_params": 87523584, + "steps": [ + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 4.5024425983428955, + 4.468092918395996, + 4.429566860198975, + 4.560781478881836, + 4.413970470428467, + 4.311157464981079, + 4.488478660583496, + 4.41364312171936, + 4.3376405239105225, + 4.288746118545532, + 4.506059885025024, + 4.299712896347046, + 4.233811855316162, + 4.324281930923462, + 4.355874061584473, + 4.329144239425659, + 4.4718077182769775, + 4.3286216259002686, + 4.236961603164673, + 4.218304634094238, + 4.3703320026397705, + 4.3349573612213135, + 4.331709623336792, + 4.378791093826294, + 4.275615453720093, + 4.277340650558472, + 4.238267421722412, + 4.385381460189819, + 4.064323306083679, + 4.334716320037842, + 4.21702241897583, + 4.323324680328369, + 4.3267982006073, + 4.263216972351074, + 4.344519853591919, + 4.299509048461914, + 4.013149380683899, + 4.313512325286865, + 4.2620322704315186, + 4.224957466125488, + 4.329805850982666, + 4.305279970169067, + 4.243348836898804, + 4.292041778564453, + 4.298918724060059, + 4.289833068847656, + 4.133715629577637, + 4.298936128616333, + 4.167287111282349, + 4.233927488327026, + 4.183442115783691, + 4.096070766448975, + 4.154967308044434, + 4.167285442352295, + 4.136126279830933, + 4.245786666870117, + 4.170946836471558, + 4.130416631698608, + 4.208202123641968, + 4.145423173904419, + 4.114028453826904, + 4.019385576248169, + 4.182729482650757, + 4.329261064529419, + 4.062877655029297, + 4.24544095993042, + 4.319783926010132, + 4.085592746734619, + 4.110560417175293, + 4.0773186683654785, + 4.132183313369751, + 4.1558167934417725, + 4.16194486618042, + 4.090403318405151, + 3.985654830932617, + 4.088601350784302, + 4.1716814041137695, + 4.07709002494812, + 4.193751096725464, + 4.053304195404053, + 4.133044719696045, + 4.105878114700317, + 4.081300735473633, + 4.098048448562622, + 4.004588603973389, + 4.121993780136108, + 4.090872049331665, + 4.250115633010864, + 3.972145676612854, + 4.083713531494141, + 4.078212022781372, + 4.141880989074707, + 4.106627941131592, + 4.103417634963989, + 4.094427108764648, + 4.083117604255676, + 3.9998427629470825, + 4.016460061073303, + 4.0013532638549805, + 4.011073708534241, + 4.095525741577148, + 4.167583465576172, + 3.9838247299194336, + 3.9683178663253784, + 4.012606620788574, + 4.069121360778809, + 4.1097776889801025, + 4.074034214019775, + 4.029638767242432, + 4.083118677139282, + 4.0093066692352295, + 4.12061071395874, + 4.026567220687866, + 4.063299655914307, + 3.9766770601272583, + 3.8590123653411865, + 3.900875687599182, + 4.0140416622161865, + 4.0387667417526245, + 3.830199956893921, + 4.0081866979599, + 3.9501123428344727, + 4.054723501205444, + 4.029770851135254, + 3.8794217109680176, + 4.007715344429016, + 3.809969663619995, + 3.967097043991089, + 4.02674126625061, + 4.022894620895386, + 3.951961040496826, + 3.9209083318710327, + 3.904967427253723, + 3.974241018295288, + 3.951310634613037, + 3.9022144079208374, + 3.963714361190796, + 3.9573569297790527, + 3.926609516143799, + 3.977901339530945, + 3.885162591934204, + 3.9001221656799316, + 3.9381349086761475, + 3.906125068664551, + 3.8888442516326904, + 3.8674486875534058, + 3.900127410888672, + 3.874857544898987, + 4.0245795249938965, + 3.8734554052352905, + 3.9075413942337036, + 4.025832414627075, + 3.887361168861389, + 3.980427622795105, + 3.86274790763855, + 4.01492714881897, + 3.742480754852295, + 3.763209342956543, + 3.9467806816101074, + 3.9851306676864624, + 3.7893779277801514, + 3.9630502462387085, + 3.906758189201355, + 3.96023166179657, + 3.91033673286438, + 3.8919990062713623, + 3.8404940366744995, + 3.85002338886261, + 3.8296302556991577, + 3.8440003395080566, + 3.85775089263916, + 3.848176598548889, + 3.7992255687713623, + 3.8611841201782227, + 3.929301142692566, + 3.8713254928588867, + 3.801248073577881, + 3.947381377220154, + 3.8378196954727173, + 3.8926055431365967, + 3.9495214223861694, + 3.8436354398727417, + 3.787097454071045, + 3.9244099855422974, + 3.8860204219818115, + 3.841749668121338, + 3.8792335987091064, + 3.8747198581695557, + 3.9883562326431274, + 3.7784820795059204, + 3.8018314838409424, + 3.930801510810852, + 3.8347902297973633, + 3.8905043601989746, + 3.7092095613479614, + 3.9001749753952026, + 3.7912235260009766, + 3.7776650190353394, + 3.834569811820984, + 3.9527705907821655, + 3.8901541233062744, + 3.9876664876937866, + 3.9174797534942627, + 3.6839176416397095, + 3.764274477958679, + 3.799177885055542, + 3.840021848678589, + 3.9595015048980713, + 3.8038939237594604, + 3.993904232978821, + 3.7238502502441406, + 3.8281935453414917, + 3.8173753023147583, + 3.945682168006897, + 3.929852604866028, + 3.830051898956299, + 3.83593213558197, + 3.878395915031433, + 3.884958028793335, + 3.906067132949829, + 3.730005979537964, + 3.8031086921691895, + 3.806025505065918, + 3.822263240814209, + 3.8474212884902954, + 3.8043622970581055, + 3.8703161478042603, + 3.841068387031555, + 3.8488887548446655, + 3.9996330738067627, + 3.7569891214370728, + 3.7325679063796997, + 3.79353666305542, + 3.888606548309326, + 3.8929795026779175, + 3.815623164176941, + 3.834168314933777, + 3.8726627826690674, + 3.7598884105682373, + 3.796869993209839, + 3.8594954013824463, + 3.8379422426223755, + 3.813020348548889, + 3.8730764389038086, + 3.806831121444702, + 3.8565627336502075, + 3.8156384229660034, + 3.906099557876587, + 3.7554584741592407, + 3.8091214895248413, + 3.851854681968689, + 3.8599891662597656, + 3.790593147277832, + 3.646838665008545, + 3.605772018432617, + 3.5789860486984253, + 3.771310806274414, + 3.6618958711624146, + 3.5213563442230225, + 3.6762301921844482, + 3.696259021759033, + 3.6061981916427612, + 3.6927478313446045, + 3.7088736295700073, + 3.8558396100997925, + 3.6626170873641968, + 3.647547721862793, + 3.7418383359909058, + 3.62151300907135, + 3.6910526752471924, + 3.6497552394866943, + 3.698857069015503, + 3.8387376070022583, + 3.7894150018692017, + 3.6598894596099854, + 3.587012529373169, + 3.62155282497406, + 3.721085786819458, + 3.8066290616989136, + 3.623848557472229, + 3.7949001789093018, + 3.7451133728027344, + 3.6933270692825317, + 3.7018028497695923, + 3.739799380302429, + 3.48617947101593, + 3.745692014694214, + 3.724798798561096, + 3.695114254951477, + 3.6719712018966675, + 3.6504743099212646, + 3.7622772455215454, + 3.711551308631897, + 3.7524174451828003, + 3.5881030559539795, + 3.7226362228393555, + 3.7685132026672363, + 3.6303458213806152, + 3.7248165607452393, + 3.851698875427246, + 3.7034504413604736, + 3.6076329946517944, + 3.6520023345947266, + 3.6564563512802124, + 3.81744122505188, + 3.7753227949142456, + 3.700054168701172, + 3.714677929878235, + 3.7772079706192017, + 3.6887998580932617, + 3.6489373445510864, + 3.754772424697876, + 3.8281333446502686, + 3.6091595888137817, + 3.8249340057373047, + 3.7260574102401733, + 3.7141754627227783, + 3.6771105527877808, + 3.6149126291275024, + 3.7933530807495117, + 3.6727641820907593, + 3.675557017326355, + 3.5170291662216187, + 3.717762589454651, + 3.65347421169281, + 3.6737728118896484, + 3.724383592605591, + 3.7417144775390625, + 3.6886301040649414, + 3.7018572092056274, + 3.790647506713867, + 3.701850175857544, + 3.7597233057022095, + 3.6753650903701782, + 3.6245466470718384, + 3.6307015419006348, + 3.5898720026016235, + 3.7817214727401733, + 3.6386457681655884, + 3.7110711336135864, + 3.7234933376312256, + 3.6384037733078003, + 3.6212135553359985, + 3.7792398929595947, + 3.69230055809021, + 3.60904061794281, + 3.7706879377365112, + 3.5109076499938965, + 3.5830239057540894, + 3.705811858177185, + 3.790782332420349, + 3.7325403690338135, + 3.6006991863250732, + 3.688300371170044, + 3.6581190824508667, + 3.769905924797058, + 3.774994730949402, + 3.78274667263031, + 3.6983784437179565, + 3.7420722246170044, + 3.6480640172958374, + 3.630893111228943, + 3.5700706243515015, + 3.7814029455184937, + 3.6774849891662598, + 3.662749409675598, + 3.6243454217910767, + 3.728690266609192, + 3.699141263961792, + 3.766811966896057, + 3.753794550895691, + 3.5958582162857056, + 3.631053924560547, + 3.468283534049988, + 3.5892413854599, + 3.6111594438552856, + 3.6866722106933594, + 3.636015295982361, + 3.656308174133301, + 3.563575029373169, + 3.6483101844787598, + 3.64354407787323, + 3.615964889526367, + 3.4625725746154785, + 3.6128435134887695, + 3.569602608680725, + 3.527541756629944, + 3.613108277320862, + 3.6754400730133057, + 3.6374614238739014, + 3.679067611694336, + 3.5450977087020874, + 3.6050010919570923, + 3.5864319801330566, + 3.51783287525177, + 3.602718710899353, + 3.7193427085876465, + 3.5533705949783325, + 3.537316083908081, + 3.6609379053115845, + 3.547311305999756, + 3.4589943885803223, + 3.742072820663452, + 3.6176466941833496, + 3.620596766471863, + 3.5505062341690063, + 3.5658504962921143, + 3.5622748136520386, + 3.459414005279541, + 3.5190019607543945, + 3.4960083961486816, + 3.591121196746826, + 3.490598201751709, + 3.470845103263855, + 3.4876155853271484, + 3.429843783378601, + 3.511515974998474, + 3.5762925148010254, + 3.6423319578170776, + 3.571129322052002, + 3.4184606075286865, + 3.6751670837402344, + 3.54019296169281, + 3.6345505714416504, + 3.5005738735198975, + 3.7520506381988525, + 3.4919114112854004, + 3.563592553138733, + 3.4217573404312134, + 3.6205493211746216, + 3.503166675567627, + 3.6566691398620605, + 3.530607581138611, + 3.618657946586609, + 3.5010584592819214, + 3.40998637676239, + 3.3929054737091064, + 3.568002223968506, + 3.5835269689559937, + 3.515200138092041, + 3.5570945739746094, + 3.550769805908203, + 3.5353591442108154, + 3.6095736026763916, + 3.602874279022217, + 3.6105717420578003, + 3.5213295221328735, + 3.4958536624908447, + 3.637737274169922, + 3.6335325241088867, + 3.5649253129959106, + 3.5429458618164062, + 3.6481817960739136, + 3.5692726373672485, + 3.480798602104187, + 3.5345035791397095, + 3.582723617553711, + 3.4474689960479736, + 3.4440858364105225, + 3.7068746089935303, + 3.5870801210403442, + 3.615915298461914, + 3.657145380973816, + 3.525084137916565, + 3.5036652088165283, + 3.6512595415115356, + 3.4312039613723755, + 3.5651562213897705, + 3.435302972793579, + 3.467521905899048, + 3.457751750946045, + 3.562069535255432, + 3.4124817848205566, + 3.663902163505554, + 3.6108726263046265, + 3.5527758598327637, + 3.4737457036972046, + 3.5442466735839844, + 3.553661584854126, + 3.5129603147506714, + 3.440941333770752, + 3.496729016304016, + 3.5222088098526, + 3.6671721935272217, + 3.648693799972534, + 3.591854453086853, + 3.5245245695114136, + 3.378915309906006, + 3.564280867576599, + 3.4879618883132935, + 3.5113375186920166, + 3.3985536098480225, + 3.5057071447372437, + 3.4730383157730103, + 3.5121309757232666, + 3.5154393911361694, + 3.5284836292266846, + 3.539665102958679, + 3.5430155992507935, + 3.6509146690368652, + 3.6259459257125854, + 3.6058531999588013, + 3.45303213596344, + 3.6181800365448, + 3.571165084838867, + 3.450115203857422, + 3.368178963661194, + 3.3984038829803467, + 3.554864287376404, + 3.4639689922332764, + 3.4423974752426147, + 3.4215972423553467, + 3.5092084407806396, + 3.390926241874695, + 3.435087203979492, + 3.5543102025985718, + 3.582571864128113, + 3.5328131914138794, + 3.520340919494629, + 3.3588743209838867, + 3.441779613494873, + 3.485478401184082, + 3.432421326637268, + 3.4062920808792114, + 3.401870846748352, + 3.3953824043273926, + 3.5536967515945435, + 3.4704819917678833, + 3.4403066635131836, + 3.4618955850601196, + 3.675554394721985, + 3.377699613571167, + 3.458659529685974, + 3.432286500930786, + 3.3768047094345093, + 3.3905032873153687, + 3.366214871406555, + 3.545986533164978, + 3.3896764516830444, + 3.5687906742095947, + 3.4435486793518066, + 3.4257861375808716, + 3.3533729314804077, + 3.5134809017181396, + 3.3683066368103027, + 3.4894367456436157, + 3.3798269033432007, + 3.4408979415893555, + 3.568733811378479, + 3.387907028198242, + 3.403924345970154, + 3.411397933959961, + 3.4116939306259155, + 3.5333386659622192, + 3.2846471071243286, + 3.547568678855896, + 3.352077603340149, + 3.391191244125366, + 3.5046627521514893, + 3.371350049972534, + 3.3631224632263184, + 3.3920825719833374, + 3.3435884714126587, + 3.5188034772872925, + 3.492167115211487, + 3.4194518327713013, + 3.361618757247925, + 3.5157439708709717, + 3.3937478065490723, + 3.346032738685608, + 3.3015259504318237, + 3.5768463611602783, + 3.5398993492126465, + 3.3738410472869873, + 3.484295606613159, + 3.482356309890747, + 3.456892251968384, + 3.428765296936035, + 3.425976872444153, + 3.4862654209136963, + 3.404293656349182, + 3.480660080909729, + 3.4409810304641724, + 3.3990960121154785, + 3.5123379230499268, + 3.5138286352157593, + 3.4333548545837402, + 3.314183235168457, + 3.579620838165283, + 3.4151052236557007, + 3.3843507766723633, + 3.5318167209625244, + 3.388174891471863, + 3.367847204208374, + 3.411698579788208, + 3.3242292404174805, + 3.429729461669922, + 3.423126220703125, + 3.5257362127304077, + 3.4651150703430176, + 3.5007498264312744, + 3.4464712142944336, + 3.500484585762024, + 3.3903716802597046, + 3.3842393159866333, + 3.4880810976028442, + 3.424102306365967, + 3.502313494682312, + 3.4826892614364624, + 3.3592084646224976, + 3.4513274431228638, + 3.3746808767318726, + 3.4931766986846924, + 3.45245361328125, + 3.505487322807312, + 3.5702526569366455, + 3.3277201652526855, + 3.3773072957992554, + 3.26437246799469, + 3.430042862892151, + 3.2792450189590454, + 3.4644259214401245, + 3.4238479137420654, + 3.494865894317627, + 3.438154697418213, + 3.469334840774536, + 3.4599040746688843, + 3.3532888889312744, + 3.4620949029922485, + 3.3622041940689087, + 3.3744826316833496, + 3.5777443647384644, + 3.30825138092041, + 3.3749440908432007, + 3.443471908569336, + 3.3992698192596436, + 3.3617697954177856, + 3.3014070987701416, + 3.434025764465332, + 3.2272831201553345, + 3.290935754776001, + 3.3358906507492065, + 3.424896478652954, + 3.441941022872925, + 3.480691432952881, + 3.3721418380737305, + 3.5448246002197266, + 3.516611933708191, + 3.2686790227890015, + 3.3757126331329346, + 3.3399693965911865, + 3.2082087993621826, + 3.3385581970214844, + 3.365271806716919, + 3.33294141292572, + 3.37730872631073, + 3.3551084995269775, + 3.386712074279785, + 3.383080840110779, + 3.38191556930542, + 3.271131157875061, + 3.320854663848877, + 3.363195776939392, + 3.329080581665039, + 3.526355504989624, + 3.3353134393692017, + 3.366730809211731, + 3.466625213623047, + 3.28440523147583, + 3.2933502197265625, + 3.5286911725997925, + 3.4168137311935425, + 3.443525791168213, + 3.28889799118042, + 3.4607807397842407, + 3.3456605672836304, + 3.347493529319763, + 3.19347882270813, + 3.3523155450820923, + 3.3599435091018677, + 3.3613065481185913, + 3.347580075263977, + 3.2886096239089966, + 3.3708159923553467, + 3.236055374145508, + 3.334215521812439, + 3.13210666179657, + 3.4117062091827393, + 3.3690543174743652, + 3.2195346355438232, + 3.3949564695358276, + 3.323195219039917, + 3.368937373161316, + 3.321356177330017, + 3.284334182739258, + 3.297294497489929, + 3.294533848762512, + 3.323978304862976, + 3.395013928413391, + 3.419131278991699, + 3.3641844987869263, + 3.4438910484313965, + 3.584657311439514, + 3.424986481666565, + 3.267403244972229, + 3.4395158290863037, + 3.3514227867126465, + 3.458735704421997, + 3.331602692604065, + 3.2390129566192627, + 3.359619617462158, + 3.3861799240112305, + 3.391942262649536, + 3.388194441795349, + 3.4276838302612305, + 3.327718496322632, + 3.458178997039795, + 3.373885750770569, + 3.46193790435791, + 3.369583010673523, + 3.332551121711731, + 3.333505392074585, + 3.388451933860779, + 3.434967279434204, + 3.4158084392547607, + 3.3154659271240234, + 3.3702422380447388, + 3.3424675464630127, + 3.488411068916321, + 3.3183950185775757, + 3.2632700204849243, + 3.3876826763153076, + 3.3987066745758057, + 3.416196584701538, + 3.3704264163970947, + 3.279834747314453, + 3.313433051109314, + 3.1908280849456787, + 3.381169319152832, + 3.3727052211761475, + 3.329067587852478, + 3.327434778213501, + 3.399052143096924, + 3.445934772491455, + 3.4478061199188232, + 3.410723090171814, + 3.2644155025482178, + 3.29873526096344, + 3.3145546913146973, + 3.4631924629211426, + 3.318668007850647, + 3.2919065952301025, + 3.3495925664901733, + 3.384203553199768, + 3.4506548643112183, + 3.3430213928222656, + 3.2797552347183228, + 3.3354963064193726, + 3.4302014112472534, + 3.1651540994644165, + 3.1854560375213623, + 3.2548983097076416, + 3.22235369682312, + 3.2352988719940186, + 3.142474055290222, + 3.3397783041000366, + 3.340211868286133, + 3.2482582330703735, + 3.3611546754837036, + 3.4178884029388428, + 3.29935622215271, + 3.3737635612487793, + 3.287811279296875, + 3.3191524744033813, + 3.318901300430298, + 3.375439405441284, + 3.3514868021011353, + 3.287872552871704, + 3.244981050491333, + 3.362825393676758, + 3.361084222793579, + 3.331384778022766, + 3.3821702003479004, + 3.4281954765319824, + 3.4679572582244873, + 3.2839226722717285, + 3.283582329750061, + 3.3049280643463135, + 3.256649971008301, + 3.2781500816345215, + 3.2950271368026733, + 3.3513070344924927, + 3.424069404602051, + 3.2900333404541016, + 3.26210618019104, + 3.3072558641433716, + 3.1920430660247803, + 3.3527413606643677, + 3.3102067708969116, + 3.393989682197571, + 3.306860566139221, + 3.1575119495391846, + 3.3116873502731323, + 3.4171496629714966, + 3.2117313146591187, + 3.262837529182434, + 3.316166400909424, + 3.308104395866394, + 3.2729088068008423, + 3.2461706399917603, + 3.502267003059387, + 3.3446308374404907, + 3.2167409658432007, + 3.113181471824646, + 3.2737029790878296, + 3.240670084953308, + 3.3323771953582764, + 3.4173743724823, + 3.239070415496826, + 3.4516440629959106, + 3.3172630071640015, + 3.4400126934051514, + 3.437349796295166, + 3.344210386276245, + 3.2561622858047485, + 3.214780330657959, + 3.397491931915283, + 3.3889925479888916, + 3.2462133169174194, + 3.217327117919922, + 3.3339086771011353, + 3.3045605421066284, + 3.3554129600524902, + 3.2499661445617676, + 3.1959744691848755, + 3.2576853036880493, + 3.309076428413391, + 3.312196373939514, + 3.4026596546173096, + 3.2660064697265625, + 3.2228145599365234, + 3.239320755004883, + 3.2645946741104126, + 3.093870162963867, + 3.4761546850204468, + 3.3504425287246704, + 3.3590242862701416, + 3.3939430713653564, + 3.314289927482605, + 3.2744739055633545, + 3.1476540565490723, + 3.2074278593063354, + 3.39190936088562, + 3.358878493309021, + 3.296781063079834, + 3.2188456058502197, + 3.4024587869644165, + 3.3925081491470337, + 3.22115421295166, + 3.23686945438385, + 3.3543474674224854, + 3.347301483154297, + 3.3807427883148193, + 3.3010473251342773, + 3.5377105474472046, + 3.3763188123703003, + 3.2535719871520996, + 3.2743924856185913, + 3.3690532445907593, + 3.3065967559814453, + 3.335619807243347, + 3.347419023513794, + 3.2372573614120483, + 3.2921640872955322, + 3.397454023361206, + 3.2188831567764282, + 3.326140284538269, + 3.262446165084839, + 3.3203357458114624, + 3.3161444664001465, + 3.2155033349990845, + 3.3979018926620483, + 3.2416387796401978, + 3.184965133666992, + 3.3910118341445923, + 3.285348892211914, + 3.4052062034606934, + 3.4486371278762817, + 3.257728338241577, + 3.22599458694458, + 3.226393222808838, + 3.3769962787628174, + 3.277567744255066, + 3.335546374320984, + 3.416748046875, + 3.23548686504364, + 3.1625900268554688, + 3.330114006996155, + 3.354585289955139, + 3.2604933977127075, + 3.3887003660202026, + 3.3666146993637085, + 3.3273019790649414, + 3.3371654748916626, + 3.249754548072815, + 3.2814890146255493, + 3.0779353380203247, + 3.4359614849090576, + 3.3554251194000244, + 3.2487353086471558, + 3.228274941444397, + 3.0748265981674194, + 3.4091686010360718, + 3.342919111251831, + 3.2558385133743286, + 3.2544286251068115, + 3.3665157556533813, + 3.286022901535034, + 3.3242603540420532, + 3.3667043447494507 + ], + "val_steps": [ + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 70.31695797623706, + 57.06390368703425, + 50.475701619580725, + 46.7109180309009, + 43.73186800939149, + 41.32193672664172, + 39.03696140077867, + 37.133796360646095, + 36.187330835686865, + 35.095674584509815, + 34.23256804410032, + 33.240589725763925, + 32.3517027110649, + 32.05377134538009, + 31.449236154872224, + 31.265006804477174, + 31.048027256923206, + 30.947108749875678, + 30.94688931259243 + ], + "best_ppl": 30.94688931259243, + "train_time_s": 21219.59129834175 +} \ No newline at end of file diff --git a/063_sparse_dispatch/shared_deltanet_1plus2of5_s137_best.pt b/063_sparse_dispatch/shared_deltanet_1plus2of5_s137_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..60e83acd24718fa9b9c969261a96185a57b9e796 --- /dev/null +++ b/063_sparse_dispatch/shared_deltanet_1plus2of5_s137_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72cad7c1d0ce6420e7145e753df0982c85e60eb5b40adb4ae860dd5a59d68ae6 +size 350411813 diff --git a/063_sparse_dispatch/shared_deltanet_1plus2of5_s137_results.json b/063_sparse_dispatch/shared_deltanet_1plus2of5_s137_results.json new file mode 100644 index 0000000000000000000000000000000000000000..dd3e21d6973173b084ea018622a4fed89774d96a --- /dev/null +++ b/063_sparse_dispatch/shared_deltanet_1plus2of5_s137_results.json @@ -0,0 +1,2055 @@ +{ + "label": "shared_deltanet_1plus2of5_s137", + "seed": 137, + "total_params": 87523584, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 18.941777229309082, + 15.764310836791992, + 8.104741096496582, + 7.185344457626343, + 7.064902305603027, + 7.002978324890137, + 6.88677191734314, + 6.772162675857544, + 6.668334245681763, + 6.656241178512573, + 6.432089567184448, + 6.3513946533203125, + 6.272685766220093, + 6.121657371520996, + 6.085039138793945, + 6.073081970214844, + 5.966714382171631, + 5.826366424560547, + 5.812524318695068, + 5.739185094833374, + 5.626312732696533, + 5.528106927871704, + 5.5974695682525635, + 5.531108140945435, + 5.486811399459839, + 5.263707160949707, + 5.441848516464233, + 5.32504677772522, + 5.259361982345581, + 5.333548069000244, + 5.24284553527832, + 5.274545192718506, + 5.160192251205444, + 4.966359376907349, + 5.153849124908447, + 5.073065280914307, + 5.074450969696045, + 5.046684503555298, + 5.177412509918213, + 4.919404029846191, + 4.8956427574157715, + 4.930259704589844, + 5.023513317108154, + 4.976104497909546, + 4.8771443367004395, + 4.908196210861206, + 4.889198303222656, + 4.843958139419556, + 4.876513481140137, + 4.699960231781006, + 4.7568957805633545, + 4.664006233215332, + 4.782400131225586, + 4.786868333816528, + 4.717353105545044, + 4.774379730224609, + 4.855673789978027, + 4.777788400650024, + 4.642184495925903, + 4.647238254547119, + 4.694804430007935, + 4.751414060592651, + 4.490250587463379, + 4.779927968978882, + 4.6449408531188965, + 4.599801540374756, + 4.4571921825408936, + 4.479560852050781, + 4.64618706703186, + 4.529693841934204, + 4.570716857910156, + 4.525323152542114, + 4.574446201324463, + 4.465540409088135, + 4.557370662689209, + 4.629255294799805, + 4.604754209518433, + 4.601460218429565, + 4.426803112030029, + 4.574995040893555, + 4.521845102310181, + 4.343456029891968, + 4.4940385818481445, + 4.45644736289978, + 4.51319146156311, + 4.450756788253784, + 4.5390448570251465, + 4.438964605331421, + 4.570893287658691, + 4.480825424194336, + 4.523808002471924, + 4.491350412368774, + 4.406061887741089, + 4.407317876815796, + 4.4820637702941895, + 4.2124550342559814, + 4.42357611656189, + 4.33710503578186, + 4.378388404846191, + 4.3313233852386475, + 4.537184000015259, + 4.284789085388184, + 4.399476528167725, + 4.399019956588745, + 4.455707550048828, + 4.154561519622803, + 4.436404705047607, + 4.286954402923584, + 4.266961097717285, + 4.203166723251343, + 4.2720046043396, + 4.191550016403198, + 4.3600194454193115, + 4.352091550827026, + 4.233492612838745, + 4.300542593002319, + 4.304054498672485, + 4.3066322803497314, + 4.124436855316162, + 4.310654163360596, + 4.289762735366821, + 4.373227596282959, + 4.241454124450684, + 4.1954967975616455, + 4.350762367248535, + 4.164233922958374, + 4.1572136878967285, + 4.199434518814087, + 4.245365858078003, + 4.179540395736694, + 4.155853986740112, + 4.018292427062988, + 4.2686927318573, + 4.083863258361816, + 4.050938010215759, + 4.201676845550537, + 4.193606853485107, + 4.142446994781494, + 4.1521172523498535, + 4.216662883758545, + 4.1779420375823975, + 4.265936613082886, + 4.163487195968628, + 4.039025068283081, + 4.142197847366333, + 4.097287178039551, + 4.1637046337127686, + 4.187143325805664, + 4.113623857498169, + 4.203746557235718, + 4.160208225250244, + 4.08643364906311, + 4.149158835411072, + 4.159756660461426, + 4.0776073932647705, + 4.262147426605225, + 4.220786809921265, + 4.142898321151733, + 4.108215570449829, + 4.0752129554748535, + 4.155359983444214, + 4.194197654724121, + 4.1352880001068115, + 4.006859302520752, + 4.026576399803162, + 4.090734958648682, + 4.090590000152588, + 4.315392732620239, + 4.152803063392639, + 4.013688921928406, + 4.146138429641724, + 4.099579811096191, + 4.0496827363967896, + 4.130584239959717, + 4.027363300323486, + 4.115361928939819, + 4.124344825744629, + 4.123448848724365, + 4.12929105758667, + 4.073091506958008, + 3.9361687898635864, + 4.074041128158569, + 3.9834206104278564, + 4.10864782333374, + 4.164301872253418, + 4.126657485961914, + 3.9957919120788574, + 4.073460578918457, + 4.068069696426392, + 3.9938992261886597, + 3.9913936853408813, + 3.8863285779953003, + 4.074028968811035, + 3.848401665687561, + 4.054916620254517, + 4.150609493255615, + 4.049559235572815, + 3.9286351203918457, + 4.159280300140381, + 3.9927176237106323, + 4.010643124580383, + 4.106925010681152, + 3.9514960050582886, + 4.010464549064636, + 4.008621573448181, + 3.988955855369568, + 3.962343454360962, + 4.002312898635864, + 3.984949231147766, + 3.9826085567474365, + 3.9585026502609253, + 3.9992334842681885, + 3.9570411443710327, + 3.98173189163208, + 4.034876823425293, + 3.9305055141448975, + 4.096616983413696, + 3.9346885681152344, + 3.9072474241256714, + 3.9900444746017456, + 3.9765706062316895, + 3.927355408668518, + 3.930047392845154, + 4.0856711864471436, + 4.063689112663269, + 4.089742422103882, + 3.919206142425537, + 3.9186164140701294, + 4.056308269500732, + 3.9501049518585205, + 3.99215567111969, + 4.059466600418091, + 4.005337595939636, + 3.9741369485855103, + 3.9492238759994507, + 3.926459550857544, + 3.9375858306884766, + 3.9553146362304688, + 3.855429172515869, + 3.90035080909729, + 3.8984153270721436, + 3.903868556022644, + 3.9961451292037964, + 3.8598573207855225, + 3.8207908868789673, + 3.8538904190063477, + 3.9901304244995117, + 3.8769304752349854, + 3.8905404806137085, + 3.7835031747817993, + 3.981021285057068, + 4.047118186950684, + 3.790432333946228, + 3.9400405883789062, + 3.6919749975204468, + 3.864953637123108, + 3.924049735069275, + 3.735214948654175, + 3.7926018238067627, + 3.842797636985779, + 3.941739320755005, + 3.854304552078247, + 3.8060868978500366, + 3.765012264251709, + 3.879847764968872, + 3.885768175125122, + 3.935972809791565, + 3.8581290245056152, + 3.832010865211487, + 3.8717503547668457, + 3.9826619625091553, + 3.804728388786316, + 3.795129179954529, + 3.8058550357818604, + 3.988190174102783, + 3.873183250427246, + 3.8781063556671143, + 3.6298792362213135, + 3.690362811088562, + 3.860408067703247, + 3.883488178253174, + 3.8667534589767456, + 3.8830153942108154, + 3.835168480873108, + 3.9347838163375854, + 3.8569283485412598, + 3.971316933631897, + 3.8672066926956177, + 3.8731768131256104, + 3.9208626747131348, + 3.775568127632141, + 3.8604443073272705, + 3.729188084602356, + 3.929098129272461, + 3.816988468170166, + 3.866003155708313, + 3.8405678272247314, + 3.8026586771011353, + 3.8656646013259888, + 3.7855454683303833, + 3.9160006046295166, + 3.85211980342865, + 3.8168903589248657, + 3.7963905334472656, + 3.736222743988037, + 3.9004236459732056, + 3.907849431037903, + 3.931775212287903, + 3.8409518003463745, + 3.8540971279144287, + 3.820549249649048, + 3.864237070083618, + 3.841920256614685, + 3.7287243604660034, + 3.773058295249939, + 3.7155712842941284, + 3.732282280921936, + 3.656073212623596, + 3.814535140991211, + 3.742068886756897, + 3.8485149145126343, + 3.716635227203369, + 3.7668713331222534, + 3.7947603464126587, + 3.6751911640167236, + 3.7932217121124268, + 3.802618384361267, + 3.883987784385681, + 3.835110664367676, + 3.807641625404358, + 3.7312315702438354, + 3.7463613748550415, + 3.922690272331238, + 3.781259536743164, + 3.7941813468933105, + 3.837184190750122, + 3.9119802713394165, + 3.841110110282898, + 3.7971309423446655, + 3.8222692012786865, + 3.769304037094116, + 3.923048734664917, + 3.7771941423416138, + 3.7341365814208984, + 3.7693363428115845, + 3.770275115966797, + 3.8488203287124634, + 3.745301127433777, + 3.619191288948059, + 3.8029271364212036, + 3.8589359521865845, + 3.733610153198242, + 3.755769371986389, + 3.817331552505493, + 3.8719420433044434, + 3.698808193206787, + 3.792050361633301, + 3.693178415298462, + 3.871019124984741, + 3.8173367977142334, + 3.6985838413238525, + 3.7788314819335938, + 3.755073666572571, + 3.731559157371521, + 3.7649723291397095, + 3.863584876060486, + 3.79696786403656, + 3.737496018409729, + 3.7651110887527466, + 3.650453805923462, + 3.7952593564987183, + 3.8267537355422974, + 3.697815775871277, + 3.72512686252594, + 3.8581230640411377, + 3.7648186683654785, + 3.838330030441284, + 3.7423906326293945, + 3.7193254232406616, + 3.6801884174346924, + 3.7287607192993164, + 3.645251750946045, + 3.588897466659546, + 3.6884149312973022, + 3.58026659488678, + 3.6646082401275635, + 3.745276927947998, + 3.623902678489685, + 3.695500135421753, + 3.5956220626831055, + 3.715994119644165, + 3.6208564043045044, + 3.6043004989624023, + 3.770340323448181, + 3.5749000310897827, + 3.565005660057068, + 3.541889786720276, + 3.713123917579651, + 3.7126060724258423, + 3.681518793106079, + 3.693743586540222, + 3.7739368677139282, + 3.64349901676178, + 3.706398844718933, + 3.7548718452453613, + 3.5176409482955933, + 3.6810202598571777, + 3.6329963207244873, + 3.614756464958191, + 3.4455466270446777, + 3.642755627632141, + 3.6634650230407715, + 3.7249934673309326, + 3.8011515140533447, + 3.642464518547058, + 3.7990797758102417, + 3.843010902404785, + 3.7517077922821045, + 3.6941800117492676, + 3.7457605600357056, + 3.6521953344345093, + 3.6054247617721558, + 3.6694644689559937, + 3.6549121141433716, + 3.6620324850082397, + 3.5802568197250366, + 3.715681552886963, + 3.689553141593933, + 3.57406485080719, + 3.585954427719116, + 3.696063995361328, + 3.6341620683670044, + 3.7039897441864014, + 3.7317713499069214, + 3.6583510637283325, + 3.63007390499115, + 3.6751760244369507, + 3.6769165992736816, + 3.627637028694153, + 3.726460814476013, + 3.713007092475891, + 3.690404772758484, + 3.638501524925232, + 3.73853075504303, + 3.6955068111419678, + 3.632373332977295, + 3.6084141731262207, + 3.6204482316970825, + 3.7868117094039917, + 3.613319993019104, + 3.7311099767684937, + 3.6988911628723145, + 3.7007206678390503, + 3.738382339477539, + 3.46859073638916, + 3.6389100551605225, + 3.7784820795059204, + 3.6253933906555176, + 3.7300525903701782, + 3.7700427770614624, + 3.817785382270813, + 3.517158031463623, + 3.7184947729110718, + 3.641974925994873, + 3.626129150390625, + 3.6364861726760864, + 3.7290016412734985, + 3.553427577018738, + 3.6050363779067993, + 3.681622862815857, + 3.635372519493103, + 3.617640256881714, + 3.4703136682510376, + 3.599144458770752, + 3.5701597929000854, + 3.5109370946884155, + 3.7061740159988403, + 3.6671884059906006, + 3.67902410030365, + 3.539308190345764, + 3.621895670890808, + 3.7281371355056763, + 3.572255253791809, + 3.4853116273880005, + 3.7085026502609253, + 3.6000711917877197, + 3.6456276178359985, + 3.6432093381881714, + 3.773697853088379, + 3.60128116607666, + 3.657463788986206, + 3.7003941535949707, + 3.611391067504883, + 3.5708881616592407, + 3.6601184606552124, + 3.6439547538757324, + 3.571390986442566, + 3.841564893722534, + 3.7361819744110107, + 3.7021600008010864, + 3.5801873207092285, + 3.694510340690613, + 3.5719363689422607, + 3.778481125831604, + 3.5136029720306396, + 3.504683256149292, + 3.505714178085327, + 3.4318536520004272, + 3.4978502988815308, + 3.6815720796585083, + 3.6070152521133423, + 3.5169724225997925, + 3.5893982648849487, + 3.481666326522827, + 3.5125685930252075, + 3.583233952522278, + 3.5120266675949097, + 3.480935573577881, + 3.647367477416992, + 3.4096546173095703, + 3.5200737714767456, + 3.441987991333008, + 3.55959689617157, + 3.5438294410705566, + 3.4944756031036377, + 3.495790719985962, + 3.5334564447402954, + 3.6642143726348877, + 3.6328707933425903, + 3.660754919052124, + 3.5074002742767334, + 3.389809012413025, + 3.4775121212005615, + 3.5799448490142822, + 3.524537682533264, + 3.571561574935913, + 3.489955425262451, + 3.509091854095459, + 3.5482025146484375, + 3.5234521627426147, + 3.5825700759887695, + 3.5221171379089355, + 3.454630970954895, + 3.5728565454483032, + 3.5747616291046143, + 3.6651718616485596, + 3.521888017654419, + 3.545959234237671, + 3.530877470970154, + 3.53591251373291, + 3.3667988777160645, + 3.5320966243743896, + 3.4719971418380737, + 3.4537785053253174, + 3.5227487087249756, + 3.4952672719955444, + 3.5966320037841797, + 3.5760955810546875, + 3.671232581138611, + 3.5634288787841797, + 3.455249786376953, + 3.456734538078308, + 3.5999960899353027, + 3.541906476020813, + 3.504241466522217, + 3.5445122718811035, + 3.564682364463806, + 3.442945718765259, + 3.4828391075134277, + 3.5545248985290527, + 3.540656089782715, + 3.440898060798645, + 3.5621076822280884, + 3.344092845916748, + 3.640984058380127, + 3.597145676612854, + 3.6843783855438232, + 3.555282473564148, + 3.4633349180221558, + 3.625698685646057, + 3.5440754890441895, + 3.4229050874710083, + 3.604792356491089, + 3.5982072353363037, + 3.417584180831909, + 3.597968101501465, + 3.5262820720672607, + 3.577353358268738, + 3.550953984260559, + 3.7305409908294678, + 3.5166702270507812, + 3.522059440612793, + 3.4982863664627075, + 3.533779263496399, + 3.5284582376480103, + 3.5712236166000366, + 3.520127773284912, + 3.530381202697754, + 3.5217779874801636, + 3.495668053627014, + 3.482102632522583, + 3.55375874042511, + 3.6334619522094727, + 3.5245317220687866, + 3.4654040336608887, + 3.4387303590774536, + 3.4820151329040527, + 3.6074811220169067, + 3.6179646253585815, + 3.523916482925415, + 3.5098772048950195, + 3.4890575408935547, + 3.4362844228744507, + 3.3822407722473145, + 3.462313175201416, + 3.420566201210022, + 3.3691381216049194, + 3.5787436962127686, + 3.522229790687561, + 3.4832149744033813, + 3.4278913736343384, + 3.5062990188598633, + 3.5384713411331177, + 3.5236912965774536, + 3.553407907485962, + 3.519641160964966, + 3.4790170192718506, + 3.5662848949432373, + 3.5802690982818604, + 3.456521153450012, + 3.435853123664856, + 3.565173625946045, + 3.36722993850708, + 3.4041991233825684, + 3.421254515647888, + 3.4877630472183228, + 3.5263770818710327, + 3.456773281097412, + 3.366018056869507, + 3.4940932989120483, + 3.537660241127014, + 3.427043318748474, + 3.337604284286499, + 3.523084878921509, + 3.4357450008392334, + 3.3010305166244507, + 3.4706637859344482, + 3.4061765670776367, + 3.274258613586426, + 3.4833873510360718, + 3.444193720817566, + 3.374045491218567, + 3.3950828313827515, + 3.4072587490081787, + 3.581055521965027, + 3.4299733638763428, + 3.543277382850647, + 3.3706167936325073, + 3.480367422103882, + 3.462949275970459, + 3.438576102256775, + 3.537240505218506, + 3.428028106689453, + 3.3907957077026367, + 3.534442901611328, + 3.4219508171081543, + 3.5617363452911377, + 3.40406596660614, + 3.5079283714294434, + 3.34023380279541, + 3.41963529586792, + 3.427605986595154, + 3.3302966356277466, + 3.540364980697632, + 3.4974383115768433, + 3.440663695335388, + 3.4239237308502197, + 3.471192240715027, + 3.36127507686615, + 3.4375598430633545, + 3.3436814546585083, + 3.56763231754303, + 3.328776240348816, + 3.31784725189209, + 3.4471813440322876, + 3.3191261291503906, + 3.3907898664474487, + 3.507511258125305, + 3.453782558441162, + 3.4378916025161743, + 3.4047359228134155, + 3.3383142948150635, + 3.40701162815094, + 3.483872175216675, + 3.416894316673279, + 3.543187737464905, + 3.304083466529846, + 3.367531657218933, + 3.2994890213012695, + 3.585242986679077, + 3.4965033531188965, + 3.4392547607421875, + 3.503606915473938, + 3.4292033910751343, + 3.3990020751953125, + 3.458396077156067, + 3.4337127208709717, + 3.5608692169189453, + 3.4056371450424194, + 3.3996829986572266, + 3.3513858318328857, + 3.5007433891296387, + 3.481908082962036, + 3.4734503030776978, + 3.375829815864563, + 3.494436025619507, + 3.335653305053711, + 3.378273129463196, + 3.415248394012451, + 3.3355754613876343, + 3.4688236713409424, + 3.4272481203079224, + 3.515550971031189, + 3.4150811433792114, + 3.4083688259124756, + 3.470395088195801, + 3.4794660806655884, + 3.367236375808716, + 3.486580967903137, + 3.550161838531494, + 3.4042128324508667, + 3.568140745162964, + 3.4374635219573975, + 3.5405311584472656, + 3.4914772510528564, + 3.464326858520508, + 3.358823776245117, + 3.3805904388427734, + 3.363668441772461, + 3.4097766876220703, + 3.4352331161499023, + 3.289743423461914, + 3.41496741771698, + 3.365707516670227, + 3.5134177207946777, + 3.546859860420227, + 3.4438040256500244, + 3.353269100189209, + 3.446694493293762, + 3.492984414100647, + 3.3685452938079834, + 3.422739267349243, + 3.3783568143844604, + 3.323965549468994, + 3.5166221857070923, + 3.3662798404693604, + 3.3861539363861084, + 3.434678316116333, + 3.5349091291427612, + 3.31563138961792, + 3.256402373313904, + 3.388108491897583, + 3.3223096132278442, + 3.4660420417785645, + 3.2788753509521484, + 3.380955934524536, + 3.364289402961731, + 3.514632225036621, + 3.2345389127731323, + 3.2587552070617676, + 3.2795798778533936, + 3.4693721532821655, + 3.312353253364563, + 3.386391282081604, + 3.39284348487854, + 3.3206199407577515, + 3.3244067430496216, + 3.4839296340942383, + 3.372278928756714, + 3.356748104095459, + 3.427808165550232, + 3.180917263031006, + 3.416467308998108, + 3.366558313369751, + 3.3340747356414795, + 3.289580821990967, + 3.4128212928771973, + 3.347174048423767, + 3.393781542778015, + 3.4004679918289185, + 3.3281426429748535, + 3.2987507581710815, + 3.45681095123291, + 3.433465361595154, + 3.2927422523498535, + 3.362031936645508, + 3.473660111427307, + 3.308297872543335, + 3.285969853401184, + 3.2961925268173218, + 3.319298028945923, + 3.4455676078796387, + 3.3365224599838257, + 3.280281186103821, + 3.3360953330993652, + 3.305974841117859, + 3.396218180656433, + 3.3146283626556396, + 3.428819179534912, + 3.3682435750961304, + 3.248910069465637, + 3.3279337882995605, + 3.295907497406006, + 3.4309860467910767, + 3.4282891750335693, + 3.3371541500091553, + 3.4099035263061523, + 3.316831588745117, + 3.4520071744918823, + 3.2381999492645264, + 3.3530147075653076, + 3.362143635749817, + 3.4037941694259644, + 3.342747688293457, + 3.404160976409912, + 3.3827855587005615, + 3.3117899894714355, + 3.3509637117385864, + 3.2777081727981567, + 3.3865132331848145, + 3.3372750282287598, + 3.4384701251983643, + 3.3529006242752075, + 3.31921923160553, + 3.3707529306411743, + 3.2550268173217773, + 3.409255266189575, + 3.3392146825790405, + 3.372859001159668, + 3.322749137878418, + 3.356405735015869, + 3.322946071624756, + 3.2103254795074463, + 3.351922631263733, + 3.2616976499557495, + 3.4698283672332764, + 3.332207679748535, + 3.3387635946273804, + 3.3280956745147705, + 3.5047500133514404, + 3.264212965965271, + 3.311872124671936, + 3.3923144340515137, + 3.5028616189956665, + 3.4187567234039307, + 3.4698537588119507, + 3.3155698776245117, + 3.316903829574585, + 3.275493025779724, + 3.3421214818954468, + 3.2476059198379517, + 3.416615843772888, + 3.2419769763946533, + 3.399142265319824, + 3.428534150123596, + 3.411733865737915, + 3.38832426071167, + 3.3983709812164307, + 3.45995557308197, + 3.2631994485855103, + 3.377035975456238, + 3.2792364358901978, + 3.299603819847107, + 3.316763997077942, + 3.2556906938552856, + 3.197312355041504, + 3.2515337467193604, + 3.3044697046279907, + 3.4669549465179443, + 3.409738779067993, + 3.472737669944763, + 3.3439409732818604, + 3.4197092056274414, + 3.3220213651657104, + 3.4176902770996094, + 3.3131542205810547, + 3.3411139249801636, + 3.22750461101532, + 3.3327115774154663, + 3.281155824661255, + 3.4336416721343994, + 3.2993956804275513, + 3.3751142024993896, + 3.3036195039749146, + 3.3490543365478516, + 3.3213937282562256, + 3.2186306715011597, + 3.314642548561096, + 3.350237488746643, + 3.430603265762329, + 3.264797568321228, + 3.2327760457992554, + 3.2256795167922974, + 3.3499293327331543, + 3.2991461753845215, + 3.2908730506896973, + 3.2199597358703613, + 3.188347816467285, + 3.3834224939346313, + 3.1998270750045776, + 3.126781463623047, + 3.3422434329986572, + 3.2425094842910767, + 3.324859857559204, + 3.2855266332626343, + 3.5769013166427612, + 3.3616702556610107, + 3.3486298322677612, + 3.2922117710113525, + 3.459184765815735, + 3.292421579360962, + 3.4165045022964478, + 3.2885875701904297, + 3.251897096633911, + 3.3376331329345703, + 3.1638903617858887, + 3.366765022277832, + 3.2170779705047607, + 3.3042186498641968, + 3.319074034690857, + 3.3803653717041016, + 3.2791014909744263, + 3.3025037050247192, + 3.3225975036621094, + 3.166251540184021, + 3.2683591842651367, + 3.3787589073181152, + 3.2950897216796875, + 3.404654860496521, + 3.2990927696228027, + 3.303312063217163, + 3.3412758111953735, + 3.328326940536499, + 3.2636507749557495, + 3.3406636714935303, + 3.2920186519622803, + 3.2345722913742065, + 3.4390478134155273, + 3.342590570449829, + 3.3243695497512817, + 3.320036768913269, + 3.320066213607788, + 3.416162371635437, + 3.361230731010437, + 3.4041075706481934, + 3.412437915802002, + 3.2675594091415405, + 3.485742211341858, + 3.2665393352508545, + 3.204527974128723, + 3.4117308855056763, + 3.3799264430999756, + 3.299821138381958, + 3.324848175048828, + 3.2588080167770386, + 3.316453218460083, + 3.370276689529419, + 3.222460985183716, + 3.349473476409912, + 3.2880765199661255, + 3.2365931272506714, + 3.369808316230774, + 3.318920612335205, + 3.346177577972412, + 3.321535110473633, + 3.2468632459640503, + 3.320154547691345, + 3.3982330560684204, + 3.3593896627426147, + 3.406062960624695, + 3.3274871110916138, + 3.3783844709396362, + 3.263440489768982, + 3.283447265625, + 3.1778388023376465, + 3.2463310956954956, + 3.2498955726623535, + 3.324121117591858, + 3.292367696762085, + 3.349982738494873, + 3.426772952079773, + 3.3970117568969727, + 3.3347365856170654, + 3.2171881198883057, + 3.396557331085205, + 3.31273877620697, + 3.305031418800354, + 3.371759533882141, + 3.2297568321228027, + 3.470897912979126 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 115.34181938173244, + 72.45622318299064, + 58.04061160456038, + 51.232422334646216, + 46.84223834324035, + 43.798607871457776, + 41.5774957914099, + 39.82668107093484, + 38.38196974767637, + 36.80345024289662, + 36.077575835363035, + 34.48301312462311, + 34.068067182067566, + 33.24169721392194, + 32.369329709697794, + 32.157865630352674, + 31.774112255128998, + 31.660164078295335, + 31.58055270020379, + 31.56585874944045 + ], + "best_ppl": 31.56585874944045, + "train_time_s": 21209.277144908905 +} \ No newline at end of file diff --git a/063_sparse_dispatch/shared_deltanet_1plus2of5_s256_best.pt b/063_sparse_dispatch/shared_deltanet_1plus2of5_s256_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b87e2d64edb96597a0c334d65912492e3fd1456 --- /dev/null +++ b/063_sparse_dispatch/shared_deltanet_1plus2of5_s256_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:662850d651334cb9fae041cb6c527abda6273deac878653905677cc92e6de681 +size 350411813 diff --git a/063_sparse_dispatch/shared_deltanet_1plus2of5_s256_results.json b/063_sparse_dispatch/shared_deltanet_1plus2of5_s256_results.json new file mode 100644 index 0000000000000000000000000000000000000000..d555ba9a6c290f17157add4be62b62413c2da6a3 --- /dev/null +++ b/063_sparse_dispatch/shared_deltanet_1plus2of5_s256_results.json @@ -0,0 +1,2055 @@ +{ + "label": "shared_deltanet_1plus2of5_s256", + "seed": 256, + "total_params": 87523584, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 18.965210914611816, + 15.71993637084961, + 8.012000322341919, + 7.275061130523682, + 7.142263174057007, + 6.955883026123047, + 6.864110708236694, + 6.773501873016357, + 6.591285705566406, + 6.439815282821655, + 6.3745574951171875, + 6.364466190338135, + 6.247251987457275, + 6.156131267547607, + 6.124340295791626, + 6.0180864334106445, + 5.930310964584351, + 5.914761543273926, + 5.819058418273926, + 5.753215074539185, + 5.606886863708496, + 5.6766533851623535, + 5.6303465366363525, + 5.662128210067749, + 5.558495283126831, + 5.513863563537598, + 5.4730634689331055, + 5.369608163833618, + 5.318436622619629, + 5.203479528427124, + 5.32818341255188, + 5.239964962005615, + 5.253146648406982, + 5.183717966079712, + 5.215109825134277, + 5.172573804855347, + 5.10582160949707, + 5.035707712173462, + 5.047875165939331, + 5.101406574249268, + 4.7686216831207275, + 4.887853384017944, + 5.044729471206665, + 5.048241138458252, + 5.037051439285278, + 4.949913024902344, + 4.921738624572754, + 4.750720739364624, + 4.827047824859619, + 4.846384763717651, + 4.749671220779419, + 4.883506536483765, + 4.689078092575073, + 4.63569712638855, + 4.798196792602539, + 4.676234006881714, + 4.730494260787964, + 4.551461219787598, + 4.824712038040161, + 4.687587738037109, + 4.6070380210876465, + 4.750194311141968, + 4.777299880981445, + 4.656249046325684, + 4.524984359741211, + 4.6842005252838135, + 4.7608137130737305, + 4.5824830532073975, + 4.5837483406066895, + 4.58422589302063, + 4.810152769088745, + 4.563259601593018, + 4.552090644836426, + 4.497562408447266, + 4.4492645263671875, + 4.54940128326416, + 4.630421161651611, + 4.429238557815552, + 4.547192335128784, + 4.519082546234131, + 4.570890426635742, + 4.60435938835144, + 4.52253532409668, + 4.393586158752441, + 4.498779773712158, + 4.449474811553955, + 4.467597723007202, + 4.500683784484863, + 4.455380916595459, + 4.571713209152222, + 4.428051710128784, + 4.4713757038116455, + 4.418875455856323, + 4.258285999298096, + 4.474283456802368, + 4.3622496128082275, + 4.413394927978516, + 4.410644054412842, + 4.470144033432007, + 4.340912342071533, + 4.251879692077637, + 4.378785610198975, + 4.304723262786865, + 4.302437782287598, + 4.29817795753479, + 4.348767280578613, + 4.334252595901489, + 4.447209358215332, + 4.3924102783203125, + 4.407806873321533, + 4.239576101303101, + 4.2009665966033936, + 4.396907567977905, + 4.2835774421691895, + 4.301691055297852, + 4.103354454040527, + 4.236565589904785, + 4.358220338821411, + 4.371892690658569, + 4.222761392593384, + 4.396989583969116, + 4.17390775680542, + 4.198961019515991, + 4.29058313369751, + 4.146276473999023, + 4.126185894012451, + 4.193213939666748, + 4.251794338226318, + 4.170158624649048, + 4.147890329360962, + 4.141598463058472, + 4.149410963058472, + 4.113623380661011, + 4.174185514450073, + 4.250976800918579, + 4.108235120773315, + 4.258925914764404, + 4.190160512924194, + 3.986176371574402, + 4.191905736923218, + 4.174280643463135, + 4.0094640254974365, + 4.14691686630249, + 4.1023125648498535, + 4.011502981185913, + 4.075600862503052, + 4.173590660095215, + 4.173199653625488, + 4.023149132728577, + 4.08834171295166, + 4.192897319793701, + 4.049361944198608, + 4.089154481887817, + 4.004066348075867, + 4.2009596824646, + 3.9799675941467285, + 4.20340633392334, + 4.0800392627716064, + 4.063769102096558, + 4.116014719009399, + 4.181584358215332, + 4.06409478187561, + 4.172655344009399, + 4.140131235122681, + 4.112139701843262, + 4.046640396118164, + 4.174766302108765, + 4.183933734893799, + 4.099868059158325, + 4.123291730880737, + 4.058974027633667, + 4.0738794803619385, + 3.992179751396179, + 4.018064022064209, + 4.121079444885254, + 4.1089091300964355, + 4.063783645629883, + 3.995997905731201, + 4.032284379005432, + 4.090849161148071, + 4.028010010719299, + 4.059251546859741, + 4.067955255508423, + 4.02386474609375, + 4.038537263870239, + 3.96444571018219, + 3.9170925617218018, + 3.93405020236969, + 4.095373153686523, + 3.9102511405944824, + 4.081977486610413, + 4.010729789733887, + 4.138266205787659, + 3.8983293771743774, + 4.087832689285278, + 3.992253065109253, + 4.021162509918213, + 4.029366970062256, + 4.050446629524231, + 3.8460110425949097, + 3.8986281156539917, + 4.002238631248474, + 3.973874807357788, + 4.0473785400390625, + 3.8088510036468506, + 3.968609094619751, + 3.960578203201294, + 3.9336278438568115, + 4.131093263626099, + 3.936661720275879, + 4.055932998657227, + 3.861816644668579, + 3.81759250164032, + 3.9637961387634277, + 4.004762768745422, + 3.9977266788482666, + 3.9824042320251465, + 3.882596969604492, + 4.068375110626221, + 3.936941981315613, + 4.070604562759399, + 3.9397655725479126, + 4.039819002151489, + 3.961780309677124, + 4.037268161773682, + 3.9553900957107544, + 3.8689769506454468, + 3.986534357070923, + 3.944250226020813, + 3.83663272857666, + 4.0719075202941895, + 4.030276775360107, + 3.798871874809265, + 3.910658836364746, + 3.9457772970199585, + 3.9085755348205566, + 3.9418766498565674, + 3.875907301902771, + 3.9577393531799316, + 3.932375431060791, + 3.9568417072296143, + 3.9845465421676636, + 3.918934941291809, + 3.951038360595703, + 3.731153726577759, + 4.0626620054244995, + 4.022223591804504, + 3.9002156257629395, + 3.925997257232666, + 4.0197601318359375, + 3.940005302429199, + 3.892707943916321, + 3.9068806171417236, + 3.879076838493347, + 3.7739076614379883, + 3.8793641328811646, + 3.8103965520858765, + 3.7774500846862793, + 3.8451370000839233, + 3.780333638191223, + 3.888002872467041, + 3.91287362575531, + 3.892205238342285, + 3.8142881393432617, + 3.7728954553604126, + 3.861358404159546, + 3.9049803018569946, + 3.9313682317733765, + 3.965062737464905, + 3.7954320907592773, + 3.9080251455307007, + 3.7790993452072144, + 3.892827868461609, + 3.7943649291992188, + 3.614264726638794, + 3.892747163772583, + 3.9447531700134277, + 3.8738579750061035, + 3.767918825149536, + 3.8191291093826294, + 3.7990829944610596, + 3.866418719291687, + 3.8887581825256348, + 3.824919104576111, + 3.8340829610824585, + 3.8686314821243286, + 3.7675377130508423, + 3.913134813308716, + 3.7772244215011597, + 3.8051422834396362, + 3.75715970993042, + 3.901555061340332, + 3.790483236312866, + 3.7934452295303345, + 3.776843786239624, + 3.799310564994812, + 3.9195057153701782, + 3.7843832969665527, + 3.7432092428207397, + 3.7310595512390137, + 3.8461179733276367, + 3.827359914779663, + 3.7841488122940063, + 3.8612985610961914, + 3.8634424209594727, + 3.735694169998169, + 3.857078790664673, + 3.9374369382858276, + 3.7180755138397217, + 3.744048237800598, + 3.8414069414138794, + 3.8314337730407715, + 3.7502752542495728, + 3.7407965660095215, + 3.8378002643585205, + 3.82778263092041, + 3.8010475635528564, + 3.785697102546692, + 3.746701717376709, + 3.7296236753463745, + 3.702855706214905, + 3.76929771900177, + 3.8124542236328125, + 3.770027756690979, + 3.7500041723251343, + 3.812458038330078, + 3.7069091796875, + 3.717629313468933, + 3.804221034049988, + 3.687339663505554, + 3.8315632343292236, + 3.75238299369812, + 3.790883183479309, + 3.821676254272461, + 3.70363712310791, + 3.872152090072632, + 3.7634085416793823, + 3.80833101272583, + 3.8871110677719116, + 3.7327122688293457, + 3.7655383348464966, + 3.653213858604431, + 3.7166836261749268, + 3.626132607460022, + 3.8506171703338623, + 3.8395965099334717, + 3.745466113090515, + 3.7134172916412354, + 3.9056739807128906, + 3.7681725025177, + 3.8683289289474487, + 3.760556221008301, + 3.808861017227173, + 3.745569944381714, + 3.7169528007507324, + 3.63810932636261, + 3.755447268486023, + 3.8105039596557617, + 3.8070117235183716, + 3.6228156089782715, + 3.691024899482727, + 3.6400121450424194, + 3.781859278678894, + 3.769580364227295, + 3.94293212890625, + 3.7667696475982666, + 3.7146562337875366, + 3.7553051710128784, + 3.790878415107727, + 3.660861849784851, + 3.6430686712265015, + 3.7366358041763306, + 3.8828282356262207, + 3.726565957069397, + 3.7820019721984863, + 3.5684624910354614, + 3.7592166662216187, + 3.7245970964431763, + 3.6049997806549072, + 3.7303386926651, + 3.5794060230255127, + 3.497788190841675, + 3.669184446334839, + 3.6984695196151733, + 3.699113130569458, + 3.7515487670898438, + 3.529754400253296, + 3.682843327522278, + 3.6643911600112915, + 3.670441150665283, + 3.584942579269409, + 3.6308377981185913, + 3.697574496269226, + 3.473633885383606, + 3.749365210533142, + 3.6374634504318237, + 3.5631686449050903, + 3.5811980962753296, + 3.7190223932266235, + 3.635027766227722, + 3.594738483428955, + 3.6246585845947266, + 3.632146716117859, + 3.578223705291748, + 3.549683451652527, + 3.7129980325698853, + 3.602233648300171, + 3.6381582021713257, + 3.5879944562911987, + 3.582343816757202, + 3.5816017389297485, + 3.692224860191345, + 3.6521207094192505, + 3.591196894645691, + 3.65480375289917, + 3.546807050704956, + 3.5756932497024536, + 3.635653853416443, + 3.6858327388763428, + 3.699246883392334, + 3.7636529207229614, + 3.766334056854248, + 3.7343857288360596, + 3.7404032945632935, + 3.734983444213867, + 3.7260085344314575, + 3.6383819580078125, + 3.7513811588287354, + 3.639902114868164, + 3.6159236431121826, + 3.622720718383789, + 3.780625581741333, + 3.597754716873169, + 3.6096490621566772, + 3.6277246475219727, + 3.5796821117401123, + 3.642672538757324, + 3.6985995769500732, + 3.567579507827759, + 3.649978756904602, + 3.616002917289734, + 3.6114377975463867, + 3.566946864128113, + 3.6754640340805054, + 3.7680500745773315, + 3.6377726793289185, + 3.5909286737442017, + 3.8137844800949097, + 3.6878095865249634, + 3.600006937980652, + 3.717660069465637, + 3.5671303272247314, + 3.6624423265457153, + 3.581828236579895, + 3.5855082273483276, + 3.4853087663650513, + 3.670121431350708, + 3.6299444437026978, + 3.607633590698242, + 3.6782352924346924, + 3.6258851289749146, + 3.5614899396896362, + 3.6033390760421753, + 3.7027214765548706, + 3.5822412967681885, + 3.5289143323898315, + 3.7669087648391724, + 3.583937883377075, + 3.5845320224761963, + 3.6215022802352905, + 3.7128950357437134, + 3.620901346206665, + 3.650049567222595, + 3.530160903930664, + 3.681342601776123, + 3.601104497909546, + 3.7836302518844604, + 3.6774157285690308, + 3.7009541988372803, + 3.6063159704208374, + 3.612117052078247, + 3.642396092414856, + 3.501553535461426, + 3.5734695196151733, + 3.6840068101882935, + 3.6517159938812256, + 3.6702505350112915, + 3.6168540716171265, + 3.6686766147613525, + 3.574312925338745, + 3.71735942363739, + 3.5711381435394287, + 3.631404399871826, + 3.6095997095108032, + 3.6800246238708496, + 3.633169651031494, + 3.5331443548202515, + 3.537961483001709, + 3.702887535095215, + 3.5959651470184326, + 3.480753779411316, + 3.564604878425598, + 3.7526583671569824, + 3.6758172512054443, + 3.532732367515564, + 3.4703049659729004, + 3.5723416805267334, + 3.426279306411743, + 3.4424424171447754, + 3.517205595970154, + 3.602803945541382, + 3.5031031370162964, + 3.6123175621032715, + 3.555461287498474, + 3.4171866178512573, + 3.55705726146698, + 3.6434905529022217, + 3.5458805561065674, + 3.500074625015259, + 3.576985716819763, + 3.477160930633545, + 3.3384615182876587, + 3.5373395681381226, + 3.5590755939483643, + 3.570825695991516, + 3.5766016244888306, + 3.4609665870666504, + 3.518204689025879, + 3.453917384147644, + 3.5921413898468018, + 3.5320355892181396, + 3.469822645187378, + 3.521657943725586, + 3.5128207206726074, + 3.609504222869873, + 3.4574053287506104, + 3.520823359489441, + 3.46999454498291, + 3.5408636331558228, + 3.381771445274353, + 3.53617262840271, + 3.4906545877456665, + 3.48576819896698, + 3.458554983139038, + 3.5004003047943115, + 3.6830520629882812, + 3.5089199542999268, + 3.405583381652832, + 3.4761990308761597, + 3.536158800125122, + 3.518101930618286, + 3.6174046993255615, + 3.6867836713790894, + 3.5575186014175415, + 3.3956410884857178, + 3.554073691368103, + 3.7151612043380737, + 3.3931400775909424, + 3.4868342876434326, + 3.477769136428833, + 3.4940768480300903, + 3.43681263923645, + 3.3933427333831787, + 3.5625641345977783, + 3.5931111574172974, + 3.3826630115509033, + 3.5701147317886353, + 3.3070095777511597, + 3.4791394472122192, + 3.447005867958069, + 3.553295850753784, + 3.597637176513672, + 3.4685994386672974, + 3.4534735679626465, + 3.592639923095703, + 3.5338661670684814, + 3.42186963558197, + 3.700528383255005, + 3.4915595054626465, + 3.3979166746139526, + 3.470067620277405, + 3.5477036237716675, + 3.523895740509033, + 3.5499391555786133, + 3.2480921745300293, + 3.47530198097229, + 3.553342580795288, + 3.5039438009262085, + 3.4705368280410767, + 3.4245688915252686, + 3.5368568897247314, + 3.5097142457962036, + 3.5186197757720947, + 3.6011184453964233, + 3.521665930747986, + 3.468770384788513, + 3.457956314086914, + 3.4483046531677246, + 3.4690746068954468, + 3.469915270805359, + 3.4316368103027344, + 3.4613794088363647, + 3.534166932106018, + 3.4287939071655273, + 3.5146433115005493, + 3.383878469467163, + 3.4968278408050537, + 3.5945050716400146, + 3.5300198793411255, + 3.530857801437378, + 3.490093946456909, + 3.6191909313201904, + 3.4620131254196167, + 3.457775115966797, + 3.32681941986084, + 3.4764667749404907, + 3.503644824028015, + 3.464136242866516, + 3.4493690729141235, + 3.335270047187805, + 3.5313650369644165, + 3.417723536491394, + 3.645378351211548, + 3.5630394220352173, + 3.4396560192108154, + 3.5398499965667725, + 3.459647297859192, + 3.534913420677185, + 3.5363333225250244, + 3.380615234375, + 3.5823843479156494, + 3.3491756916046143, + 3.3096163272857666, + 3.3827731609344482, + 3.501747488975525, + 3.3986566066741943, + 3.3712193965911865, + 3.4138420820236206, + 3.451640248298645, + 3.4163788557052612, + 3.429766058921814, + 3.5232856273651123, + 3.4152203798294067, + 3.4105350971221924, + 3.3773783445358276, + 3.4417725801467896, + 3.4227429628372192, + 3.390128970146179, + 3.366945505142212, + 3.507304549217224, + 3.3471455574035645, + 3.305513620376587, + 3.4016722440719604, + 3.334036350250244, + 3.4171040058135986, + 3.3626145124435425, + 3.2995930910110474, + 3.483938694000244, + 3.3938523530960083, + 3.332010507583618, + 3.397916793823242, + 3.3992661237716675, + 3.471466302871704, + 3.4687438011169434, + 3.3711990118026733, + 3.4740792512893677, + 3.4224748611450195, + 3.472665548324585, + 3.4260427951812744, + 3.483260750770569, + 3.4736533164978027, + 3.298795461654663, + 3.332780361175537, + 3.4632277488708496, + 3.32337486743927, + 3.429076075553894, + 3.4280524253845215, + 3.3653661012649536, + 3.488158702850342, + 3.350005626678467, + 3.4032033681869507, + 3.3722275495529175, + 3.4609835147857666, + 3.4417343139648438, + 3.5324665307998657, + 3.356308937072754, + 3.4266539812088013, + 3.493434190750122, + 3.3952481746673584, + 3.4763128757476807, + 3.3825188875198364, + 3.256589889526367, + 3.38289737701416, + 3.382620334625244, + 3.4287935495376587, + 3.528159499168396, + 3.4138723611831665, + 3.610537528991699, + 3.2524912357330322, + 3.327070951461792, + 3.3942569494247437, + 3.501441717147827, + 3.480937123298645, + 3.5066120624542236, + 3.2453800439834595, + 3.4790444374084473, + 3.5004962682724, + 3.3408384323120117, + 3.4320813417434692, + 3.458421230316162, + 3.334680438041687, + 3.4151391983032227, + 3.2302844524383545, + 3.413100481033325, + 3.4624584913253784, + 3.3438867330551147, + 3.323051691055298, + 3.3627253770828247, + 3.519952893257141, + 3.373901844024658, + 3.3199002742767334, + 3.321769595146179, + 3.4569143056869507, + 3.400417447090149, + 3.3960373401641846, + 3.4041649103164673, + 3.4201571941375732, + 3.392422080039978, + 3.463068127632141, + 3.356834650039673, + 3.380342483520508, + 3.561477541923523, + 3.3646188974380493, + 3.450390577316284, + 3.37519907951355, + 3.4250646829605103, + 3.4982516765594482, + 3.451043963432312, + 3.4342739582061768, + 3.506224274635315, + 3.3639265298843384, + 3.334998369216919, + 3.3970125913619995, + 3.314605712890625, + 3.409545660018921, + 3.347879409790039, + 3.4576302766799927, + 3.471870541572571, + 3.4737545251846313, + 3.4966119527816772, + 3.5642696619033813, + 3.3762452602386475, + 3.4204074144363403, + 3.504969835281372, + 3.313559651374817, + 3.317274332046509, + 3.4970325231552124, + 3.3632359504699707, + 3.538765549659729, + 3.3496053218841553, + 3.30858314037323, + 3.234132170677185, + 3.2261550426483154, + 3.2621203660964966, + 3.1280081272125244, + 3.2861002683639526, + 3.3039634227752686, + 3.3328139781951904, + 3.36578905582428, + 3.368183732032776, + 3.2890431880950928, + 3.2792065143585205, + 3.3840209245681763, + 3.398371458053589, + 3.291463851928711, + 3.271666407585144, + 3.426403284072876, + 3.2994396686553955, + 3.1468886137008667, + 3.3564462661743164, + 3.3884332180023193, + 3.419461250305176, + 3.259359836578369, + 3.302704930305481, + 3.4824795722961426, + 3.2335108518600464, + 3.3284398317337036, + 3.4265620708465576, + 3.366294264793396, + 3.2510948181152344, + 3.440271258354187, + 3.3173508644104004, + 3.4171650409698486, + 3.3495534658432007, + 3.4170894622802734, + 3.293285608291626, + 3.4112327098846436, + 3.2803757190704346, + 3.3383750915527344, + 3.3703548908233643, + 3.299899101257324, + 3.29903507232666, + 3.387877106666565, + 3.36042320728302, + 3.4529114961624146, + 3.2863656282424927, + 3.306433916091919, + 3.3349040746688843, + 3.40041720867157, + 3.289233446121216, + 3.307782769203186, + 3.3329715728759766, + 3.258899688720703, + 3.285692572593689, + 3.3320047855377197, + 3.2152196168899536, + 3.3034098148345947, + 3.3867164850234985, + 3.3596845865249634, + 3.3366897106170654, + 3.4437484741210938, + 3.185905337333679, + 3.356268882751465, + 3.264418840408325, + 3.302676796913147, + 3.380355954170227, + 3.3271321058273315, + 3.391128897666931, + 3.244078755378723, + 3.3053393363952637, + 3.2608662843704224, + 3.3493679761886597, + 3.2697131633758545, + 3.2828367948532104, + 3.3481557369232178, + 3.346317172050476, + 3.4416245222091675, + 3.3976447582244873, + 3.264118790626526, + 3.306448817253113, + 3.3671791553497314, + 3.3169989585876465, + 3.219397187232971, + 3.369734764099121, + 3.4929033517837524, + 3.384065270423889, + 3.2969192266464233, + 3.3413314819335938, + 3.262984871864319, + 3.262259006500244, + 3.354483962059021, + 3.3962944746017456, + 3.372667908668518, + 3.3542217016220093, + 3.387665867805481, + 3.228913187980652, + 3.313429355621338, + 3.1006380319595337, + 3.3337533473968506, + 3.347936987876892, + 3.414733409881592, + 3.3618922233581543, + 3.3966277837753296, + 3.2443952560424805, + 3.342180609703064, + 3.279658794403076, + 3.327603816986084, + 3.3209967613220215, + 3.45198130607605, + 3.3904600143432617, + 3.3267284631729126, + 3.331247925758362, + 3.2078698873519897, + 3.3475139141082764, + 3.2509483098983765, + 3.2307006120681763, + 3.345652461051941, + 3.404017210006714, + 3.4567887783050537, + 3.3117092847824097, + 3.443925976753235, + 3.380674362182617, + 3.2455430030822754, + 3.2892669439315796, + 3.3408706188201904, + 3.313929319381714, + 3.331480383872986, + 3.156181812286377, + 3.4157984256744385, + 3.2651835680007935, + 3.295837879180908, + 3.281759738922119, + 3.3174777030944824, + 3.3742841482162476, + 3.43958055973053, + 3.326927065849304, + 3.1600773334503174, + 3.1663719415664673, + 3.3069674968719482, + 3.243302345275879, + 3.2292988300323486, + 3.277906656265259, + 3.2773104906082153, + 3.2778491973876953, + 3.332284450531006, + 3.2953639030456543, + 3.1670782566070557, + 3.3558374643325806, + 3.171457052230835, + 3.2483363151550293, + 3.4466021060943604, + 3.3621838092803955, + 3.223124146461487, + 3.242941737174988, + 3.2574362754821777, + 3.212261438369751, + 3.323077440261841, + 3.3030238151550293, + 3.2780704498291016, + 3.2700661420822144, + 3.225946068763733, + 3.2584091424942017, + 3.4259495735168457, + 3.262848734855652, + 3.3292888402938843, + 3.2716084718704224, + 3.300563097000122, + 3.400694251060486, + 3.185656189918518, + 3.3219767808914185, + 3.3215465545654297, + 3.309634804725647, + 3.3271244764328003, + 3.26759135723114, + 3.3020434379577637, + 3.1977622509002686, + 3.3664146661758423, + 3.1709893941879272, + 3.218787908554077, + 3.296671152114868, + 3.2676504850387573, + 3.2505708932876587, + 3.321174740791321, + 3.353270411491394, + 3.2217456102371216, + 3.342200517654419, + 3.263914108276367, + 3.439639091491699, + 3.3299046754837036, + 3.2545169591903687, + 3.331283450126648, + 3.2939352989196777, + 3.386718511581421, + 3.284320116043091, + 3.410927414894104, + 3.3451207876205444, + 3.2687509059906006, + 3.2828495502471924, + 3.255639433860779, + 3.3564062118530273, + 3.355911612510681, + 3.2976233959198, + 3.2690285444259644, + 3.3082737922668457, + 3.2248021364212036, + 3.333173394203186, + 3.259408116340637, + 3.306798815727234, + 3.2925857305526733, + 3.419541120529175, + 3.3113409280776978, + 3.264176845550537, + 3.2020362615585327, + 3.3130266666412354, + 3.243380308151245, + 3.3642033338546753, + 3.405263662338257, + 3.2825416326522827, + 3.3182965517044067, + 3.343605160713196, + 3.2516497373580933, + 3.302947759628296, + 3.2690707445144653, + 3.2750500440597534, + 3.310733914375305, + 3.264320492744446, + 3.191149353981018, + 3.267757773399353, + 3.3746596574783325, + 3.2979989051818848, + 3.319579839706421, + 3.2996320724487305, + 3.3615890741348267, + 3.21724534034729, + 3.265925407409668, + 3.2945138216018677, + 3.347082018852234, + 3.212705135345459, + 3.3508665561676025, + 3.3375200033187866 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 116.10963777706908, + 72.22160813342242, + 58.535356624649474, + 51.302081120432476, + 46.58635034129638, + 43.760088489234604, + 40.970761448931775, + 39.530078994958004, + 37.66982809117454, + 36.366894896592314, + 35.37298704443625, + 34.200824766069374, + 33.65359740687399, + 32.753601940795214, + 32.12212674818991, + 31.860506183271237, + 31.409084029194098, + 31.290595248640216, + 31.217157666664164, + 31.197051352044326 + ], + "best_ppl": 31.197051352044326, + "train_time_s": 22759.850919008255 +} \ No newline at end of file diff --git a/063_sparse_dispatch/shared_deltanet_1plus3of14_best.pt b/063_sparse_dispatch/shared_deltanet_1plus3of14_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..87c2cbd26f2850ecf06f9d8b3b427ac73248f3f8 --- /dev/null +++ b/063_sparse_dispatch/shared_deltanet_1plus3of14_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dce0546c474e994e2301b9fa18087c0ca894312b260f3f68c89d1a41400b688 +size 793621220 diff --git a/063_sparse_dispatch/shared_deltanet_1plus3of14_results.json b/063_sparse_dispatch/shared_deltanet_1plus3of14_results.json new file mode 100644 index 0000000000000000000000000000000000000000..6c8290e6dc13ea05edfbea2ac57a8178d2595994 --- /dev/null +++ b/063_sparse_dispatch/shared_deltanet_1plus3of14_results.json @@ -0,0 +1,877 @@ +{ + "label": "shared_deltanet_1plus3of14", + "attention_mode": "sparse", + "routed_attn_type": "deltanet", + "top_k_blocks": 3, + "num_thin_blocks": 15, + "num_shared_blocks": 1, + "total_params": 198162304, + "steps": [ + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 3.3011069297790527, + 3.446027398109436, + 3.3510056138038635, + 3.395783543586731, + 3.361626982688904, + 3.4127039313316345, + 3.4075623750686646, + 3.5260360836982727, + 3.4781943559646606, + 3.5117949843406677, + 3.472162902355194, + 3.329823911190033, + 3.4023295640945435, + 3.4410318732261658, + 3.3393014073371887, + 3.3931159377098083, + 3.4234129190444946, + 3.2396730184555054, + 3.376291513442993, + 3.4915135502815247, + 3.4786826372146606, + 3.4147780537605286, + 3.3084591031074524, + 3.376173436641693, + 3.409691870212555, + 3.5266886353492737, + 3.3110706210136414, + 3.343675374984741, + 3.3418567776679993, + 3.317708432674408, + 3.499760150909424, + 3.293021857738495, + 3.3742966055870056, + 3.3831936717033386, + 3.3002204298973083, + 3.320598542690277, + 3.2974650859832764, + 3.410097301006317, + 3.582844913005829, + 3.483644425868988, + 3.449106276035309, + 3.318218171596527, + 3.4465532302856445, + 3.381685733795166, + 3.4348809123039246, + 3.4255545139312744, + 3.33855402469635, + 3.40653532743454, + 3.546676516532898, + 3.3826258778572083, + 3.287880301475525, + 3.392186999320984, + 3.5026623606681824, + 3.3986164927482605, + 3.4325249791145325, + 3.3875280618667603, + 3.332910716533661, + 3.4940658807754517, + 3.444811761379242, + 3.317669689655304, + 3.3318751454353333, + 3.5243483781814575, + 3.436042010784149, + 3.37057363986969, + 3.3997941613197327, + 3.436142385005951, + 3.40479439496994, + 3.330371081829071, + 3.4792802333831787, + 3.3780974745750427, + 3.4986985325813293, + 3.335853397846222, + 3.212773621082306, + 3.2877249717712402, + 3.3789226412773132, + 3.408601999282837, + 3.336510181427002, + 3.3930322527885437, + 3.331311047077179, + 3.4265912771224976, + 3.3858758211135864, + 3.304465353488922, + 3.4893544912338257, + 3.384105384349823, + 3.3823968172073364, + 3.4424701929092407, + 3.391747534275055, + 3.3396825194358826, + 3.542342245578766, + 3.407067894935608, + 3.407038927078247, + 3.396577537059784, + 3.4100197553634644, + 3.3847396969795227, + 3.3935956358909607, + 3.4740589261054993, + 3.377112865447998, + 3.4074930548667908, + 3.4463701248168945, + 3.3276885747909546, + 3.37268990278244, + 3.426459491252899, + 3.432569980621338, + 3.31551730632782, + 3.3440628051757812, + 3.378406345844269, + 3.3432281613349915, + 3.305702567100525, + 3.1898974180221558, + 3.535149872303009, + 3.343954086303711, + 3.3253214359283447, + 3.3403550386428833, + 3.256632924079895, + 3.314651846885681, + 3.392135798931122, + 3.422861397266388, + 3.4187868237495422, + 3.325214385986328, + 3.3850829005241394, + 3.2167271971702576, + 3.2487125992774963, + 3.434311032295227, + 3.4610827565193176, + 3.440936505794525, + 3.3869956731796265, + 3.1424307227134705, + 3.197014093399048, + 3.1727998852729797, + 3.3219701051712036, + 3.3493295311927795, + 3.1894261240959167, + 3.1535066962242126, + 3.3241798281669617, + 3.2368953824043274, + 3.228658974170685, + 3.2916091084480286, + 3.0651795864105225, + 3.2124063372612, + 3.1668395400047302, + 3.2503820657730103, + 3.127500295639038, + 3.2332760095596313, + 3.2849250435829163, + 3.1379141211509705, + 3.3230836391448975, + 3.2533901929855347, + 3.244717240333557, + 3.3057116270065308, + 3.2608912587165833, + 3.167478561401367, + 3.2925865650177, + 3.268193781375885, + 3.2267903089523315, + 3.341377079486847, + 3.26982718706131, + 3.3056614995002747, + 3.372184157371521, + 3.3482291102409363, + 3.2215123772621155, + 3.2234310507774353, + 3.078993499279022, + 3.2797388434410095, + 3.2822675108909607, + 3.4040281772613525, + 3.277656853199005, + 3.3136595487594604, + 3.140284538269043, + 3.227102816104889, + 3.176835298538208, + 3.373098373413086, + 3.044971227645874, + 3.232684910297394, + 3.2153143286705017, + 3.373078465461731, + 3.181230306625366, + 3.305374026298523, + 3.32633900642395, + 3.3102471232414246, + 3.2469199895858765, + 3.2138900756835938, + 3.2028976678848267, + 3.129115343093872, + 3.301414966583252, + 3.3212053179740906, + 3.1973769068717957, + 3.2872363924980164, + 3.265541911125183, + 3.24467134475708, + 3.2571690678596497, + 3.290630280971527, + 3.2967474460601807, + 3.274805963039398, + 3.298877775669098, + 3.21401184797287, + 3.2924908995628357, + 3.1571799516677856, + 3.1718422770500183, + 3.3168466687202454, + 3.3341678380966187, + 3.403030812740326, + 3.3024569749832153, + 3.388028144836426, + 3.2667983174324036, + 3.2322720289230347, + 3.340302288532257, + 3.170567512512207, + 3.3355661630630493, + 3.43593031167984, + 3.459572434425354, + 3.3538485169410706, + 3.3300876021385193, + 3.219836115837097, + 3.3115832805633545, + 3.2796261310577393, + 3.1858112812042236, + 3.3396560549736023, + 3.2126166224479675, + 3.3082869052886963, + 3.260067582130432, + 3.2663575410842896, + 3.3802290558815002, + 3.3264832496643066, + 3.34655225276947, + 3.3607420921325684, + 3.3054884672164917, + 3.105477035045624, + 3.3935450315475464, + 3.3066316843032837, + 3.263315737247467, + 3.3040730953216553, + 3.057393193244934, + 3.4063660502433777, + 3.316799819469452, + 3.2864338159561157, + 3.247461438179016, + 3.346256136894226, + 3.261677086353302, + 3.3143688440322876, + 3.355050265789032, + 3.258825421333313, + 3.367133617401123, + 3.2914782762527466, + 3.2655946016311646, + 3.340275764465332, + 3.4383267164230347, + 3.284054756164551, + 3.3176000118255615, + 3.378874123096466, + 3.250009775161743, + 3.396986484527588, + 3.374873101711273, + 3.306040406227112, + 3.2697983980178833, + 3.280588150024414, + 3.229267120361328, + 3.1918880343437195, + 3.230972170829773, + 3.1419365406036377, + 3.213904321193695, + 3.273516535758972, + 3.233780026435852, + 3.253082752227783, + 3.103553593158722, + 3.187732994556427, + 3.1819035410881042, + 3.1650431156158447, + 3.192185699939728, + 3.206122398376465, + 3.1057820916175842, + 3.2165404558181763, + 3.2809155583381653, + 3.12763112783432, + 3.2008492946624756, + 3.248573362827301, + 3.1628924012184143, + 3.167323052883148, + 3.120785713195801, + 3.2112757563591003, + 3.1367437839508057, + 3.279905140399933, + 3.155295729637146, + 3.2514325380325317, + 2.998795747756958, + 3.1318089365959167, + 3.2111738324165344, + 3.222928285598755, + 3.2540664076805115, + 3.1325284242630005, + 3.1991764307022095, + 3.350243389606476, + 3.278160035610199, + 3.3519363403320312, + 3.10278582572937, + 3.1156614422798157, + 3.210382342338562, + 3.216593086719513, + 3.1796346306800842, + 3.3100653290748596, + 3.271186888217926, + 3.162985861301422, + 3.173380672931671, + 3.156835198402405, + 3.256184160709381, + 3.2206040620803833, + 3.031699240207672, + 3.3006827235221863, + 3.2803632020950317, + 3.290664315223694, + 3.115189254283905, + 3.039691209793091, + 3.201204001903534, + 3.3244885206222534, + 3.1968077421188354, + 3.074625074863434, + 3.2275421619415283, + 3.251653552055359, + 3.2890714406967163, + 3.2232785820961, + 3.152096927165985, + 3.0945791006088257, + 3.172852873802185, + 3.2503989934921265, + 3.1948752403259277, + 3.1612823009490967, + 3.12576824426651, + 3.1998395323753357, + 3.348929286003113, + 3.1848422288894653, + 3.1407018899917603, + 3.265146255493164, + 3.055180251598358, + 3.2440096735954285, + 3.0969766974449158, + 3.1567772030830383, + 3.1400290727615356, + 3.115308463573456, + 3.101990044116974, + 3.2019366025924683, + 3.349894344806671, + 3.201279103755951, + 3.2575660943984985, + 3.2201634645462036, + 3.2365715503692627, + 3.2080121636390686, + 3.1043378114700317, + 3.1031737327575684, + 3.2516555786132812, + 3.2280704379081726, + 3.0826423168182373, + 3.293536126613617, + 3.314238488674164, + 3.2389661073684692, + 3.1556732058525085, + 3.2479458451271057, + 3.2836028933525085, + 3.181821823120117, + 3.1562746167182922, + 3.282842218875885, + 3.330852210521698, + 3.0410695672035217, + 3.1503743529319763, + 3.0325823426246643, + 3.211082935333252, + 3.2237204909324646, + 3.220243453979492, + 3.354797065258026, + 3.134026825428009, + 3.260929763317108, + 3.106544077396393, + 3.2335670590400696, + 3.089933931827545, + 3.2172353267669678, + 3.218445122241974, + 3.214735507965088, + 3.219271242618561, + 3.1320210695266724, + 3.1946666836738586, + 3.276325225830078, + 3.0704615116119385, + 3.150572419166565, + 3.252513289451599, + 3.2391814589500427, + 3.139066517353058, + 3.230575382709503, + 3.151190400123596, + 3.189992845058441, + 3.258715808391571, + 3.1868631839752197, + 3.1316040754318237, + 3.165907859802246, + 3.2741326689720154, + 3.1424142122268677, + 3.2029353976249695, + 3.3266127705574036, + 3.250337541103363, + 3.1336550116539, + 3.1634944677352905, + 3.205815076828003, + 3.1082793474197388, + 3.1343966126441956, + 3.2170210480690002, + 3.0860401391983032, + 3.140053391456604, + 3.074195921421051, + 3.2834903597831726, + 3.2727912068367004, + 3.189286530017853, + 3.193311929702759, + 3.282823860645294, + 3.2051191926002502, + 3.1602914333343506, + 3.159366190433502, + 3.158388137817383, + 3.044834554195404, + 3.139878213405609, + 3.277008295059204, + 3.2881075739860535, + 3.1778354048728943, + 3.1676658391952515 + ], + "val_steps": [ + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 33.2388870955996, + 32.20278874421016, + 31.370066414294612, + 31.185965389441655, + 30.367566975170543, + 30.202562567967966, + 30.083915630080618, + 29.96062769398296, + 29.983677995005316 + ], + "best_ppl": 29.96062769398296, + "train_time_s": 39367.383808374405 +} \ No newline at end of file diff --git a/063_sparse_dispatch/shared_deltanet_1plus3of14_s137_best.pt b/063_sparse_dispatch/shared_deltanet_1plus3of14_s137_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec98c0b5f6948a2d6268b3a87b6e88d329c72b26 --- /dev/null +++ b/063_sparse_dispatch/shared_deltanet_1plus3of14_s137_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c9952936cd423b40d068282299e582d62656563ac8da2fc16de366e39faf775 +size 793632505 diff --git a/063_sparse_dispatch/shared_deltanet_1plus3of14_s137_results.json b/063_sparse_dispatch/shared_deltanet_1plus3of14_s137_results.json new file mode 100644 index 0000000000000000000000000000000000000000..abbcb790e4062f3b2ae678bc152132c5f114f7af --- /dev/null +++ b/063_sparse_dispatch/shared_deltanet_1plus3of14_s137_results.json @@ -0,0 +1,509 @@ +{ + "label": "shared_deltanet_1plus3of14_s137", + "attention_mode": "sparse", + "routed_attn_type": "deltanet", + "top_k_blocks": 3, + "num_thin_blocks": 15, + "num_shared_blocks": 1, + "total_params": 198162304, + "steps": [ + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 3.1790178418159485, + 3.361535668373108, + 3.2869909405708313, + 3.2485504150390625, + 3.138944089412689, + 3.2284116744995117, + 3.1340253949165344, + 3.2839935421943665, + 3.2661383152008057, + 3.194003701210022, + 3.1933178305625916, + 3.1672229766845703, + 3.223256289958954, + 3.1503701210021973, + 3.290842592716217, + 3.301255941390991, + 3.2859421968460083, + 3.178034722805023, + 3.251354396343231, + 3.2968695759773254, + 3.1146817803382874, + 3.285467743873596, + 3.1932525634765625, + 3.254716634750366, + 3.289695620536804, + 3.266287624835968, + 3.211413085460663, + 3.025796353816986, + 3.243861496448517, + 3.2353760600090027, + 3.232231914997101, + 3.2363837361335754, + 3.237721025943756, + 3.2511550188064575, + 3.2395176887512207, + 3.198659896850586, + 3.150604248046875, + 3.2923856377601624, + 3.2604060769081116, + 3.240048050880432, + 3.285155713558197, + 3.197970390319824, + 3.257248640060425, + 3.152850568294525, + 3.2915534377098083, + 3.134064733982086, + 3.1506868600845337, + 3.1669639945030212, + 3.2779109477996826, + 3.1507975459098816, + 3.165447175502777, + 3.198488473892212, + 3.312743902206421, + 3.2840139865875244, + 3.109395444393158, + 3.2332016229629517, + 3.2703052759170532, + 3.382263660430908, + 3.340417444705963, + 3.273165464401245, + 3.2294009923934937, + 3.3241982460021973, + 3.3774101734161377, + 3.237842321395874, + 3.228154420852661, + 3.248302936553955, + 3.174240469932556, + 3.178771734237671, + 3.2950803637504578, + 3.2374883890151978, + 3.2331807613372803, + 3.3402695655822754, + 3.235790193080902, + 3.254418194293976, + 3.040130078792572, + 3.264219105243683, + 3.3032771348953247, + 3.2794174551963806, + 3.205581545829773, + 3.2318274974823, + 3.190579831600189, + 3.2240536212921143, + 3.2675729990005493, + 3.1213735342025757, + 3.215216636657715, + 3.211502969264984, + 3.3485928177833557, + 3.221096992492676, + 3.234474241733551, + 3.26051127910614, + 3.244112730026245, + 3.1336780786514282, + 3.24263733625412, + 3.2511982917785645, + 3.1797215342521667, + 3.2925336956977844, + 3.2230530381202698, + 3.2383888363838196, + 3.2360019087791443, + 3.269525170326233, + 3.260856807231903, + 3.2916452884674072, + 3.223530352115631, + 3.329437017440796, + 3.199472486972809, + 3.248312830924988, + 3.1861166954040527, + 3.261384665966034, + 3.1355835795402527, + 3.260693609714508, + 3.182935059070587, + 3.1279608011245728, + 3.2528252601623535, + 3.2813507318496704, + 3.149928092956543, + 3.1712167263031006, + 3.1948532462120056, + 3.4282172322273254, + 3.329008162021637, + 3.1551337242126465, + 3.11638867855072, + 3.1185089349746704, + 3.2663373947143555, + 3.0933905839920044, + 3.2046223878860474, + 3.3722128868103027, + 3.214718222618103, + 3.1023630499839783, + 3.182297646999359, + 3.295023560523987, + 3.1169005632400513, + 3.229159951210022, + 3.229397714138031, + 3.222218155860901, + 3.2788758873939514, + 3.1699201464653015, + 3.177276909351349, + 3.143637955188751, + 3.2955046892166138, + 3.1865656971931458, + 3.168478548526764, + 3.1130863428115845, + 3.17856228351593, + 3.0933157205581665, + 3.152744770050049, + 3.2289482951164246, + 3.2036989331245422, + 3.204250752925873, + 3.233960449695587, + 3.146390736103058, + 3.2354644536972046, + 3.191212296485901, + 3.208091378211975, + 3.0197485089302063, + 3.1845536828041077, + 3.166961193084717, + 3.1495460867881775, + 3.2455697059631348, + 3.2080047130584717, + 3.164670407772064, + 3.3391619324684143, + 3.117949664592743, + 3.257068932056427, + 3.308716595172882, + 3.2506116032600403, + 3.1041290760040283, + 3.3207961916923523, + 3.3055407404899597, + 3.298076868057251, + 3.260402798652649, + 3.158174991607666, + 3.166881799697876, + 3.201426386833191, + 3.167394459247589, + 3.1726537942886353, + 3.249425709247589, + 3.174528479576111, + 3.2980366349220276, + 3.211656332015991, + 3.139594614505768, + 3.2118977308273315, + 3.031608998775482, + 3.0140774250030518, + 3.1629591584205627, + 3.2517696022987366, + 3.1237809658050537, + 3.114691972732544, + 3.222800374031067, + 3.2893598079681396, + 3.0199074149131775, + 3.2628564834594727, + 3.2221822142601013, + 3.230771064758301, + 3.068610727787018, + 3.217560648918152, + 3.1998871564865112, + 3.1704198718070984, + 3.1280778646469116, + 3.187176823616028, + 3.208182990550995, + 3.185747444629669, + 3.219425320625305, + 3.2845340967178345, + 3.0846288800239563, + 3.270413637161255, + 3.167137920856476, + 3.1390108466148376, + 3.083063006401062, + 3.1559375524520874, + 3.222133696079254, + 3.1150444746017456, + 3.194677948951721, + 3.2014201879501343, + 3.233451724052429, + 3.187688708305359, + 3.1078810691833496, + 3.34978711605072, + 3.076117992401123, + 3.3011956810951233, + 3.223429501056671, + 3.273516893386841, + 3.296511650085449, + 3.2215448021888733, + 3.18707412481308, + 3.1612666845321655, + 3.045747697353363, + 3.246656358242035, + 3.286717712879181, + 3.3126355409622192, + 3.125333607196808, + 3.0616981983184814, + 3.227598786354065, + 3.1743870973587036, + 3.117215633392334, + 3.299328088760376, + 3.2005701065063477, + 3.1869155764579773, + 3.2353869676589966, + 3.205810010433197, + 3.180010437965393 + ], + "val_steps": [ + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 30.551719447735913, + 30.108112534096264, + 30.01374462156388, + 29.94712957225196, + 29.939773897530866 + ], + "best_ppl": 29.939773897530866, + "train_time_s": 22768.160760879517 +} \ No newline at end of file diff --git a/063_sparse_dispatch/shared_deltanet_1plus3of14_s256_best.pt b/063_sparse_dispatch/shared_deltanet_1plus3of14_s256_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..2dd7846cbdd21349214a1cd73e6e82d308344529 --- /dev/null +++ b/063_sparse_dispatch/shared_deltanet_1plus3of14_s256_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a808c847d30194d28740cab46a5ddaa9894d3e51713cc7b8ddb3df3a3b90476c +size 793632505 diff --git a/063_sparse_dispatch/shared_deltanet_1plus3of14_s256_results.json b/063_sparse_dispatch/shared_deltanet_1plus3of14_s256_results.json new file mode 100644 index 0000000000000000000000000000000000000000..9a56dc63269717c0e2b82c59767e7ad0a8baedad --- /dev/null +++ b/063_sparse_dispatch/shared_deltanet_1plus3of14_s256_results.json @@ -0,0 +1,469 @@ +{ + "label": "shared_deltanet_1plus3of14_s256", + "attention_mode": "sparse", + "routed_attn_type": "deltanet", + "top_k_blocks": 3, + "num_thin_blocks": 15, + "num_shared_blocks": 1, + "total_params": 198162304, + "steps": [ + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 3.271580398082733, + 3.233698785305023, + 3.1553163528442383, + 3.238657832145691, + 3.1846946477890015, + 3.1482388377189636, + 3.3374069929122925, + 3.243934154510498, + 3.1852933764457703, + 3.092354893684387, + 3.2014933228492737, + 3.316309094429016, + 3.1874226927757263, + 3.233596622943878, + 3.296329379081726, + 3.059349834918976, + 3.211909532546997, + 3.3920289874076843, + 3.237040162086487, + 3.2539325952529907, + 3.2262442111968994, + 3.2300055027008057, + 3.1384770274162292, + 3.283572018146515, + 3.2243531346321106, + 3.3516860008239746, + 3.1789477467536926, + 3.2770442366600037, + 3.251610577106476, + 3.1384871006011963, + 3.0827934741973877, + 3.324459433555603, + 3.1378195881843567, + 3.348997473716736, + 3.1376696825027466, + 3.220693051815033, + 3.105282485485077, + 3.238626182079315, + 3.3160610795021057, + 3.3297781348228455, + 3.2246652841567993, + 3.2933812141418457, + 3.045259714126587, + 3.1512657403945923, + 3.326297104358673, + 3.149048328399658, + 3.229825258255005, + 3.2151777744293213, + 3.260684013366699, + 3.0966402888298035, + 3.2917149662971497, + 3.2554067373275757, + 3.202645778656006, + 3.3227555751800537, + 3.211499512195587, + 3.281427025794983, + 3.194169282913208, + 3.080756902694702, + 3.2762985229492188, + 3.3539289832115173, + 3.217350423336029, + 3.4156461358070374, + 3.4019535779953003, + 3.1632670164108276, + 3.319831967353821, + 3.2845652103424072, + 3.177123785018921, + 3.1331974267959595, + 3.263761341571808, + 3.2118170261383057, + 3.244733452796936, + 3.071764886379242, + 3.2494759559631348, + 3.1277610063552856, + 3.2849007844924927, + 3.2711914777755737, + 3.3040801882743835, + 3.0995683670043945, + 3.3156930208206177, + 3.334414780139923, + 3.208107650279999, + 3.211061120033264, + 3.3186922669410706, + 3.335366189479828, + 3.3020249605178833, + 3.03622305393219, + 3.1618884801864624, + 3.1935920119285583, + 3.138321578502655, + 3.2544131875038147, + 3.181714355945587, + 3.324799597263336, + 3.261107921600342, + 3.288636326789856, + 3.2075213193893433, + 3.180125117301941, + 3.3059555888175964, + 3.1973150968551636, + 3.2866761088371277, + 3.3630685806274414, + 3.2090815901756287, + 3.2890354990959167, + 3.2840510606765747, + 3.2698824405670166, + 3.2627822756767273, + 3.172643542289734, + 3.2176605463027954, + 3.2508970499038696, + 3.2819265723228455, + 3.318573772907257, + 3.2866501808166504, + 3.2918347120285034, + 3.093239665031433, + 3.296912670135498, + 3.244502305984497, + 3.13728529214859, + 3.08354252576828, + 3.1945632100105286, + 3.201170563697815, + 3.299436569213867, + 3.18574059009552, + 3.157462418079376, + 3.1698593497276306, + 3.293003797531128, + 3.1576178669929504, + 3.150218963623047, + 3.1851435899734497, + 3.3082967400550842, + 3.255796253681183, + 3.1880853176116943, + 3.2297441959381104, + 3.1793909668922424, + 2.941390573978424, + 3.1446509957313538, + 3.0488356351852417, + 3.132507562637329, + 3.2039453387260437, + 3.2116600275039673, + 3.1985409259796143, + 3.2913583517074585, + 3.2420926690101624, + 3.1625859141349792, + 3.0991095900535583, + 3.110986828804016, + 3.0638473629951477, + 3.2493685483932495, + 3.0511325001716614, + 3.2010039687156677, + 3.2446328997612, + 3.197023570537567, + 3.2629971504211426, + 3.0566259026527405, + 3.135387659072876, + 3.079987406730652, + 3.217785060405731, + 3.060091555118561, + 3.166021943092346, + 3.1233555674552917, + 3.106074333190918, + 3.1095082759857178, + 3.2039119601249695, + 3.1989879608154297, + 3.109674394130707, + 3.224468469619751, + 3.187771737575531, + 3.188271224498749, + 3.165545642375946, + 3.288039743900299, + 3.1639636754989624, + 3.289075493812561, + 3.212912380695343, + 3.2479053139686584, + 3.1260878443717957, + 3.261200964450836, + 3.180641293525696, + 3.1406851410865784, + 3.235730826854706, + 3.2144469022750854, + 3.112558662891388, + 3.18584281206131, + 3.16837215423584, + 3.1207187175750732, + 3.1090038418769836, + 3.250825047492981, + 3.125275671482086, + 3.199085772037506, + 3.248224139213562, + 3.1913411021232605, + 3.192950129508972, + 3.10829496383667, + 3.1445164680480957, + 3.2600017189979553, + 3.1932275891304016, + 3.148731052875519, + 3.188366413116455, + 3.1595653891563416, + 3.2935534715652466, + 3.088918387889862, + 3.169668197631836, + 3.1537808775901794, + 3.1684553027153015, + 3.139264464378357, + 3.192494511604309, + 3.18748277425766, + 3.1408028602600098, + 3.23759388923645, + 3.13320529460907, + 3.261858105659485, + 3.11908882856369, + 3.155086040496826, + 3.2518513798713684, + 3.187950909137726, + 3.1668444871902466, + 3.203123450279236, + 3.117341935634613, + 3.163973331451416, + 3.1424219012260437, + 3.0870957374572754, + 3.1694981455802917, + 3.029814898967743 + ], + "val_steps": [ + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 30.71478556111574, + 30.411874780179318, + 30.078485771637173, + 30.0992705566202, + 30.09093982505957 + ], + "best_ppl": 30.078485771637173, + "train_time_s": 20866.347957134247 +} \ No newline at end of file diff --git a/063_sparse_dispatch/shared_softmax_1plus2of5_best.pt b/063_sparse_dispatch/shared_softmax_1plus2of5_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..00193ee6de38f790e6657aabc607eaabeaf06726 --- /dev/null +++ b/063_sparse_dispatch/shared_softmax_1plus2of5_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63c2ae19326a5bec6fa3395abaffe52e2d9f8905c47ef760d36f05fad43f13dd +size 341411935 diff --git a/063_sparse_dispatch/shared_softmax_1plus2of5_results.json b/063_sparse_dispatch/shared_softmax_1plus2of5_results.json new file mode 100644 index 0000000000000000000000000000000000000000..2a0b55555c01966aaaba9644c928474ef5bd7da6 --- /dev/null +++ b/063_sparse_dispatch/shared_softmax_1plus2of5_results.json @@ -0,0 +1,1040 @@ +{ + "label": "shared_softmax_1plus2of5", + "seed": 42, + "attention_mode": "sparse", + "routed_attn_type": "softmax", + "top_k_blocks": 2, + "num_thin_blocks": 5, + "num_shared_blocks": 1, + "total_params": 85293056, + "steps": [ + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 3.6083518266677856, + 3.617443323135376, + 3.7295976877212524, + 3.712302803993225, + 3.6362812519073486, + 3.5672576427459717, + 3.5720770359039307, + 3.579551339149475, + 3.652143716812134, + 3.7311590909957886, + 3.567141532897949, + 3.611405372619629, + 3.6820610761642456, + 3.5733327865600586, + 3.652596592903137, + 3.713971257209778, + 3.7153526544570923, + 3.6786985397338867, + 3.6578776836395264, + 3.645725965499878, + 3.6625555753707886, + 3.650027275085449, + 3.7017802000045776, + 3.5309892892837524, + 3.5268043279647827, + 3.5971521139144897, + 3.632554054260254, + 3.6731910705566406, + 3.741512417793274, + 3.610128879547119, + 3.6152586936950684, + 3.785200834274292, + 3.4503750801086426, + 3.826375126838684, + 3.6294844150543213, + 3.6845279932022095, + 3.607928156852722, + 3.6133742332458496, + 3.6545424461364746, + 3.681681513786316, + 3.6994056701660156, + 3.6764410734176636, + 3.6946834325790405, + 3.5719927549362183, + 3.679455518722534, + 3.6010371446609497, + 3.6601967811584473, + 3.7879444360733032, + 3.6054943799972534, + 3.6594916582107544, + 3.676219344139099, + 3.6813101768493652, + 3.642167091369629, + 3.639889717102051, + 3.5708905458450317, + 3.696524500846863, + 3.5874134302139282, + 3.6461312770843506, + 3.684667706489563, + 3.625743269920349, + 3.6690688133239746, + 3.5464245080947876, + 3.6885889768600464, + 3.615282416343689, + 3.719055652618408, + 3.5640047788619995, + 3.6580100059509277, + 3.6788692474365234, + 3.5977957248687744, + 3.5949110984802246, + 3.6985031366348267, + 3.482587218284607, + 3.5751004219055176, + 3.677226424217224, + 3.5593827962875366, + 3.551208972930908, + 3.560399889945984, + 3.73794949054718, + 3.7549997568130493, + 3.6640018224716187, + 3.51772940158844, + 3.6257635354995728, + 3.6517465114593506, + 3.779793858528137, + 3.5792198181152344, + 3.5952887535095215, + 3.426683783531189, + 3.720228672027588, + 3.6149317026138306, + 3.6267229318618774, + 3.4159196615219116, + 3.5339343547821045, + 3.5696059465408325, + 3.651535987854004, + 3.627320647239685, + 3.7182990312576294, + 3.5463268756866455, + 3.589683175086975, + 3.5319172143936157, + 3.526711583137512, + 3.5195542573928833, + 3.693802833557129, + 3.65901780128479, + 3.5430057048797607, + 3.713660478591919, + 3.624492883682251, + 3.626517415046692, + 3.578974962234497, + 3.7296053171157837, + 3.5731054544448853, + 3.4655762910842896, + 3.6748491525650024, + 3.649127721786499, + 3.678430914878845, + 3.617204785346985, + 3.657150149345398, + 3.59822678565979, + 3.518629789352417, + 3.585882067680359, + 3.6027764081954956, + 3.6480785608291626, + 3.678100109100342, + 3.612151622772217, + 3.7235403060913086, + 3.6397976875305176, + 3.652080535888672, + 3.5722548961639404, + 3.5214706659317017, + 3.5502564907073975, + 3.549044966697693, + 3.425241231918335, + 3.6062567234039307, + 3.663543224334717, + 3.7452136278152466, + 3.583325147628784, + 3.5805752277374268, + 3.623031973838806, + 3.5880645513534546, + 3.505559802055359, + 3.5303760766983032, + 3.7450627088546753, + 3.454151153564453, + 3.480127453804016, + 3.4835102558135986, + 3.6425013542175293, + 3.6748422384262085, + 3.451498508453369, + 3.5949591398239136, + 3.653231143951416, + 3.639170289039612, + 3.6041035652160645, + 3.4780144691467285, + 3.507498621940613, + 3.546748399734497, + 3.5783804655075073, + 3.424217700958252, + 3.514679789543152, + 3.4578524827957153, + 3.472074270248413, + 3.543735384941101, + 3.5907342433929443, + 3.5740398168563843, + 3.6696428060531616, + 3.6030850410461426, + 3.620068073272705, + 3.4487775564193726, + 3.488271951675415, + 3.4990309476852417, + 3.53454852104187, + 3.516539692878723, + 3.4878714084625244, + 3.637736201286316, + 3.493032693862915, + 3.5864157676696777, + 3.6430463790893555, + 3.6769834756851196, + 3.504554510116577, + 3.5667686462402344, + 3.575843095779419, + 3.536365270614624, + 3.631547212600708, + 3.4846911430358887, + 3.610027551651001, + 3.566365122795105, + 3.5841366052627563, + 3.4702959060668945, + 3.6190366744995117, + 3.493306517601013, + 3.5534794330596924, + 3.588853597640991, + 3.599913477897644, + 3.52876615524292, + 3.5664411783218384, + 3.4881184101104736, + 3.6531611680984497, + 3.6187076568603516, + 3.5515064001083374, + 3.651723623275757, + 3.4852182865142822, + 3.494141459465027, + 3.537196397781372, + 3.41710889339447, + 3.44149649143219, + 3.481061339378357, + 3.5677181482315063, + 3.477903127670288, + 3.376874804496765, + 3.452937126159668, + 3.6020482778549194, + 3.511450171470642, + 3.5582104921340942, + 3.4963245391845703, + 3.4490036964416504, + 3.442155122756958, + 3.482515335083008, + 3.293032169342041, + 3.5172154903411865, + 3.48720383644104, + 3.363356590270996, + 3.6077336072921753, + 3.421395182609558, + 3.5739986896514893, + 3.590048909187317, + 3.4384931325912476, + 3.630783200263977, + 3.5215651988983154, + 3.4876843690872192, + 3.4968947172164917, + 3.521423578262329, + 3.467708706855774, + 3.5595961809158325, + 3.6111427545547485, + 3.4631383419036865, + 3.5073468685150146, + 3.40625536441803, + 3.5261353254318237, + 3.50848126411438, + 3.519644618034363, + 3.518497109413147, + 3.5096715688705444, + 3.567242383956909, + 3.4615172147750854, + 3.5262739658355713, + 3.519886016845703, + 3.258573532104492, + 3.6366811990737915, + 3.4558463096618652, + 3.503255605697632, + 3.455098867416382, + 3.5097711086273193, + 3.503081440925598, + 3.466107487678528, + 3.4031264781951904, + 3.51539945602417, + 3.3784362077713013, + 3.446903109550476, + 3.511398196220398, + 3.5130993127822876, + 3.361214518547058, + 3.518704414367676, + 3.4994720220565796, + 3.486672043800354, + 3.433280110359192, + 3.4757182598114014, + 3.449890613555908, + 3.5136486291885376, + 3.440657138824463, + 3.3433997631073, + 3.2992366552352905, + 3.4743082523345947, + 3.469030499458313, + 3.458367347717285, + 3.5044779777526855, + 3.4330222606658936, + 3.5186437368392944, + 3.5639636516571045, + 3.4685128927230835, + 3.508505344390869, + 3.4427106380462646, + 3.5575973987579346, + 3.4300689697265625, + 3.4150384664535522, + 3.447471857070923, + 3.386540174484253, + 3.4140655994415283, + 3.3869963884353638, + 3.4196581840515137, + 3.430400013923645, + 3.4538460969924927, + 3.407589316368103, + 3.4028146266937256, + 3.574006676673889, + 3.3886715173721313, + 3.5382401943206787, + 3.4221279621124268, + 3.4199094772338867, + 3.5536309480667114, + 3.516419291496277, + 3.4661927223205566, + 3.571042060852051, + 3.54723060131073, + 3.471065640449524, + 3.434295177459717, + 3.4741311073303223, + 3.4556405544281006, + 3.49492347240448, + 3.4121590852737427, + 3.4773097038269043, + 3.436905264854431, + 3.424375891685486, + 3.4495917558670044, + 3.423703908920288, + 3.410847783088684, + 3.3947677612304688, + 3.5955268144607544, + 3.5147539377212524, + 3.3649595975875854, + 3.5461241006851196, + 3.468516707420349, + 3.342689871788025, + 3.5424418449401855, + 3.4241353273391724, + 3.555933356285095, + 3.4582310914993286, + 3.49621844291687, + 3.511426329612732, + 3.4613765478134155, + 3.5629189014434814, + 3.452308773994446, + 3.5120609998703003, + 3.409445881843567, + 3.3589751720428467, + 3.438714027404785, + 3.4173730611801147, + 3.4602240324020386, + 3.497611880302429, + 3.492923855781555, + 3.5291624069213867, + 3.5523715019226074, + 3.4475982189178467, + 3.3995331525802612, + 3.5589756965637207, + 3.2780728340148926, + 3.561224937438965, + 3.3467921018600464, + 3.402703881263733, + 3.3939777612686157, + 3.4750036001205444, + 3.377066135406494, + 3.3744245767593384, + 3.4410181045532227, + 3.495090365409851, + 3.506938099861145, + 3.3880698680877686, + 3.5287803411483765, + 3.347741723060608, + 3.493304491043091, + 3.403043746948242, + 3.4163620471954346, + 3.4564138650894165, + 3.5217223167419434, + 3.3419153690338135, + 3.4071494340896606, + 3.4867093563079834, + 3.4913322925567627, + 3.471901059150696, + 3.475080370903015, + 3.487777352333069, + 3.4717601537704468, + 3.46796715259552, + 3.524390459060669, + 3.4564850330352783, + 3.5299761295318604, + 3.535548448562622, + 3.5213963985443115, + 3.476629853248596, + 3.4829827547073364, + 3.5211294889450073, + 3.4952917098999023, + 3.42410147190094, + 3.470067024230957, + 3.3996392488479614, + 3.4548416137695312, + 3.579703450202942, + 3.2801095247268677, + 3.2997814416885376, + 3.5252243280410767, + 3.3782368898391724, + 3.3683922290802, + 3.4832394123077393, + 3.4147685766220093, + 3.349594235420227, + 3.3765493631362915, + 3.359038233757019, + 3.4105597734451294, + 3.431021213531494, + 3.395572304725647, + 3.4251832962036133, + 3.534596085548401, + 3.4603493213653564, + 3.359512209892273, + 3.3890496492385864, + 3.434372305870056, + 3.505436420440674, + 3.2300448417663574, + 3.5355273485183716, + 3.3328700065612793, + 3.383004665374756, + 3.348633646965027, + 3.3711514472961426, + 3.3112053871154785, + 3.5036020278930664, + 3.388237953186035, + 3.3512226343154907, + 3.4060628414154053, + 3.4298683404922485, + 3.288511276245117, + 3.4836246967315674, + 3.5236856937408447, + 3.386712908744812, + 3.3557159900665283, + 3.501689314842224, + 3.3745850324630737, + 3.39342200756073, + 3.4854958057403564, + 3.387757420539856, + 3.4464656114578247, + 3.3793710470199585, + 3.390943765640259, + 3.4230291843414307, + 3.3977383375167847, + 3.307376742362976, + 3.317682147026062, + 3.2900612354278564, + 3.487459421157837, + 3.3569990396499634, + 3.435261845588684, + 3.552346110343933, + 3.457302212715149, + 3.483385443687439, + 3.3705724477767944, + 3.397168755531311, + 3.4961615800857544, + 3.307496190071106, + 3.4056836366653442, + 3.370774030685425, + 3.40269136428833, + 3.463863253593445, + 3.404985547065735, + 3.447320342063904, + 3.5635966062545776, + 3.4910225868225098, + 3.4804004430770874, + 3.4100005626678467, + 3.45903742313385, + 3.42643141746521, + 3.549372673034668, + 3.4653899669647217, + 3.353666305541992, + 3.37154757976532, + 3.380896806716919, + 3.4873924255371094, + 3.3504579067230225, + 3.48051381111145, + 3.397234559059143, + 3.323536515235901, + 3.4845447540283203, + 3.37676739692688, + 3.394395112991333, + 3.4281346797943115, + 3.4352985620498657, + 3.4575542211532593, + 3.494913935661316, + 3.297554612159729, + 3.3564796447753906, + 3.417369246482849, + 3.4429352283477783, + 3.302910089492798, + 3.3926113843917847, + 3.4046382904052734, + 3.358651876449585, + 3.484888792037964, + 3.359411120414734, + 3.3941829204559326, + 3.4453272819519043, + 3.3978153467178345, + 3.4100072383880615, + 3.391700029373169, + 3.389965772628784, + 3.351758360862732, + 3.446449637413025, + 3.5307613611221313, + 3.4041818380355835, + 3.4151382446289062, + 3.5147838592529297, + 3.4543042182922363, + 3.430955171585083, + 3.326687216758728, + 3.465772271156311, + 3.4421095848083496 + ], + "val_steps": [ + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 37.15884349407581, + 36.07247865356619, + 35.144515937602314, + 34.244505221972, + 33.474171007983934, + 33.155867400783926, + 32.755262132718386, + 32.57977455852037, + 32.47445450233072, + 32.45808550762491 + ], + "best_ppl": 32.45808550762491, + "train_time_s": 9913.7885825634 +} \ No newline at end of file diff --git a/063_sparse_dispatch/shared_softmax_1plus2of5_s137_best.pt b/063_sparse_dispatch/shared_softmax_1plus2of5_s137_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd3d62449abc8ac2475c2a057d8b96efcb5acda4 --- /dev/null +++ b/063_sparse_dispatch/shared_softmax_1plus2of5_s137_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd675803b66d8bf9d148ad985db640b39fc65aa3aa2d72afcb01609758f57835 +size 341414820 diff --git a/063_sparse_dispatch/shared_softmax_1plus2of5_s137_results.json b/063_sparse_dispatch/shared_softmax_1plus2of5_s137_results.json new file mode 100644 index 0000000000000000000000000000000000000000..5109367e1583d06ae85b12cfff343d7702adeb68 --- /dev/null +++ b/063_sparse_dispatch/shared_softmax_1plus2of5_s137_results.json @@ -0,0 +1,1244 @@ +{ + "label": "shared_softmax_1plus2of5_s137", + "seed": 137, + "attention_mode": "sparse", + "routed_attn_type": "softmax", + "top_k_blocks": 2, + "num_thin_blocks": 5, + "num_shared_blocks": 1, + "total_params": 85293056, + "steps": [ + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 3.7975908517837524, + 3.7301357984542847, + 3.761266589164734, + 3.753808379173279, + 3.8484164476394653, + 3.757461190223694, + 3.736055612564087, + 3.740812063217163, + 3.735013008117676, + 3.7747050523757935, + 3.7797651290893555, + 3.7279250621795654, + 3.6794967651367188, + 3.7366186380386353, + 3.745791435241699, + 3.807852029800415, + 3.809494137763977, + 3.7086799144744873, + 3.6646097898483276, + 3.835037589073181, + 3.836900234222412, + 3.7321161031723022, + 3.846287250518799, + 3.756702423095703, + 3.7990728616714478, + 3.7094753980636597, + 3.762157917022705, + 3.7579210996627808, + 3.814616322517395, + 3.814478635787964, + 3.7212051153182983, + 3.677432656288147, + 3.7508301734924316, + 3.803544759750366, + 3.8559919595718384, + 3.8215506076812744, + 3.789250135421753, + 3.765315055847168, + 3.7641834020614624, + 3.8472920656204224, + 3.6314167976379395, + 3.741890549659729, + 3.637342929840088, + 3.778608798980713, + 3.775521993637085, + 3.7831943035125732, + 3.7063440084457397, + 3.844983220100403, + 3.852973461151123, + 3.823923110961914, + 3.839143395423889, + 3.7402454614639282, + 3.729805588722229, + 3.7002124786376953, + 3.74562406539917, + 3.6924320459365845, + 3.6980990171432495, + 3.8626595735549927, + 3.584112524986267, + 3.784016251564026, + 3.749390721321106, + 3.662333130836487, + 3.702446937561035, + 3.6989612579345703, + 3.6618287563323975, + 3.620550036430359, + 3.792686939239502, + 3.781956195831299, + 3.9012593030929565, + 3.889512062072754, + 3.740596890449524, + 3.6436911821365356, + 3.6589678525924683, + 3.8361133337020874, + 3.6808916330337524, + 3.761923670768738, + 3.7245341539382935, + 3.747377634048462, + 3.807379722595215, + 3.7504290342330933, + 3.551582098007202, + 3.666191577911377, + 3.6958017349243164, + 3.675804615020752, + 3.622006058692932, + 3.7616446018218994, + 3.7352957725524902, + 3.785843849182129, + 3.714940905570984, + 3.5799169540405273, + 3.5456000566482544, + 3.7152702808380127, + 3.768324613571167, + 3.85641872882843, + 3.7494558095932007, + 3.6437450647354126, + 3.6918365955352783, + 3.802944540977478, + 3.7710835933685303, + 3.7090444564819336, + 3.819976806640625, + 3.6634178161621094, + 3.7289122343063354, + 3.88814914226532, + 3.84882652759552, + 3.7500284910202026, + 3.6940656900405884, + 3.485509991645813, + 3.654901623725891, + 3.505782961845398, + 3.7541550397872925, + 3.6920955181121826, + 3.6498748064041138, + 3.7520443201065063, + 3.7480705976486206, + 3.6171300411224365, + 3.7007588148117065, + 3.715493083000183, + 3.616887331008911, + 3.705137014389038, + 3.6314642429351807, + 3.594167947769165, + 3.5962307453155518, + 3.8097875118255615, + 3.69995379447937, + 3.7515201568603516, + 3.5353312492370605, + 3.684958338737488, + 3.6705198287963867, + 3.601815342903137, + 3.519493579864502, + 3.4659162759780884, + 3.469617486000061, + 3.6150914430618286, + 3.7240357398986816, + 3.792175531387329, + 3.690554976463318, + 3.5791287422180176, + 3.628494381904602, + 3.4916772842407227, + 3.633202910423279, + 3.589402675628662, + 3.6025601625442505, + 3.569049835205078, + 3.515577554702759, + 3.569417715072632, + 3.5357627868652344, + 3.5007482767105103, + 3.598637342453003, + 3.5943727493286133, + 3.7316700220108032, + 3.5034700632095337, + 3.5770713090896606, + 3.5745608806610107, + 3.5331588983535767, + 3.591213583946228, + 3.6840038299560547, + 3.590444564819336, + 3.542853593826294, + 3.567659854888916, + 3.692060351371765, + 3.6314834356307983, + 3.4902697801589966, + 3.6527663469314575, + 3.5301607847213745, + 3.733170509338379, + 3.591602087020874, + 3.515843629837036, + 3.5632885694503784, + 3.679442882537842, + 3.7028218507766724, + 3.5472267866134644, + 3.683313250541687, + 3.6703693866729736, + 3.628785729408264, + 3.67348575592041, + 3.5175247192382812, + 3.643404245376587, + 3.621638059616089, + 3.6681177616119385, + 3.6837817430496216, + 3.681200861930847, + 3.6435688734054565, + 3.519810676574707, + 3.615163564682007, + 3.458023190498352, + 3.6521295309066772, + 3.6801247596740723, + 3.676377296447754, + 3.633094549179077, + 3.6527785062789917, + 3.5746251344680786, + 3.6551347970962524, + 3.707301378250122, + 3.5685060024261475, + 3.674931287765503, + 3.576985001564026, + 3.63736093044281, + 3.54171884059906, + 3.646854043006897, + 3.6154563426971436, + 3.6664116382598877, + 3.568432927131653, + 3.5446321964263916, + 3.5624910593032837, + 3.564754843711853, + 3.667320489883423, + 3.5695443153381348, + 3.5583938360214233, + 3.4727983474731445, + 3.6282697916030884, + 3.6387226581573486, + 3.619164228439331, + 3.6313165426254272, + 3.529019594192505, + 3.6755517721176147, + 3.3808642625808716, + 3.632899284362793, + 3.3927974700927734, + 3.499962091445923, + 3.7169458866119385, + 3.6190602779388428, + 3.5460468530654907, + 3.587334632873535, + 3.4703532457351685, + 3.600459933280945, + 3.3603079319000244, + 3.5913177728652954, + 3.5477374792099, + 3.5772026777267456, + 3.6718101501464844, + 3.7108463048934937, + 3.5307016372680664, + 3.569750428199768, + 3.641095757484436, + 3.5149558782577515, + 3.6402429342269897, + 3.4806015491485596, + 3.635839581489563, + 3.651107907295227, + 3.654148578643799, + 3.607413172721863, + 3.5455753803253174, + 3.487762689590454, + 3.5156562328338623, + 3.6813910007476807, + 3.575406074523926, + 3.685858964920044, + 3.6501224040985107, + 3.6556214094161987, + 3.5393306016921997, + 3.5704076290130615, + 3.5543606281280518, + 3.5462323427200317, + 3.5465961694717407, + 3.5327826738357544, + 3.633247494697571, + 3.495035409927368, + 3.4839329719543457, + 3.5312174558639526, + 3.5693657398223877, + 3.5518500804901123, + 3.50870943069458, + 3.4896745681762695, + 3.6750911474227905, + 3.5938422679901123, + 3.5898929834365845, + 3.4743021726608276, + 3.519731283187866, + 3.620553970336914, + 3.522654414176941, + 3.484848380088806, + 3.5137107372283936, + 3.4311317205429077, + 3.5368846654891968, + 3.419950246810913, + 3.573848247528076, + 3.600067615509033, + 3.5311756134033203, + 3.4289402961730957, + 3.5681744813919067, + 3.5297160148620605, + 3.531831979751587, + 3.6208584308624268, + 3.471658229827881, + 3.484176516532898, + 3.551692008972168, + 3.5098265409469604, + 3.6293766498565674, + 3.5873085260391235, + 3.601275682449341, + 3.4426318407058716, + 3.4735634326934814, + 3.5273276567459106, + 3.525161385536194, + 3.503636121749878, + 3.573759913444519, + 3.50439453125, + 3.512181878089905, + 3.5335065126419067, + 3.555940270423889, + 3.4077911376953125, + 3.631941080093384, + 3.3885852098464966, + 3.4003394842147827, + 3.562801718711853, + 3.594393253326416, + 3.4664305448532104, + 3.3415441513061523, + 3.509742856025696, + 3.568755030632019, + 3.5923861265182495, + 3.500836491584778, + 3.4378498792648315, + 3.489390015602112, + 3.413914442062378, + 3.4619163274765015, + 3.4967830181121826, + 3.423284411430359, + 3.489006519317627, + 3.500425100326538, + 3.5231218338012695, + 3.423506498336792, + 3.314966917037964, + 3.4969747066497803, + 3.33166241645813, + 3.547495126724243, + 3.5508599281311035, + 3.449802041053772, + 3.527969479560852, + 3.611414074897766, + 3.5524001121520996, + 3.4472780227661133, + 3.5076234340667725, + 3.5351967811584473, + 3.4710400104522705, + 3.5331451892852783, + 3.6268447637557983, + 3.6440069675445557, + 3.5776376724243164, + 3.4796292781829834, + 3.5316916704177856, + 3.443505048751831, + 3.548789381980896, + 3.4118294715881348, + 3.524917721748352, + 3.49139940738678, + 3.592089891433716, + 3.359334111213684, + 3.54110324382782, + 3.397619843482971, + 3.4983317852020264, + 3.5045498609542847, + 3.601523280143738, + 3.467759132385254, + 3.4971024990081787, + 3.468778610229492, + 3.4466397762298584, + 3.5590105056762695, + 3.5726420879364014, + 3.4113550186157227, + 3.342229962348938, + 3.4764270782470703, + 3.490393877029419, + 3.4583910703659058, + 3.5047380924224854, + 3.509934186935425, + 3.4080737829208374, + 3.5504320859909058, + 3.494781970977783, + 3.5247561931610107, + 3.4310070276260376, + 3.572764039039612, + 3.52864933013916, + 3.378200650215149, + 3.4968982934951782, + 3.5151731967926025, + 3.590795874595642, + 3.572613835334778, + 3.5005946159362793, + 3.476187825202942, + 3.449217677116394, + 3.4591269493103027, + 3.432083249092102, + 3.46061372756958, + 3.377468466758728, + 3.4564216136932373, + 3.367648720741272, + 3.3852806091308594, + 3.300488829612732, + 3.4229410886764526, + 3.4455617666244507, + 3.4601783752441406, + 3.43026340007782, + 3.339339852333069, + 3.3821003437042236, + 3.405618190765381, + 3.4151811599731445, + 3.315365195274353, + 3.367695927619934, + 3.3534477949142456, + 3.4688899517059326, + 3.4646995067596436, + 3.5341334342956543, + 3.4315621852874756, + 3.31160306930542, + 3.4385690689086914, + 3.433867573738098, + 3.352426052093506, + 3.387973189353943, + 3.4883052110671997, + 3.448954224586487, + 3.397101044654846, + 3.3967567682266235, + 3.3941017389297485, + 3.5820623636245728, + 3.4111393690109253, + 3.6077884435653687, + 3.4280096292495728, + 3.366872191429138, + 3.4475762844085693, + 3.459404468536377, + 3.539838433265686, + 3.3447905778884888, + 3.5527477264404297, + 3.3625327348709106, + 3.4315167665481567, + 3.4137136936187744, + 3.521307110786438, + 3.304227113723755, + 3.492941379547119, + 3.483128547668457, + 3.543595552444458, + 3.574081778526306, + 3.406911253929138, + 3.4908939599990845, + 3.453269124031067, + 3.429818630218506, + 3.298244595527649, + 3.4641356468200684, + 3.2499481439590454, + 3.3941720724105835, + 3.4253703355789185, + 3.4585230350494385, + 3.507864475250244, + 3.4957730770111084, + 3.529025077819824, + 3.577229619026184, + 3.479914903640747, + 3.5290404558181763, + 3.340277075767517, + 3.477020740509033, + 3.368487596511841, + 3.423038959503174, + 3.285338282585144, + 3.42309033870697, + 3.509195327758789, + 3.4117345809936523, + 3.479198455810547, + 3.4041099548339844, + 3.4048882722854614, + 3.47487211227417, + 3.522694706916809, + 3.361073136329651, + 3.556267261505127, + 3.518179416656494, + 3.538499116897583, + 3.3028903007507324, + 3.3747674226760864, + 3.4064072370529175, + 3.4097036123275757, + 3.353351354598999, + 3.4995920658111572, + 3.4673017263412476, + 3.4418660402297974, + 3.5059739351272583, + 3.318342089653015, + 3.504016637802124, + 3.5193623304367065, + 3.4642432928085327, + 3.4556868076324463, + 3.420403003692627, + 3.4855809211730957, + 3.3551806211471558, + 3.401688575744629, + 3.3765279054641724, + 3.362011671066284, + 3.46755051612854, + 3.435500979423523, + 3.4118019342422485, + 3.502179503440857, + 3.4327703714370728, + 3.3546693325042725, + 3.459604859352112, + 3.468916893005371, + 3.443753480911255, + 3.377006411552429, + 3.4429633617401123, + 3.4662522077560425, + 3.5198839902877808, + 3.3907761573791504, + 3.4692142009735107, + 3.4143978357315063, + 3.458670496940613, + 3.4117560386657715, + 3.286026358604431, + 3.4408196210861206, + 3.3282923698425293, + 3.4873056411743164, + 3.2930171489715576, + 3.388630509376526, + 3.4776272773742676, + 3.4439197778701782, + 3.4050556421279907, + 3.4272886514663696, + 3.4853334426879883, + 3.3441747426986694, + 3.3741517066955566, + 3.5030298233032227, + 3.2547961473464966, + 3.4090592861175537, + 3.4303154945373535, + 3.305390238761902, + 3.3796249628067017, + 3.4499645233154297, + 3.4616276025772095, + 3.329697012901306, + 3.4158674478530884, + 3.3863205909729004, + 3.4574127197265625, + 3.4968708753585815, + 3.3592653274536133, + 3.4206454753875732, + 3.387264609336853, + 3.4197237491607666, + 3.3913930654525757, + 3.271401047706604, + 3.356421113014221, + 3.209736943244934, + 3.396239995956421, + 3.52887499332428, + 3.4086331129074097, + 3.368772268295288, + 3.4472392797470093, + 3.338272452354431, + 3.373502492904663, + 3.2824113368988037, + 3.400740385055542, + 3.478199601173401, + 3.3859857320785522, + 3.4776453971862793, + 3.3800172805786133, + 3.447510242462158, + 3.3756574392318726, + 3.338657021522522, + 3.475361704826355, + 3.310557723045349, + 3.3997315168380737, + 3.4718176126480103, + 3.3852553367614746, + 3.265729308128357, + 3.3100757598876953, + 3.41592800617218, + 3.4946765899658203, + 3.431267023086548, + 3.417058229446411, + 3.345238208770752, + 3.394507646560669, + 3.399405002593994, + 3.4440505504608154, + 3.334700107574463, + 3.3602380752563477, + 3.4826107025146484, + 3.4009928703308105, + 3.3815388679504395, + 3.430790066719055, + 3.417680025100708, + 3.414226531982422, + 3.4116300344467163, + 3.498847484588623, + 3.4107357263565063, + 3.284797787666321, + 3.431666612625122, + 3.327583074569702, + 3.423146963119507, + 3.4994579553604126, + 3.339475154876709, + 3.4914129972457886, + 3.523780584335327, + 3.384685754776001, + 3.376194715499878, + 3.4672635793685913, + 3.31325900554657, + 3.35227108001709, + 3.438124179840088, + 3.4284950494766235, + 3.3728822469711304, + 3.445939302444458, + 3.3422056436538696, + 3.3297181129455566 + ], + "val_steps": [ + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 40.3037750541647, + 38.46429059378383, + 37.15405843601783, + 36.118786511893205, + 34.9315644971078, + 34.30418784622365, + 33.53232067099816, + 33.24746684652265, + 32.694924015716694, + 32.45617584667061, + 32.421156385537515, + 32.40805243256242 + ], + "best_ppl": 32.40805243256242, + "train_time_s": 12427.142152547836 +} \ No newline at end of file diff --git a/063_sparse_dispatch/shared_softmax_1plus2of5_s256_best.pt b/063_sparse_dispatch/shared_softmax_1plus2of5_s256_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..aaafe7026b94bfe9573c4ef05b862bcbba8b1f49 --- /dev/null +++ b/063_sparse_dispatch/shared_softmax_1plus2of5_s256_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b5ef4ce8190b1b8ba9c93d42c89fcf0846b1c15bb22d69a54f75984f9b2565e +size 341414820 diff --git a/063_sparse_dispatch/shared_softmax_1plus2of5_s256_results.json b/063_sparse_dispatch/shared_softmax_1plus2of5_s256_results.json new file mode 100644 index 0000000000000000000000000000000000000000..258e597c003b3484468781bd622e65435a2ebab3 --- /dev/null +++ b/063_sparse_dispatch/shared_softmax_1plus2of5_s256_results.json @@ -0,0 +1,1040 @@ +{ + "label": "shared_softmax_1plus2of5_s256", + "seed": 256, + "attention_mode": "sparse", + "routed_attn_type": "softmax", + "top_k_blocks": 2, + "num_thin_blocks": 5, + "num_shared_blocks": 1, + "total_params": 85293056, + "steps": [ + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 3.6139286756515503, + 3.564822196960449, + 3.5513455867767334, + 3.6561728715896606, + 3.6686679124832153, + 3.6693395376205444, + 3.5361403226852417, + 3.6762720346450806, + 3.652359962463379, + 3.673388957977295, + 3.5603299140930176, + 3.5871719121932983, + 3.5228241682052612, + 3.7338467836380005, + 3.682217597961426, + 3.603365898132324, + 3.567533254623413, + 3.6563574075698853, + 3.66309916973114, + 3.5757288932800293, + 3.6675108671188354, + 3.5996607542037964, + 3.5992170572280884, + 3.8042008876800537, + 3.5960798263549805, + 3.631595730781555, + 3.5789196491241455, + 3.551571846008301, + 3.595996618270874, + 3.5393584966659546, + 3.6580787897109985, + 3.5860763788223267, + 3.645536422729492, + 3.667800545692444, + 3.680486559867859, + 3.6626068353652954, + 3.65179979801178, + 3.6270185708999634, + 3.643920421600342, + 3.6369357109069824, + 3.6444854736328125, + 3.5172687768936157, + 3.6800652742385864, + 3.544533848762512, + 3.7163087129592896, + 3.5174078941345215, + 3.7233288288116455, + 3.5749995708465576, + 3.6403136253356934, + 3.691555976867676, + 3.6683837175369263, + 3.6608734130859375, + 3.667007803916931, + 3.5804810523986816, + 3.5869609117507935, + 3.552309274673462, + 3.592350721359253, + 3.674847364425659, + 3.6088286638259888, + 3.753693699836731, + 3.709653615951538, + 3.6113109588623047, + 3.5758450031280518, + 3.594469904899597, + 3.6983888149261475, + 3.6767852306365967, + 3.614983916282654, + 3.552048921585083, + 3.588937520980835, + 3.527063250541687, + 3.5852431058883667, + 3.6462894678115845, + 3.6078312397003174, + 3.509821057319641, + 3.589626908302307, + 3.4964901208877563, + 3.6114490032196045, + 3.5597469806671143, + 3.551209807395935, + 3.566055178642273, + 3.3567001819610596, + 3.6523842811584473, + 3.4724409580230713, + 3.605995774269104, + 3.624527931213379, + 3.571159839630127, + 3.5568714141845703, + 3.446228861808777, + 3.6121031045913696, + 3.4023314714431763, + 3.574854850769043, + 3.597166419029236, + 3.664646863937378, + 3.678259015083313, + 3.529382348060608, + 3.498476028442383, + 3.57179057598114, + 3.6115564107894897, + 3.549982190132141, + 3.52272367477417, + 3.8368929624557495, + 3.7562971115112305, + 3.689221501350403, + 3.7143439054489136, + 3.6087743043899536, + 3.7511844635009766, + 3.691062808036804, + 3.6396872997283936, + 3.7106237411499023, + 3.611129403114319, + 3.6421680450439453, + 3.6682735681533813, + 3.5310438871383667, + 3.7047349214553833, + 3.6619092226028442, + 3.720720648765564, + 3.56352698802948, + 3.698511004447937, + 3.6660807132720947, + 3.474899172782898, + 3.6278666257858276, + 3.5237478017807007, + 3.666640281677246, + 3.6534284353256226, + 3.603872060775757, + 3.6312978267669678, + 3.5527979135513306, + 3.482003927230835, + 3.510802388191223, + 3.4917978048324585, + 3.56147038936615, + 3.432500720024109, + 3.45205819606781, + 3.568400502204895, + 3.5609225034713745, + 3.5743640661239624, + 3.5815645456314087, + 3.6421350240707397, + 3.4603205919265747, + 3.486321210861206, + 3.630584478378296, + 3.565995216369629, + 3.456138849258423, + 3.4331859350204468, + 3.538335680961609, + 3.5361135005950928, + 3.4898635149002075, + 3.4780153036117554, + 3.548304319381714, + 3.585038423538208, + 3.584738254547119, + 3.5222911834716797, + 3.5504666566848755, + 3.5415265560150146, + 3.49010968208313, + 3.6272926330566406, + 3.470623254776001, + 3.502824306488037, + 3.496679902076721, + 3.4575170278549194, + 3.561548948287964, + 3.5194674730300903, + 3.427576780319214, + 3.5393646955490112, + 3.6068016290664673, + 3.4945279359817505, + 3.4941831827163696, + 3.5522239208221436, + 3.597838044166565, + 3.5540525913238525, + 3.5427201986312866, + 3.4656914472579956, + 3.499984622001648, + 3.6210111379623413, + 3.5526282787323, + 3.641533851623535, + 3.5873292684555054, + 3.565147280693054, + 3.5577059984207153, + 3.544412612915039, + 3.4595389366149902, + 3.575563907623291, + 3.5962544679641724, + 3.5512988567352295, + 3.541281819343567, + 3.546692967414856, + 3.5809298753738403, + 3.6426074504852295, + 3.5034044981002808, + 3.5831992626190186, + 3.5319658517837524, + 3.504828095436096, + 3.5573493242263794, + 3.5273640155792236, + 3.555483102798462, + 3.5324816703796387, + 3.5868314504623413, + 3.6243098974227905, + 3.571492910385132, + 3.383292317390442, + 3.5246741771698, + 3.5720221996307373, + 3.5577659606933594, + 3.4739893674850464, + 3.63768208026886, + 3.545847177505493, + 3.6021944284439087, + 3.4615418910980225, + 3.4803606271743774, + 3.599310874938965, + 3.5014253854751587, + 3.633537769317627, + 3.5665985345840454, + 3.560060739517212, + 3.516171097755432, + 3.5990532636642456, + 3.4947080612182617, + 3.5709415674209595, + 3.409012198448181, + 3.467111825942993, + 3.583667516708374, + 3.461452841758728, + 3.39157235622406, + 3.4678375720977783, + 3.5139421224594116, + 3.572359800338745, + 3.6215012073516846, + 3.5122815370559692, + 3.3929609060287476, + 3.664069175720215, + 3.4955971240997314, + 3.6661781072616577, + 3.5238773822784424, + 3.388503670692444, + 3.360305666923523, + 3.5716896057128906, + 3.485956311225891, + 3.5306588411331177, + 3.4870556592941284, + 3.577880382537842, + 3.471289277076721, + 3.3186641931533813, + 3.5487406253814697, + 3.5052621364593506, + 3.435567259788513, + 3.4544581174850464, + 3.4761292934417725, + 3.465989828109741, + 3.506397843360901, + 3.5185797214508057, + 3.577370524406433, + 3.4120160341262817, + 3.4748144149780273, + 3.50559401512146, + 3.4511817693710327, + 3.3820117712020874, + 3.489043712615967, + 3.340386152267456, + 3.414193630218506, + 3.3551955223083496, + 3.4556130170822144, + 3.4591633081436157, + 3.537256956100464, + 3.3870811462402344, + 3.4616893529891968, + 3.482573986053467, + 3.500360608100891, + 3.6156668663024902, + 3.4393173456192017, + 3.4257967472076416, + 3.5402438640594482, + 3.368922710418701, + 3.5508092641830444, + 3.4416409730911255, + 3.473939538002014, + 3.3748947381973267, + 3.3455941677093506, + 3.5320297479629517, + 3.398194670677185, + 3.3439050912857056, + 3.5267502069473267, + 3.6219911575317383, + 3.4663381576538086, + 3.3746339082717896, + 3.3960211277008057, + 3.411483645439148, + 3.405348539352417, + 3.5518299341201782, + 3.6127742528915405, + 3.364256501197815, + 3.4117531776428223, + 3.4130499362945557, + 3.3767762184143066, + 3.320313811302185, + 3.5209120512008667, + 3.5456970930099487, + 3.4765607118606567, + 3.502060294151306, + 3.408673048019409, + 3.5294137001037598, + 3.5013591051101685, + 3.454072952270508, + 3.5398396253585815, + 3.392304539680481, + 3.403326392173767, + 3.3853025436401367, + 3.4571832418441772, + 3.483029365539551, + 3.2766016721725464, + 3.35406231880188, + 3.487215042114258, + 3.516708731651306, + 3.4425967931747437, + 3.445489525794983, + 3.4932990074157715, + 3.3981415033340454, + 3.4984214305877686, + 3.4630234241485596, + 3.436562657356262, + 3.4202851057052612, + 3.4657726287841797, + 3.4656732082366943, + 3.4797505140304565, + 3.4735918045043945, + 3.3527075052261353, + 3.4080793857574463, + 3.4585548639297485, + 3.478497624397278, + 3.4463800191879272, + 3.539190888404846, + 3.525006413459778, + 3.4142030477523804, + 3.486333131790161, + 3.406217098236084, + 3.4724504947662354, + 3.327945590019226, + 3.4167938232421875, + 3.3512461185455322, + 3.4006199836730957, + 3.4389973878860474, + 3.5179662704467773, + 3.3563414812088013, + 3.544864296913147, + 3.390462636947632, + 3.3334890604019165, + 3.4255824089050293, + 3.5594778060913086, + 3.5509687662124634, + 3.3789279460906982, + 3.4279985427856445, + 3.3978768587112427, + 3.563710331916809, + 3.3762526512145996, + 3.343205451965332, + 3.537587881088257, + 3.4804208278656006, + 3.3491320610046387, + 3.3973599672317505, + 3.526912569999695, + 3.33281934261322, + 3.486328125, + 3.4369252920150757, + 3.4745407104492188, + 3.5173555612564087, + 3.460842728614807, + 3.3607559204101562, + 3.3336716890335083, + 3.435203433036804, + 3.53762948513031, + 3.4349939823150635, + 3.393891215324402, + 3.5019131898880005, + 3.3763140439987183, + 3.5682270526885986, + 3.436336040496826, + 3.371470808982849, + 3.289105176925659, + 3.455348491668701, + 3.392285108566284, + 3.37539005279541, + 3.469522476196289, + 3.311200976371765, + 3.2988821268081665, + 3.3339463472366333, + 3.3769750595092773, + 3.3634798526763916, + 3.4026342630386353, + 3.261156439781189, + 3.326251268386841, + 3.342133641242981, + 3.3823848962783813, + 3.478516697883606, + 3.3617023229599, + 3.46001398563385, + 3.372698187828064, + 3.2111778259277344, + 3.3909311294555664, + 3.270578384399414, + 3.4073396921157837, + 3.2722439765930176, + 3.4294328689575195, + 3.4738951921463013, + 3.377263903617859, + 3.4732409715652466, + 3.4277262687683105, + 3.458717703819275, + 3.3460140228271484, + 3.3939555883407593, + 3.5021497011184692, + 3.3815640211105347, + 3.3656357526779175, + 3.445565700531006, + 3.3356631994247437, + 3.3902876377105713, + 3.350213885307312, + 3.494278311729431, + 3.3510247468948364, + 3.371082305908203, + 3.429424285888672, + 3.413759231567383, + 3.448172092437744, + 3.3471285104751587, + 3.5192631483078003, + 3.3516159057617188, + 3.4436566829681396, + 3.3955858945846558, + 3.3176324367523193, + 3.3406407833099365, + 3.402479410171509, + 3.460821032524109, + 3.5363922119140625, + 3.2876490354537964, + 3.464179277420044, + 3.282936215400696, + 3.404146432876587, + 3.498607635498047, + 3.4294326305389404, + 3.4823555946350098, + 3.5046610832214355, + 3.4691576957702637, + 3.28176212310791, + 3.227392315864563, + 3.2881468534469604, + 3.4290847778320312, + 3.292625308036804, + 3.491361975669861, + 3.4468029737472534, + 3.491860866546631, + 3.3519952297210693, + 3.4701801538467407, + 3.3347328901290894, + 3.5272070169448853, + 3.460266947746277, + 3.468534231185913, + 3.361109733581543, + 3.323657274246216, + 3.411508083343506, + 3.387665271759033, + 3.3605711460113525, + 3.4632065296173096, + 3.4520297050476074, + 3.3836599588394165, + 3.3984991312026978, + 3.435736060142517, + 3.5412368774414062, + 3.4030314683914185, + 3.4030035734176636, + 3.4023306369781494, + 3.308967113494873, + 3.536448836326599, + 3.5050379037857056, + 3.4418030977249146, + 3.4389564990997314, + 3.420434832572937, + 3.367828130722046, + 3.45099937915802, + 3.348376989364624, + 3.3144699335098267, + 3.4983465671539307, + 3.29232120513916, + 3.426355004310608, + 3.4256335496902466, + 3.4473695755004883, + 3.4029654264450073, + 3.322973608970642, + 3.451107978820801, + 3.3729870319366455, + 3.313674569129944, + 3.3600451946258545, + 3.385237455368042, + 3.4368454217910767, + 3.3770651817321777, + 3.3593372106552124, + 3.3359298706054688, + 3.3487526178359985, + 3.3773739337921143, + 3.408285975456238, + 3.440983533859253, + 3.4205148220062256, + 3.3713066577911377 + ], + "val_steps": [ + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 37.11451475668667, + 36.139048733440674, + 34.92628010626397, + 34.05890116613372, + 33.328263055035364, + 32.96421632561909, + 32.4679047099242, + 32.34167418788146, + 32.22403308566965, + 32.187272198884344 + ], + "best_ppl": 32.187272198884344, + "train_time_s": 10073.08019733429 +} \ No newline at end of file diff --git a/063_sparse_dispatch/single_block_deltanet_best.pt b/063_sparse_dispatch/single_block_deltanet_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..826255f1d467a1398afe2e4421ab36ed4e0dd988 --- /dev/null +++ b/063_sparse_dispatch/single_block_deltanet_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69627fbf80f7465b77a80d37f522893e60c36303b6a47206420dcba99676f703 +size 175406404 diff --git a/063_sparse_dispatch/single_block_deltanet_d1024_best.pt b/063_sparse_dispatch/single_block_deltanet_d1024_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..762be9ef1a6e3b0b8cb9c6b334bfa209e7646db4 --- /dev/null +++ b/063_sparse_dispatch/single_block_deltanet_d1024_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3a013e4bea34f4d17ca4cb29b959d6a1659c6bdbd2bc83116d631adb43b9bd3 +size 636011834 diff --git a/063_sparse_dispatch/single_block_deltanet_d1024_results.json b/063_sparse_dispatch/single_block_deltanet_d1024_results.json new file mode 100644 index 0000000000000000000000000000000000000000..1f28a4fc11b39f36ec4e9ee734ef11aea0c09943 --- /dev/null +++ b/063_sparse_dispatch/single_block_deltanet_d1024_results.json @@ -0,0 +1,2057 @@ +{ + "label": "single_block_deltanet_d1024", + "seed": 42, + "attention_type": "deltanet", + "d_expert": 1024, + "total_params": 158984960, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 9.441194534301758, + 7.933443069458008, + 7.202243089675903, + 6.941619396209717, + 6.7596518993377686, + 6.467680931091309, + 6.323258399963379, + 6.196579933166504, + 6.016123294830322, + 5.948401212692261, + 5.829460382461548, + 5.770024061203003, + 5.558311223983765, + 5.542412996292114, + 5.481109619140625, + 5.390916585922241, + 5.299463987350464, + 5.243495464324951, + 5.282023668289185, + 5.161922931671143, + 4.98694920539856, + 5.07087779045105, + 5.0291664600372314, + 4.967472791671753, + 5.122082233428955, + 4.940332651138306, + 4.894984245300293, + 4.9127583503723145, + 4.813004732131958, + 4.763156175613403, + 4.801075220108032, + 4.826950550079346, + 4.747664928436279, + 4.77633810043335, + 4.674427509307861, + 4.786617279052734, + 4.612589120864868, + 4.625530004501343, + 4.647680759429932, + 4.666627883911133, + 4.562906503677368, + 4.615633487701416, + 4.520433187484741, + 4.342959403991699, + 4.5184221267700195, + 4.447348594665527, + 4.409886598587036, + 4.4565393924713135, + 4.456019401550293, + 4.343780755996704, + 4.4578588008880615, + 4.348638296127319, + 4.419095754623413, + 4.309273719787598, + 4.423097610473633, + 4.429887533187866, + 4.295227527618408, + 4.365715026855469, + 4.365703105926514, + 4.386266231536865, + 4.385082483291626, + 4.390299081802368, + 4.343928813934326, + 4.324118614196777, + 4.375619173049927, + 4.1732988357543945, + 4.238001346588135, + 4.240652799606323, + 4.3015594482421875, + 4.251695394515991, + 4.212055921554565, + 4.190514326095581, + 4.1152184009552, + 4.180185317993164, + 4.123560667037964, + 4.143439769744873, + 4.286000967025757, + 4.149010419845581, + 4.102220296859741, + 4.1106438636779785, + 4.177608251571655, + 4.0987138748168945, + 4.236621856689453, + 4.056572914123535, + 4.1836161613464355, + 4.251004695892334, + 4.137420892715454, + 4.094475269317627, + 4.079790830612183, + 4.1127917766571045, + 4.125061750411987, + 4.138470649719238, + 4.052344083786011, + 4.061721086502075, + 4.04686975479126, + 3.908336639404297, + 4.049590349197388, + 4.016438841819763, + 4.0032700300216675, + 4.08280873298645, + 4.1223883628845215, + 4.072943568229675, + 4.008711814880371, + 4.08024263381958, + 3.911435604095459, + 3.967969059944153, + 4.076989412307739, + 4.1067280769348145, + 3.890181541442871, + 4.093204855918884, + 4.081700325012207, + 4.0200464725494385, + 3.9913073778152466, + 3.9810636043548584, + 4.107130527496338, + 3.973360776901245, + 3.889490008354187, + 3.941973090171814, + 4.0276594161987305, + 3.88630747795105, + 3.940511107444763, + 4.039770126342773, + 3.8965669870376587, + 4.001201868057251, + 3.965373992919922, + 4.078036904335022, + 3.92662250995636, + 3.850066304206848, + 3.89709210395813, + 3.9091572761535645, + 3.89085853099823, + 3.9184188842773438, + 3.8662365674972534, + 3.863369941711426, + 3.9180309772491455, + 3.881345748901367, + 3.8551286458969116, + 3.831861734390259, + 3.8919270038604736, + 3.6769131422042847, + 3.970818877220154, + 3.7897861003875732, + 3.8237541913986206, + 3.9092798233032227, + 3.747146964073181, + 3.740718126296997, + 3.853326678276062, + 3.92792010307312, + 3.824026942253113, + 3.887296199798584, + 3.8162697553634644, + 3.8608129024505615, + 3.8342427015304565, + 3.9231032133102417, + 3.8459781408309937, + 3.867903232574463, + 3.8710503578186035, + 3.9129750728607178, + 3.8583637475967407, + 3.7969855070114136, + 3.8733850717544556, + 3.896308422088623, + 3.837049722671509, + 3.8324568271636963, + 3.951678991317749, + 3.932835817337036, + 3.7703802585601807, + 3.8675999641418457, + 3.8198697566986084, + 3.8645763397216797, + 3.794243335723877, + 3.751132845878601, + 3.807586193084717, + 3.8197803497314453, + 3.7277556657791138, + 3.68441104888916, + 3.9005370140075684, + 3.795928716659546, + 3.8674118518829346, + 3.734315276145935, + 3.911805748939514, + 3.7265968322753906, + 3.798113226890564, + 3.8092901706695557, + 3.7096368074417114, + 3.7868412733078003, + 3.8553223609924316, + 3.758013963699341, + 3.8665324449539185, + 3.7414785623550415, + 3.781942367553711, + 3.7734698057174683, + 3.7429898977279663, + 3.8068939447402954, + 3.8405107259750366, + 3.75906503200531, + 3.859291911125183, + 3.899956703186035, + 3.8244510889053345, + 3.699654698371887, + 3.760887861251831, + 3.752305507659912, + 3.886402726173401, + 3.645959734916687, + 3.6351802349090576, + 3.5942662954330444, + 3.9563827514648438, + 3.7858952283859253, + 3.8042773008346558, + 3.7150968313217163, + 3.803526520729065, + 3.7224483489990234, + 3.8246997594833374, + 3.834257483482361, + 3.7827483415603638, + 3.656190514564514, + 3.7407734394073486, + 3.5938223600387573, + 3.9002803564071655, + 3.8803926706314087, + 3.7500057220458984, + 3.690161108970642, + 3.7734264135360718, + 3.7900859117507935, + 3.717552661895752, + 3.7383921146392822, + 3.7680643796920776, + 3.72342312335968, + 3.8129923343658447, + 3.88036847114563, + 3.8086501359939575, + 3.75359570980072, + 3.6876665353775024, + 3.66921603679657, + 3.7961636781692505, + 3.832047462463379, + 3.7310186624526978, + 3.7789844274520874, + 3.778975009918213, + 3.7849661111831665, + 3.6922645568847656, + 3.741192102432251, + 3.7345603704452515, + 3.6401432752609253, + 3.6047455072402954, + 3.7514246702194214, + 3.740574836730957, + 3.6553550958633423, + 3.6887634992599487, + 3.648791193962097, + 3.696069359779358, + 3.8521264791488647, + 3.604073405265808, + 3.710410952568054, + 3.585644006729126, + 3.5760035514831543, + 3.579104423522949, + 3.616130828857422, + 3.577275276184082, + 3.636904001235962, + 3.5341997146606445, + 3.483621597290039, + 3.43295955657959, + 3.537109136581421, + 3.538241744041443, + 3.669822573661804, + 3.6636931896209717, + 3.61113703250885, + 3.540234088897705, + 3.5927361249923706, + 3.5926077365875244, + 3.688621163368225, + 3.6669691801071167, + 3.7137149572372437, + 3.6561834812164307, + 3.623073935508728, + 3.6709084510803223, + 3.5340813398361206, + 3.4625720977783203, + 3.686166286468506, + 3.6230698823928833, + 3.610043168067932, + 3.6144105195999146, + 3.5564541816711426, + 3.5058388710021973, + 3.581457018852234, + 3.5558438301086426, + 3.4759254455566406, + 3.615372657775879, + 3.5048667192459106, + 3.5952893495559692, + 3.5008679628372192, + 3.6458280086517334, + 3.5666964054107666, + 3.69300639629364, + 3.513121008872986, + 3.580873966217041, + 3.6422231197357178, + 3.518296241760254, + 3.5558764934539795, + 3.63676118850708, + 3.527003526687622, + 3.6072405576705933, + 3.471741795539856, + 3.491933226585388, + 3.6147828102111816, + 3.603295087814331, + 3.483695864677429, + 3.518795609474182, + 3.5605990886688232, + 3.6168686151504517, + 3.583265542984009, + 3.5970211029052734, + 3.643500804901123, + 3.644758701324463, + 3.6805078983306885, + 3.571773886680603, + 3.63030207157135, + 3.5652517080307007, + 3.4673839807510376, + 3.6365091800689697, + 3.557229161262512, + 3.487233877182007, + 3.5237410068511963, + 3.589851975440979, + 3.5417481660842896, + 3.559935212135315, + 3.6170310974121094, + 3.637959122657776, + 3.685092568397522, + 3.570498466491699, + 3.6735297441482544, + 3.5735623836517334, + 3.5337769985198975, + 3.625627279281616, + 3.5748857259750366, + 3.516817808151245, + 3.455453038215637, + 3.6492279767990112, + 3.698225736618042, + 3.679876685142517, + 3.7151254415512085, + 3.7310097217559814, + 3.683109760284424, + 3.550729513168335, + 3.636681079864502, + 3.6752673387527466, + 3.6190322637557983, + 3.6109694242477417, + 3.6370551586151123, + 3.403395891189575, + 3.6549456119537354, + 3.5137709379196167, + 3.574335217475891, + 3.5986552238464355, + 3.6486719846725464, + 3.5552845001220703, + 3.7072486877441406, + 3.4706438779830933, + 3.7217646837234497, + 3.606058359146118, + 3.581726551055908, + 3.5333774089813232, + 3.5998833179473877, + 3.4916213750839233, + 3.568041682243347, + 3.414697766304016, + 3.4685816764831543, + 3.6180535554885864, + 3.568359375, + 3.5877078771591187, + 3.6017571687698364, + 3.574783205986023, + 3.5901166200637817, + 3.5246522426605225, + 3.4771289825439453, + 3.565490484237671, + 3.601695418357849, + 3.4637656211853027, + 3.4051172733306885, + 3.407438278198242, + 3.3788939714431763, + 3.3975547552108765, + 3.452821373939514, + 3.346463680267334, + 3.5347381830215454, + 3.4413968324661255, + 3.3867701292037964, + 3.2617838382720947, + 3.386319160461426, + 3.4803247451782227, + 3.4566571712493896, + 3.288342833518982, + 3.5924476385116577, + 3.185446619987488, + 3.551596522331238, + 3.410515785217285, + 3.5293588638305664, + 3.465299963951111, + 3.455761432647705, + 3.441592812538147, + 3.3949209451675415, + 3.3414928913116455, + 3.3877192735671997, + 3.3374969959259033, + 3.303479552268982, + 3.408921480178833, + 3.4558217525482178, + 3.3843278884887695, + 3.4344695806503296, + 3.560470223426819, + 3.3973453044891357, + 3.4009616374969482, + 3.484461545944214, + 3.4129384756088257, + 3.3815380334854126, + 3.3395520448684692, + 3.3132811784744263, + 3.311589241027832, + 3.4047257900238037, + 3.485808491706848, + 3.4211004972457886, + 3.4864253997802734, + 3.3668709993362427, + 3.4585376977920532, + 3.3646010160446167, + 3.403785824775696, + 3.464330315589905, + 3.395800232887268, + 3.4141942262649536, + 3.3719297647476196, + 3.469270348548889, + 3.2826679944992065, + 3.4141918420791626, + 3.418137311935425, + 3.4228904247283936, + 3.3520020246505737, + 3.423614501953125, + 3.4156559705734253, + 3.3114659786224365, + 3.4465352296829224, + 3.360669255256653, + 3.455850839614868, + 3.382319927215576, + 3.3760461807250977, + 3.3709352016448975, + 3.5229934453964233, + 3.424422860145569, + 3.549954652786255, + 3.480764150619507, + 3.3928956985473633, + 3.458194136619568, + 3.4707576036453247, + 3.4394086599349976, + 3.5421990156173706, + 3.491618275642395, + 3.372188448905945, + 3.366454243659973, + 3.398768663406372, + 3.4765655994415283, + 3.516991972923279, + 3.4216076135635376, + 3.423837661743164, + 3.3992836475372314, + 3.356876850128174, + 3.38007652759552, + 3.393303871154785, + 3.3270699977874756, + 3.4896328449249268, + 3.4725464582443237, + 3.3271985054016113, + 3.5201727151870728, + 3.4512137174606323, + 3.5203665494918823, + 3.340600848197937, + 3.466023802757263, + 3.4904178380966187, + 3.386704206466675, + 3.384313702583313, + 3.445882797241211, + 3.4511479139328003, + 3.4399224519729614, + 3.478604555130005, + 3.4457653760910034, + 3.51553475856781, + 3.38865864276886, + 3.395672559738159, + 3.3025840520858765, + 3.3669353723526, + 3.285555124282837, + 3.3895533084869385, + 3.2975893020629883, + 3.442480206489563, + 3.365136742591858, + 3.48742139339447, + 3.402677536010742, + 3.4072163105010986, + 3.4558775424957275, + 3.47994065284729, + 3.5029529333114624, + 3.3960925340652466, + 3.4701279401779175, + 3.438959002494812, + 3.49176025390625, + 3.597607970237732, + 3.4376590251922607, + 3.381365418434143, + 3.1203070878982544, + 3.2086169719696045, + 3.322957992553711, + 3.4196301698684692, + 3.4246411323547363, + 3.2650848627090454, + 3.380413293838501, + 3.2461880445480347, + 3.195039987564087, + 3.2951414585113525, + 3.1487045288085938, + 3.092039704322815, + 3.168151021003723, + 3.290979266166687, + 3.3480160236358643, + 3.2263340950012207, + 3.347996950149536, + 3.3212896585464478, + 3.383515238761902, + 3.331233501434326, + 3.3347188234329224, + 3.1955302953720093, + 3.272647976875305, + 3.414863109588623, + 3.313363194465637, + 3.362542986869812, + 3.0552375316619873, + 3.2424439191818237, + 3.304333448410034, + 3.300650954246521, + 3.3387486934661865, + 3.30116868019104, + 3.131225109100342, + 3.1048614978790283, + 3.2560852766036987, + 3.174112915992737, + 3.394514799118042, + 3.2896082401275635, + 3.3071391582489014, + 3.39039146900177, + 3.3701086044311523, + 3.2804486751556396, + 3.3178634643554688, + 3.288488984107971, + 3.3982770442962646, + 3.1353893280029297, + 3.3473228216171265, + 3.221498727798462, + 3.322996497154236, + 3.3431378602981567, + 3.323641300201416, + 3.3540596961975098, + 3.2629120349884033, + 3.095208168029785, + 3.3488447666168213, + 3.3442412614822388, + 3.30949866771698, + 3.358530879020691, + 3.2542113065719604, + 3.1841681003570557, + 3.3840469121932983, + 3.2193719148635864, + 3.16725754737854, + 3.2272056341171265, + 3.290015459060669, + 3.3476128578186035, + 3.2144428491592407, + 3.3539716005325317, + 3.3690741062164307, + 3.2695356607437134, + 3.2482314109802246, + 3.3310115337371826, + 3.2606420516967773, + 3.156588912010193, + 3.291528344154358, + 3.157717227935791, + 3.229349374771118, + 3.343589425086975, + 3.3574178218841553, + 3.2335453033447266, + 3.088212490081787, + 3.320304751396179, + 3.28389835357666, + 3.274657964706421, + 3.222601532936096, + 3.2572420835494995, + 3.2014856338500977, + 3.3036073446273804, + 3.1674834489822388, + 3.409790873527527, + 3.3486454486846924, + 3.182699203491211, + 3.319581627845764, + 3.2011687755584717, + 3.3655104637145996, + 3.41057813167572, + 3.234405517578125, + 3.329830765724182, + 3.2092604637145996, + 3.3082977533340454, + 3.2292189598083496, + 3.2446106672286987, + 3.3090147972106934, + 3.2101436853408813, + 3.29221248626709, + 3.2244335412979126, + 3.27782666683197, + 3.074887275695801, + 3.286861300468445, + 3.27061665058136, + 3.20496666431427, + 3.259763717651367, + 3.3469479084014893, + 3.341874599456787, + 3.2914087772369385, + 3.2313427925109863, + 3.261439085006714, + 3.2266271114349365, + 3.3299388885498047, + 3.3214904069900513, + 3.3565303087234497, + 3.2564064264297485, + 3.292073965072632, + 3.261255621910095, + 3.1288208961486816, + 3.117754101753235, + 3.1455130577087402, + 2.991409420967102, + 3.1408934593200684, + 3.111838698387146, + 3.1088725328445435, + 3.1903316974639893, + 3.10725998878479, + 3.241766333580017, + 3.2497787475585938, + 3.3217129707336426, + 3.0904664993286133, + 3.312910795211792, + 3.1138442754745483, + 2.9741368293762207, + 3.270134449005127, + 2.9233927726745605, + 3.051644206047058, + 3.2325135469436646, + 3.1526741981506348, + 3.227070927619934, + 3.176130175590515, + 3.1718764305114746, + 3.17825186252594, + 3.2242883443832397, + 3.1178797483444214, + 3.0988494157791138, + 2.962613344192505, + 3.1988037824630737, + 3.1738240718841553, + 3.0101590156555176, + 3.074531316757202, + 3.174954414367676, + 3.153780460357666, + 3.1571741104125977, + 3.126607656478882, + 3.112839460372925, + 3.1657975912094116, + 3.187814950942993, + 3.1756783723831177, + 3.200363874435425, + 3.1572128534317017, + 3.099084258079529, + 3.1067861318588257, + 3.242748498916626, + 3.232401132583618, + 3.22625732421875, + 3.144346237182617, + 3.1969581842422485, + 3.1559882164001465, + 3.069933533668518, + 3.1274735927581787, + 3.117428779602051, + 3.0743271112442017, + 3.1078771352767944, + 3.209242582321167, + 3.122663140296936, + 3.091658353805542, + 3.033542275428772, + 3.139016032218933, + 3.051363229751587, + 3.145002007484436, + 3.1580278873443604, + 3.1920512914657593, + 3.0215624570846558, + 3.005703091621399, + 3.112545967102051, + 3.164561152458191, + 3.1627256870269775, + 3.0563511848449707, + 3.0532147884368896, + 3.22655987739563, + 3.156064748764038, + 3.150635242462158, + 3.1131352186203003, + 3.113579273223877, + 3.1191970109939575, + 3.1086071729660034, + 3.167325735092163, + 3.0934433937072754, + 3.1982786655426025, + 3.173852324485779, + 2.994867444038391, + 3.143771171569824, + 3.1431132555007935, + 3.1012773513793945, + 3.1872012615203857, + 3.069075584411621, + 3.135877251625061, + 3.2719870805740356, + 3.0688101053237915, + 3.1280930042266846, + 3.0795223712921143, + 3.361398458480835, + 3.149059772491455, + 3.084301471710205, + 3.2449796199798584, + 3.1712255477905273, + 3.1727406978607178, + 3.1666442155838013, + 3.1738070249557495, + 3.1638063192367554, + 3.1439327001571655, + 3.116500735282898, + 3.1534037590026855, + 3.084177851676941, + 3.164637565612793, + 3.1126015186309814, + 3.0833077430725098, + 3.100751519203186, + 3.1471937894821167, + 3.1939674615859985, + 3.0635602474212646, + 3.155782103538513, + 3.126296877861023, + 3.153210401535034, + 3.100607752799988, + 3.2163201570510864, + 3.1264142990112305, + 3.2486729621887207, + 3.245413661003113, + 3.2404470443725586, + 3.2363550662994385, + 3.1001193523406982, + 3.0769059658050537, + 3.193349838256836, + 3.065327763557434, + 2.9499433040618896, + 3.093143343925476, + 3.0716004371643066, + 3.0952342748641968, + 3.09187912940979, + 2.905077338218689, + 3.036065101623535, + 3.0286929607391357, + 3.0174657106399536, + 2.9544023275375366, + 2.9379870891571045, + 2.8818405866622925, + 3.0062440633773804, + 2.9384682178497314, + 3.1570751667022705, + 3.099317193031311, + 3.0275992155075073, + 3.048185348510742, + 3.028981328010559, + 3.048768162727356, + 2.994906783103943, + 2.95372998714447, + 3.0796501636505127, + 3.000985026359558, + 2.868029832839966, + 3.043113350868225, + 3.088267683982849, + 2.9616734981536865, + 3.084057331085205, + 3.0465270280838013, + 3.00716233253479, + 2.9830809831619263, + 2.9519253969192505, + 2.951107978820801, + 3.0738638639450073, + 3.121158242225647, + 2.9923754930496216, + 3.1059972047805786, + 3.091761589050293, + 3.088719129562378, + 3.1413605213165283, + 2.914005160331726, + 3.0477453470230103, + 3.07954478263855, + 3.104109048843384, + 3.0545941591262817, + 2.9984312057495117, + 3.163705348968506, + 3.074147582054138, + 3.170159339904785, + 2.9716631174087524, + 2.9961016178131104, + 3.064895749092102, + 3.081158995628357, + 2.9811023473739624, + 3.0920597314834595, + 3.087653160095215, + 3.049172282218933, + 2.9721519947052, + 3.0638883113861084, + 2.9997209310531616, + 2.954139828681946, + 2.954690933227539, + 3.053519606590271, + 3.061469793319702, + 3.1348602771759033, + 3.092962384223938, + 3.0210635662078857, + 3.07537043094635, + 3.10610032081604, + 2.9893596172332764, + 2.9847848415374756, + 2.9326964616775513, + 3.0428253412246704, + 3.040286064147949, + 2.935806393623352, + 2.981267213821411, + 2.913454055786133, + 3.2079612016677856, + 3.0704870223999023, + 3.0838406085968018, + 2.846819043159485, + 3.0953755378723145, + 3.0552613735198975, + 3.024461507797241, + 3.011798620223999, + 3.0577088594436646, + 3.1016159057617188, + 3.0067349672317505, + 3.040288209915161, + 3.0465760231018066, + 2.991999626159668, + 3.050067901611328, + 3.019975185394287, + 3.1181617975234985, + 3.0510443449020386, + 3.0784873962402344, + 2.942264437675476, + 3.11913526058197, + 3.02625834941864, + 3.082035541534424, + 3.189604163169861, + 2.990169644355774, + 3.007034182548523, + 3.044898271560669, + 3.0586124658584595, + 3.0413761138916016, + 2.9202823638916016, + 3.103692889213562, + 2.981856107711792, + 3.0795161724090576, + 3.1092649698257446, + 2.982102870941162, + 3.143343448638916, + 2.97349750995636, + 3.082936406135559, + 3.139275074005127, + 3.109671950340271, + 2.949470281600952, + 2.8815141916275024, + 3.099013924598694, + 2.890159249305725, + 3.0183956623077393, + 3.030442237854004, + 3.0357078313827515, + 2.997027039527893, + 2.9715158939361572, + 2.976630449295044, + 2.946263313293457, + 2.8726603984832764, + 2.8599196672439575, + 2.687881827354431, + 2.9500635862350464, + 2.9629387855529785, + 2.9713833332061768, + 2.9178872108459473, + 3.033493995666504, + 3.027716279029846, + 2.917633295059204, + 3.024730086326599, + 2.9539339542388916, + 2.9490355253219604, + 3.0028998851776123, + 3.0282636880874634, + 2.91374671459198, + 3.0350934267044067, + 2.920055866241455, + 2.9221349954605103, + 2.995439648628235, + 2.9352437257766724, + 2.8319361209869385, + 3.078695774078369, + 2.885285496711731, + 2.9608665704727173, + 3.022103190422058, + 3.0119986534118652, + 3.063531994819641, + 3.0874253511428833, + 2.9092857837677, + 2.9492639303207397, + 2.948055148124695, + 3.0651379823684692, + 2.9899935722351074, + 2.9848166704177856, + 3.0003045797348022, + 3.0406912565231323, + 3.01247239112854, + 3.027072787284851, + 3.074270486831665, + 2.937234878540039, + 2.871590495109558, + 2.8855546712875366, + 2.852136015892029, + 3.0183463096618652, + 3.0155686140060425, + 3.2068450450897217, + 2.953763246536255, + 2.9582271575927734, + 2.9655091762542725, + 3.003245711326599, + 2.9363409280776978, + 2.8884034156799316, + 2.9685604572296143, + 2.9459714889526367, + 3.0311769247055054, + 2.956712245941162, + 3.050840735435486, + 2.8366000652313232, + 3.008135437965393, + 2.9120672941207886, + 2.995321035385132, + 3.012014150619507, + 3.057630777359009, + 2.9984018802642822, + 2.8971850872039795, + 2.9502590894699097, + 3.0106234550476074, + 3.016491651535034, + 3.093169689178467, + 3.018316626548767, + 2.9688326120376587, + 2.9986050128936768, + 2.971876382827759, + 2.947181224822998, + 3.0359424352645874, + 2.9338232278823853, + 2.8127694129943848, + 2.9612350463867188, + 3.0704349279403687, + 2.9939866065979004, + 3.0182723999023438, + 3.077737808227539, + 2.9719398021698, + 2.9389601945877075, + 2.979018211364746, + 3.058683753013611, + 2.844109535217285, + 2.9743382930755615, + 2.9767428636550903, + 2.9317729473114014, + 3.1164900064468384, + 3.0128839015960693, + 2.9009019136428833, + 2.9989144802093506, + 2.8534218072891235, + 2.994154930114746, + 2.890394926071167, + 2.935548424720764, + 2.953291654586792, + 2.9813976287841797, + 2.975316882133484, + 2.898871660232544, + 3.0802069902420044, + 2.9653836488723755, + 2.9704030752182007, + 2.9240976572036743, + 2.9301952123641968, + 3.12628972530365, + 3.0155893564224243, + 2.9788230657577515, + 2.994244337081909 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 77.10607524684075, + 53.69647532432713, + 45.75389424962399, + 41.39845471595954, + 38.28388364275797, + 36.19725217000984, + 34.65698458499944, + 33.498743349220085, + 32.28977269597786, + 31.056655846744647, + 30.677100609066347, + 29.51525700551607, + 29.166684401563735, + 28.57149743464807, + 27.98644463296729, + 28.0665198474802, + 27.77670612646945, + 27.772786138555162, + 27.718155271357233, + 27.70374754141318 + ], + "best_ppl": 27.70374754141318, + "train_time_s": 20830.89481163025 +} \ No newline at end of file diff --git a/063_sparse_dispatch/single_block_deltanet_d128_best.pt b/063_sparse_dispatch/single_block_deltanet_d128_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..93a27ba11cda68b2412868b80ab5eb07051c18bf --- /dev/null +++ b/063_sparse_dispatch/single_block_deltanet_d128_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa30ba562c227fdc7ee0bba42100fccd9b4afa473f79bf17c54a26c609db8b3e +size 146464593 diff --git a/063_sparse_dispatch/single_block_deltanet_d128_results.json b/063_sparse_dispatch/single_block_deltanet_d128_results.json new file mode 100644 index 0000000000000000000000000000000000000000..adcca13ab71ba5f42d6321263a5aca469d5e7ec5 --- /dev/null +++ b/063_sparse_dispatch/single_block_deltanet_d128_results.json @@ -0,0 +1,2057 @@ +{ + "label": "single_block_deltanet_d128", + "seed": 42, + "attention_type": "deltanet", + "d_expert": 128, + "total_params": 36598304, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.180683135986328, + 18.398048400878906, + 14.050098419189453, + 9.06129264831543, + 8.514569282531738, + 7.876091480255127, + 7.451539516448975, + 7.375479221343994, + 7.235713005065918, + 7.233298301696777, + 7.201592445373535, + 7.1335577964782715, + 7.076144218444824, + 7.006590366363525, + 7.015264987945557, + 6.846407413482666, + 6.83905029296875, + 6.70442008972168, + 6.595893383026123, + 6.55025577545166, + 6.471091270446777, + 6.463921070098877, + 6.186944484710693, + 6.143056869506836, + 6.152966499328613, + 6.0458903312683105, + 5.978487968444824, + 5.986639976501465, + 5.911312580108643, + 5.920792102813721, + 5.8297553062438965, + 5.665633201599121, + 5.741907596588135, + 5.663278579711914, + 5.64361572265625, + 5.641183376312256, + 5.547091484069824, + 5.527440071105957, + 5.512636661529541, + 5.565571308135986, + 5.4043803215026855, + 5.381940841674805, + 5.312867641448975, + 5.239109516143799, + 5.277046203613281, + 5.268712520599365, + 5.263983726501465, + 5.193672180175781, + 5.249255657196045, + 5.285642623901367, + 5.287176609039307, + 5.3511128425598145, + 5.2134809494018555, + 5.2042460441589355, + 5.172342300415039, + 5.088000774383545, + 5.113166809082031, + 5.151971340179443, + 5.062560558319092, + 5.230915069580078, + 5.087906360626221, + 5.216564178466797, + 5.083785057067871, + 5.054800987243652, + 5.136850357055664, + 5.024641036987305, + 4.989201545715332, + 5.062197685241699, + 4.985374450683594, + 5.005061626434326, + 5.043399333953857, + 5.144468307495117, + 4.9636921882629395, + 4.996500492095947, + 4.782869338989258, + 4.975974082946777, + 4.863922595977783, + 4.806883335113525, + 4.954517841339111, + 4.977800369262695, + 5.07410192489624, + 5.0066680908203125, + 4.929844856262207, + 4.9882049560546875, + 4.888546466827393, + 4.854722499847412, + 5.000518798828125, + 4.917565822601318, + 4.76135778427124, + 4.9193525314331055, + 4.938100814819336, + 4.909731864929199, + 4.875566005706787, + 4.830265045166016, + 4.854340076446533, + 4.729918956756592, + 4.854221820831299, + 4.801285266876221, + 4.820751667022705, + 4.809510231018066, + 4.795724868774414, + 4.780104637145996, + 4.61081600189209, + 4.75101375579834, + 4.778407096862793, + 4.904709815979004, + 4.697776794433594, + 4.911148548126221, + 4.769454002380371, + 4.664130687713623, + 4.69916296005249, + 4.778347015380859, + 4.83270788192749, + 4.663784503936768, + 4.6925554275512695, + 4.5962233543396, + 4.802492618560791, + 4.583486557006836, + 4.723672866821289, + 4.793076992034912, + 4.708590507507324, + 4.620357036590576, + 4.619991779327393, + 4.837815761566162, + 4.639176845550537, + 4.848333358764648, + 4.638319492340088, + 4.572720050811768, + 4.627636432647705, + 4.503396987915039, + 4.498851776123047, + 4.678614139556885, + 4.53713321685791, + 4.6662821769714355, + 4.70197057723999, + 4.583554267883301, + 4.613733768463135, + 4.604947566986084, + 4.676351070404053, + 4.5582804679870605, + 4.645122051239014, + 4.567746639251709, + 4.593481063842773, + 4.6483049392700195, + 4.541985511779785, + 4.465137004852295, + 4.434932708740234, + 4.535505771636963, + 4.444834232330322, + 4.5877580642700195, + 4.638883113861084, + 4.62504243850708, + 4.5772175788879395, + 4.459622859954834, + 4.55711555480957, + 4.639627933502197, + 4.562333106994629, + 4.578879356384277, + 4.53696870803833, + 4.563939571380615, + 4.515190601348877, + 4.60479736328125, + 4.584894180297852, + 4.6111884117126465, + 4.441951751708984, + 4.532848358154297, + 4.623164653778076, + 4.540895462036133, + 4.399674892425537, + 4.493044853210449, + 4.507217884063721, + 4.444229602813721, + 4.367716312408447, + 4.400421142578125, + 4.440896987915039, + 4.611143112182617, + 4.565910816192627, + 4.449689865112305, + 4.454073429107666, + 4.431896209716797, + 4.457217216491699, + 4.4544548988342285, + 4.5206732749938965, + 4.435312271118164, + 4.414412975311279, + 4.500269412994385, + 4.467385292053223, + 4.5148797035217285, + 4.369626522064209, + 4.462527751922607, + 4.353686809539795, + 4.430938243865967, + 4.557324409484863, + 4.493641376495361, + 4.455135822296143, + 4.456932544708252, + 4.425466537475586, + 4.485991477966309, + 4.441559314727783, + 4.325565814971924, + 4.341247081756592, + 4.415860176086426, + 4.469194412231445, + 4.491121292114258, + 4.406212329864502, + 4.3185954093933105, + 4.499756813049316, + 4.477635860443115, + 4.5263237953186035, + 4.453901290893555, + 4.380789756774902, + 4.391660690307617, + 4.4065117835998535, + 4.341928005218506, + 4.368319034576416, + 4.301506996154785, + 4.4474639892578125, + 4.463913917541504, + 4.33223819732666, + 4.3883209228515625, + 4.4147562980651855, + 4.556249618530273, + 4.490050315856934, + 4.346927165985107, + 4.382404327392578, + 4.358379364013672, + 4.342800617218018, + 4.342830657958984, + 4.3948140144348145, + 4.470839023590088, + 4.392463684082031, + 4.336881637573242, + 4.409450054168701, + 4.352149486541748, + 4.4047980308532715, + 4.314850807189941, + 4.380906581878662, + 4.434874057769775, + 4.303234577178955, + 4.395068645477295, + 4.431570053100586, + 4.3952178955078125, + 4.380964756011963, + 4.311021327972412, + 4.451600551605225, + 4.453512191772461, + 4.366583347320557, + 4.355143070220947, + 4.360803127288818, + 4.312671661376953, + 4.318079948425293, + 4.2469353675842285, + 4.235177040100098, + 4.160510063171387, + 4.322113513946533, + 4.314157009124756, + 4.169283390045166, + 4.2997355461120605, + 4.231490135192871, + 4.381763458251953, + 4.234740734100342, + 4.348322868347168, + 4.342977046966553, + 4.291792869567871, + 4.313636302947998, + 4.263790607452393, + 4.138399600982666, + 4.2129974365234375, + 4.216996669769287, + 4.292860984802246, + 4.2097392082214355, + 4.196089267730713, + 4.2992448806762695, + 4.3383331298828125, + 4.14984130859375, + 4.256396293640137, + 4.168440818786621, + 4.243204116821289, + 4.3286662101745605, + 4.173386096954346, + 4.186971664428711, + 4.206610202789307, + 4.233433246612549, + 4.25978946685791, + 4.234650135040283, + 4.173432350158691, + 4.21290922164917, + 4.307881832122803, + 4.1788716316223145, + 4.285676002502441, + 4.231700897216797, + 4.26480770111084, + 4.267703056335449, + 4.193459510803223, + 4.221256732940674, + 4.332178592681885, + 4.26769495010376, + 4.309901237487793, + 4.187234401702881, + 4.319408893585205, + 4.327094554901123, + 4.357929706573486, + 4.137257099151611, + 4.323658466339111, + 4.175122261047363, + 4.28308629989624, + 4.270374298095703, + 4.260572910308838, + 4.240923881530762, + 4.19804573059082, + 4.2050395011901855, + 4.384881496429443, + 4.240137100219727, + 4.209138870239258, + 4.236792087554932, + 4.2531938552856445, + 4.178893566131592, + 4.295121669769287, + 4.1499128341674805, + 4.232303619384766, + 4.189357757568359, + 4.179378032684326, + 4.285256862640381, + 4.286698818206787, + 4.309389591217041, + 4.132338523864746, + 4.144874572753906, + 4.205210208892822, + 4.2423415184021, + 4.207451820373535, + 4.273404598236084, + 4.157732963562012, + 4.271421432495117, + 4.174581527709961, + 4.289032936096191, + 4.209283351898193, + 4.163344860076904, + 4.161672115325928, + 4.192717552185059, + 4.20180606842041, + 4.319916725158691, + 4.209186553955078, + 4.075627326965332, + 4.225636005401611, + 4.305048942565918, + 4.096019268035889, + 4.2615509033203125, + 4.313490390777588, + 4.202728271484375, + 4.228420734405518, + 4.259117126464844, + 4.239459991455078, + 4.189067363739014, + 4.194685935974121, + 4.119495868682861, + 3.9290683269500732, + 4.266665935516357, + 4.249866962432861, + 4.207917213439941, + 4.274136066436768, + 4.0646443367004395, + 4.1034932136535645, + 4.244165420532227, + 4.114044666290283, + 4.246510982513428, + 4.360774517059326, + 4.259702682495117, + 4.048162937164307, + 4.132155418395996, + 4.070094108581543, + 4.086250305175781, + 4.1785759925842285, + 4.220047950744629, + 4.186407566070557, + 4.209071159362793, + 4.2118353843688965, + 4.138676643371582, + 4.331902503967285, + 4.267720699310303, + 4.15631103515625, + 4.135409355163574, + 3.9677300453186035, + 4.1316237449646, + 3.971393346786499, + 4.044503688812256, + 4.281919002532959, + 4.065462112426758, + 4.230655193328857, + 4.18438720703125, + 4.083676338195801, + 4.134618759155273, + 4.222349643707275, + 4.075482368469238, + 4.245147705078125, + 4.041881084442139, + 4.157546043395996, + 4.149017333984375, + 4.150049686431885, + 4.05891752243042, + 4.130162715911865, + 3.9645638465881348, + 4.127623081207275, + 4.150415420532227, + 4.104499816894531, + 4.118504047393799, + 4.107656955718994, + 4.176150321960449, + 4.183961391448975, + 4.063260555267334, + 4.107975959777832, + 4.030808448791504, + 4.037749767303467, + 4.009738445281982, + 4.043588161468506, + 4.149060249328613, + 4.198793888092041, + 4.231342792510986, + 4.139727592468262, + 4.16170072555542, + 4.008457183837891, + 4.057479381561279, + 4.115341663360596, + 4.145130634307861, + 4.06519889831543, + 4.065218925476074, + 4.111804485321045, + 4.09198522567749, + 4.160293102264404, + 4.011506080627441, + 4.052790641784668, + 4.061319828033447, + 4.134431838989258, + 4.054068565368652, + 4.163208484649658, + 4.077110767364502, + 4.073939800262451, + 4.04329252243042, + 4.06936502456665, + 4.046213150024414, + 4.126054286956787, + 4.126184463500977, + 4.055830955505371, + 4.068501949310303, + 4.118666172027588, + 4.143632888793945, + 4.142736434936523, + 4.021474361419678, + 4.1009087562561035, + 3.9850661754608154, + 4.102245807647705, + 4.081896781921387, + 4.204614639282227, + 4.166248321533203, + 4.0720672607421875, + 4.155318260192871, + 4.091954708099365, + 4.092930793762207, + 4.084661483764648, + 4.125166893005371, + 4.086357116699219, + 3.9650461673736572, + 4.082123756408691, + 3.9357516765594482, + 4.044614315032959, + 3.9449260234832764, + 4.234904766082764, + 4.087126731872559, + 4.136551856994629, + 4.0663323402404785, + 4.119959831237793, + 4.1004133224487305, + 4.041895866394043, + 4.122751235961914, + 4.217893600463867, + 3.999913215637207, + 4.059511184692383, + 4.110532283782959, + 4.051271915435791, + 3.993039608001709, + 4.1609206199646, + 4.098269939422607, + 4.058764457702637, + 4.0150909423828125, + 4.194244861602783, + 4.1958770751953125, + 4.006665229797363, + 4.119564056396484, + 4.018320083618164, + 4.073505878448486, + 4.053427219390869, + 4.030669212341309, + 4.118142604827881, + 4.075901508331299, + 3.963175058364868, + 4.170267105102539, + 4.037238597869873, + 4.074573993682861, + 4.1752119064331055, + 4.008391857147217, + 4.092689037322998, + 4.048241138458252, + 4.074843406677246, + 4.052072048187256, + 4.115540981292725, + 4.137927532196045, + 4.086734294891357, + 4.04236364364624, + 3.948427200317383, + 3.9735798835754395, + 4.0426859855651855, + 3.9709644317626953, + 3.9854836463928223, + 4.104913234710693, + 4.083658218383789, + 4.03668212890625, + 3.9142987728118896, + 4.016942024230957, + 3.976295232772827, + 4.01576042175293, + 4.087861061096191, + 4.009000778198242, + 4.071261405944824, + 3.9398467540740967, + 3.9737212657928467, + 3.9549038410186768, + 4.0883708000183105, + 3.938849687576294, + 4.068554401397705, + 4.112576007843018, + 4.0320563316345215, + 4.069311618804932, + 3.8983118534088135, + 3.999847173690796, + 4.170792579650879, + 4.023039817810059, + 4.0178704261779785, + 4.020315647125244, + 4.11500883102417, + 3.9570388793945312, + 4.027602195739746, + 4.033708572387695, + 3.9242348670959473, + 3.9308319091796875, + 4.099045276641846, + 4.059151649475098, + 4.05733060836792, + 3.9674952030181885, + 3.9970972537994385, + 4.03339958190918, + 3.979996919631958, + 4.032960414886475, + 4.177412509918213, + 4.067578315734863, + 4.05120325088501, + 3.981750965118408, + 4.140533924102783, + 4.008327960968018, + 3.920478105545044, + 4.0081257820129395, + 4.02121639251709, + 4.092112064361572, + 4.109582901000977, + 4.012759685516357, + 4.077979564666748, + 4.139657020568848, + 4.029391765594482, + 4.122008800506592, + 4.071514129638672, + 4.133362293243408, + 3.9567909240722656, + 4.00909423828125, + 4.015197277069092, + 3.961667776107788, + 3.9444453716278076, + 4.018621921539307, + 4.017542839050293, + 4.074538230895996, + 4.0451979637146, + 4.153993129730225, + 3.99701189994812, + 3.8613288402557373, + 4.055713653564453, + 3.999655246734619, + 3.9922335147857666, + 3.9326674938201904, + 4.012633323669434, + 4.003753185272217, + 3.94130802154541, + 3.9321231842041016, + 3.969590663909912, + 3.9812614917755127, + 4.025298118591309, + 4.043853759765625, + 4.05267333984375, + 3.981334924697876, + 4.018734931945801, + 4.032784938812256, + 4.109601020812988, + 4.063525199890137, + 4.088909149169922, + 3.978041410446167, + 3.9424149990081787, + 3.9876952171325684, + 4.028805732727051, + 4.007507801055908, + 3.973443031311035, + 3.898740768432617, + 3.944822311401367, + 4.071573734283447, + 4.012362957000732, + 3.9703636169433594, + 3.9975342750549316, + 3.9624216556549072, + 4.0479302406311035, + 3.9981114864349365, + 3.8727142810821533, + 4.065999984741211, + 4.040173530578613, + 3.9444644451141357, + 4.049898147583008, + 3.9919044971466064, + 3.935513973236084, + 4.084402084350586, + 3.9531445503234863, + 4.021856307983398, + 3.9634714126586914, + 3.8677070140838623, + 3.971623659133911, + 3.9450180530548096, + 4.0554890632629395, + 3.911479949951172, + 3.974398136138916, + 3.791246175765991, + 3.9636611938476562, + 3.889944314956665, + 3.8921244144439697, + 3.8455400466918945, + 3.986825704574585, + 3.977816343307495, + 4.058598518371582, + 3.9984426498413086, + 3.847010374069214, + 3.89776873588562, + 3.9702343940734863, + 3.938770055770874, + 3.9386284351348877, + 3.9267220497131348, + 3.811652898788452, + 4.078455448150635, + 4.012345790863037, + 3.892706871032715, + 3.8258910179138184, + 4.037933826446533, + 3.9208545684814453, + 3.956766128540039, + 3.9796957969665527, + 3.903769016265869, + 4.020031452178955, + 3.96421480178833, + 4.064696311950684, + 3.7553822994232178, + 3.989269733428955, + 3.9122791290283203, + 3.9123191833496094, + 4.043893814086914, + 4.055561065673828, + 3.8843541145324707, + 3.872927188873291, + 3.9999654293060303, + 3.8222055435180664, + 4.048491954803467, + 3.974356174468994, + 3.9474360942840576, + 3.8825297355651855, + 3.918848991394043, + 3.9259302616119385, + 3.860616445541382, + 4.035797119140625, + 4.0374345779418945, + 4.115340709686279, + 3.9483401775360107, + 3.908029556274414, + 3.978961706161499, + 3.9902257919311523, + 3.9126546382904053, + 3.9489243030548096, + 3.853865623474121, + 4.024699687957764, + 3.945723295211792, + 3.771108388900757, + 4.0098958015441895, + 3.9596619606018066, + 3.989961624145508, + 3.95054292678833, + 3.9771833419799805, + 3.862330913543701, + 4.0188517570495605, + 3.9945645332336426, + 4.069726467132568, + 3.9403393268585205, + 4.0698747634887695, + 4.000523567199707, + 4.004951000213623, + 3.937460422515869, + 3.802842855453491, + 3.9132068157196045, + 3.8569278717041016, + 3.833998203277588, + 3.93733811378479, + 3.883852005004883, + 3.9401087760925293, + 3.985931634902954, + 4.008354187011719, + 3.9805567264556885, + 4.00019645690918, + 4.0050249099731445, + 3.97782039642334, + 3.965115547180176, + 3.932115316390991, + 3.89422607421875, + 3.9906439781188965, + 3.8724398612976074, + 3.974925994873047, + 4.0169997215271, + 3.8046047687530518, + 3.9124786853790283, + 3.947866916656494, + 3.899343729019165, + 3.809018611907959, + 4.015989780426025, + 3.833592176437378, + 3.8896005153656006, + 3.953286647796631, + 4.043079376220703, + 3.945884943008423, + 3.9497416019439697, + 3.934088706970215, + 3.924872398376465, + 4.026057720184326, + 3.917456865310669, + 3.930960178375244, + 3.878629207611084, + 4.012773036956787, + 3.8510842323303223, + 3.9110872745513916, + 3.9059159755706787, + 4.0440192222595215, + 3.943007707595825, + 4.017037868499756, + 3.961059331893921, + 4.015172481536865, + 3.8393592834472656, + 3.99524188041687, + 3.867380380630493, + 3.867133378982544, + 3.9957377910614014, + 4.054886341094971, + 3.750532865524292, + 4.03328275680542, + 3.6577072143554688, + 3.8727803230285645, + 3.9567196369171143, + 3.8417043685913086, + 3.959347724914551, + 4.002448081970215, + 3.9513564109802246, + 3.937650442123413, + 3.9198853969573975, + 3.8853092193603516, + 3.863748073577881, + 3.9351165294647217, + 3.9422836303710938, + 3.954179525375366, + 3.935668468475342, + 3.8465380668640137, + 3.8772904872894287, + 3.97598934173584, + 3.886794090270996, + 4.045575141906738, + 3.943523406982422, + 3.9772160053253174, + 3.9154696464538574, + 3.8592586517333984, + 3.8589563369750977, + 3.882380247116089, + 3.998487949371338, + 3.89343523979187, + 4.063683032989502, + 4.029210567474365, + 3.8615050315856934, + 3.9581778049468994, + 3.883481025695801, + 3.833719253540039, + 3.872981548309326, + 3.850689649581909, + 3.820943593978882, + 3.8287651538848877, + 3.987445592880249, + 3.865781545639038, + 3.807330846786499, + 3.9259262084960938, + 3.9247024059295654, + 3.8259592056274414, + 3.830749034881592, + 3.912431478500366, + 3.8639256954193115, + 3.9379448890686035, + 3.8629908561706543, + 3.75398588180542, + 3.9356651306152344, + 3.8208162784576416, + 3.87754225730896, + 3.871701717376709, + 3.9084033966064453, + 3.978227376937866, + 3.9847207069396973, + 3.9277737140655518, + 3.931495189666748, + 3.913533926010132, + 3.89054012298584, + 3.826225757598877, + 3.9348857402801514, + 3.897249460220337, + 3.8583943843841553, + 3.950165271759033, + 3.910656452178955, + 3.9363691806793213, + 3.941558837890625, + 3.9963979721069336, + 3.926034450531006, + 3.9416873455047607, + 3.9573497772216797, + 3.7894558906555176, + 3.851001501083374, + 3.8815221786499023, + 3.839764356613159, + 3.858285427093506, + 3.856379270553589, + 3.9113502502441406, + 3.804365873336792, + 3.8571412563323975, + 3.8736650943756104, + 3.977400302886963, + 3.8757307529449463, + 3.8798015117645264, + 3.9827497005462646, + 3.8271586894989014, + 3.776305675506592, + 3.8008408546447754, + 4.063591957092285, + 3.8794867992401123, + 3.9026591777801514, + 3.901179313659668, + 3.8285200595855713, + 3.9874162673950195, + 3.714953899383545, + 3.885268449783325, + 3.764209508895874, + 3.8844692707061768, + 3.758042335510254, + 3.986170768737793, + 3.891394853591919, + 3.885134220123291, + 3.9291975498199463, + 4.030518531799316, + 3.8468704223632812, + 3.932849645614624, + 3.922452449798584, + 3.9668772220611572, + 3.9491617679595947, + 3.842987060546875, + 3.8550667762756348, + 3.9097707271575928, + 3.8353688716888428, + 3.885064125061035, + 4.027134895324707, + 3.922077178955078, + 3.9512903690338135, + 3.928797483444214, + 3.922722101211548, + 3.8202877044677734, + 3.892500638961792, + 3.817582845687866, + 3.7681381702423096, + 3.8794360160827637, + 3.8768482208251953, + 3.7653415203094482, + 3.889066696166992, + 3.7941763401031494, + 3.8055307865142822, + 3.923511028289795, + 3.8241825103759766, + 3.8371448516845703, + 3.7755777835845947, + 3.975802421569824, + 3.902949810028076, + 3.959162473678589, + 3.9631593227386475, + 3.8488192558288574, + 3.874091386795044, + 3.8349621295928955, + 4.064188480377197, + 3.951260566711426, + 3.8156445026397705, + 3.8616726398468018, + 3.9158341884613037, + 3.8624253273010254, + 3.8484771251678467, + 3.9885144233703613, + 3.8043291568756104, + 3.862230062484741, + 3.8849523067474365, + 3.845898389816284, + 3.8953683376312256, + 3.863888740539551, + 3.809208869934082, + 3.9490113258361816, + 3.988097667694092, + 3.9126763343811035, + 3.8870131969451904, + 3.732628345489502, + 3.866285562515259, + 3.7248570919036865, + 3.860158920288086, + 3.8889331817626953, + 3.8618814945220947, + 3.825300455093384, + 3.74538516998291, + 3.872971773147583, + 3.7485404014587402, + 3.9086649417877197, + 3.7749109268188477, + 3.9542126655578613, + 3.7168431282043457, + 3.989121913909912, + 3.8160223960876465, + 3.6842000484466553, + 3.889364719390869, + 3.738724708557129, + 3.987975597381592, + 3.7628183364868164, + 3.855407476425171, + 3.8259150981903076, + 3.793703317642212, + 3.8070449829101562, + 3.9103596210479736, + 3.755150556564331, + 3.899073362350464, + 3.967019557952881, + 3.8249576091766357, + 3.8526856899261475, + 3.919806957244873, + 3.87359619140625, + 3.834773302078247, + 3.8949081897735596, + 3.8605048656463623, + 3.8709514141082764, + 3.778364658355713, + 3.989603281021118, + 3.9078445434570312, + 3.9317564964294434, + 3.7294507026672363, + 3.8922648429870605, + 3.9402108192443848, + 3.9291512966156006, + 3.9485318660736084, + 3.803903102874756, + 3.88616681098938, + 3.8473711013793945, + 3.9094996452331543, + 3.869095802307129, + 3.8147735595703125, + 3.8301331996917725, + 3.899794578552246, + 3.9173529148101807, + 3.885568857192993, + 4.000716209411621, + 3.9222142696380615, + 3.8777518272399902, + 3.9059407711029053, + 3.840385913848877, + 3.831106185913086, + 3.848752498626709, + 3.8432188034057617, + 3.833709239959717, + 3.720686197280884, + 3.934213399887085, + 3.8328540325164795, + 3.9286751747131348, + 3.9251537322998047, + 3.8880534172058105, + 3.935915946960449, + 3.8354337215423584, + 3.8553850650787354, + 3.746135950088501, + 3.8639938831329346, + 3.882201671600342, + 3.7988274097442627, + 3.881434917449951, + 3.779639482498169 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 178.2019319576431, + 111.30269461627533, + 88.6467223320616, + 76.78520935610351, + 69.50637896047202, + 64.90231583691536, + 61.30240733829619, + 58.33494492744258, + 56.21110000417536, + 54.118937266653816, + 52.83034471488058, + 51.66322368909046, + 50.64244845765722, + 49.737776189589624, + 48.79380631882265, + 48.40214538653971, + 47.91658432852511, + 47.55924665675117, + 47.482299288732925, + 47.46156921359967 + ], + "best_ppl": 47.46156921359967, + "train_time_s": 5991.5060868263245 +} \ No newline at end of file diff --git a/063_sparse_dispatch/single_block_deltanet_d512_best.pt b/063_sparse_dispatch/single_block_deltanet_d512_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..da2c3546a8305a1050473f5b0941d3721ec54b27 --- /dev/null +++ b/063_sparse_dispatch/single_block_deltanet_d512_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb0869255f7ec549d8bff99713a450c4b90acb6ff7a04170b28cfaf4f452398b +size 274284689 diff --git a/063_sparse_dispatch/single_block_deltanet_d512_results.json b/063_sparse_dispatch/single_block_deltanet_d512_results.json new file mode 100644 index 0000000000000000000000000000000000000000..33524572f3abda92b5ea1c458b24214926b6fd8a --- /dev/null +++ b/063_sparse_dispatch/single_block_deltanet_d512_results.json @@ -0,0 +1,2057 @@ +{ + "label": "single_block_deltanet_d512", + "seed": 42, + "attention_type": "deltanet", + "d_expert": 512, + "total_params": 68553344, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 16.528886795043945, + 8.618110656738281, + 7.490260601043701, + 7.264334201812744, + 7.037673473358154, + 6.874759197235107, + 6.643682479858398, + 6.509603977203369, + 6.363207817077637, + 6.321671009063721, + 6.128136157989502, + 6.087252140045166, + 6.067698955535889, + 5.786691188812256, + 5.859809875488281, + 5.648075103759766, + 5.752217769622803, + 5.537846088409424, + 5.490428924560547, + 5.471490383148193, + 5.406922340393066, + 5.376834392547607, + 5.266941070556641, + 5.285179138183594, + 5.246026992797852, + 5.203802108764648, + 5.096153736114502, + 5.1386613845825195, + 5.062782287597656, + 5.084237575531006, + 5.015133380889893, + 5.035488605499268, + 5.042423248291016, + 4.987601280212402, + 4.8634233474731445, + 4.9670634269714355, + 4.7757248878479, + 4.794133186340332, + 4.777547836303711, + 4.741494655609131, + 4.724131107330322, + 4.70411491394043, + 4.67458438873291, + 4.767553329467773, + 4.735198497772217, + 4.816982746124268, + 4.705009460449219, + 4.576008319854736, + 4.704610347747803, + 4.547817707061768, + 4.7104034423828125, + 4.648693084716797, + 4.718600749969482, + 4.6524505615234375, + 4.542726993560791, + 4.547599792480469, + 4.512426376342773, + 4.570133686065674, + 4.565629959106445, + 4.457622528076172, + 4.646034240722656, + 4.604380130767822, + 4.500674724578857, + 4.5488457679748535, + 4.481797695159912, + 4.4227190017700195, + 4.386561393737793, + 4.483713626861572, + 4.342605113983154, + 4.332217693328857, + 4.421446800231934, + 4.4102630615234375, + 4.302608489990234, + 4.297614574432373, + 4.386703968048096, + 4.345748424530029, + 4.4429612159729, + 4.313309669494629, + 4.339961528778076, + 4.361697673797607, + 4.284136772155762, + 4.296561241149902, + 4.221492290496826, + 4.194161891937256, + 4.2890825271606445, + 4.385241985321045, + 4.313409805297852, + 4.315007209777832, + 4.249443054199219, + 4.308238983154297, + 4.280007839202881, + 4.210214138031006, + 4.365985870361328, + 4.24113130569458, + 4.262454509735107, + 4.111459255218506, + 4.273280620574951, + 4.313591480255127, + 4.24973726272583, + 4.173173427581787, + 4.309143543243408, + 4.285940647125244, + 4.26998233795166, + 4.2041335105896, + 4.243032455444336, + 4.17495059967041, + 4.199902534484863, + 4.136739253997803, + 4.047829627990723, + 4.18170690536499, + 4.179698944091797, + 4.096046447753906, + 4.004177570343018, + 4.175719261169434, + 4.00193977355957, + 4.134461402893066, + 4.117331027984619, + 4.201888084411621, + 4.2221360206604, + 4.1278181076049805, + 4.180480003356934, + 4.031133651733398, + 4.130888938903809, + 4.165482044219971, + 4.184140205383301, + 4.1549201011657715, + 3.942099094390869, + 4.032547950744629, + 3.974400758743286, + 3.989851713180542, + 4.136106491088867, + 4.045405387878418, + 4.052450180053711, + 3.9895002841949463, + 3.89585018157959, + 4.151087760925293, + 4.080943584442139, + 3.993659019470215, + 3.8633174896240234, + 3.9666481018066406, + 4.023613929748535, + 4.059356689453125, + 4.085261821746826, + 3.9771692752838135, + 3.9070839881896973, + 4.095020294189453, + 3.970780849456787, + 4.073159694671631, + 3.809168577194214, + 4.006680488586426, + 4.027744293212891, + 3.9814517498016357, + 4.068812370300293, + 4.058868408203125, + 3.898420572280884, + 3.9539825916290283, + 4.011340618133545, + 4.102452754974365, + 3.9377880096435547, + 4.03192138671875, + 3.9448299407958984, + 4.049415111541748, + 3.888700008392334, + 4.064359188079834, + 4.077908992767334, + 3.9536819458007812, + 4.108028888702393, + 4.002048015594482, + 3.9638547897338867, + 3.8977317810058594, + 3.91617488861084, + 4.092263221740723, + 3.976463794708252, + 4.041542053222656, + 3.9815919399261475, + 4.040984630584717, + 3.9303600788116455, + 3.8504185676574707, + 3.954707384109497, + 3.980611801147461, + 3.889503240585327, + 3.8553144931793213, + 3.8963396549224854, + 3.914213180541992, + 3.9073188304901123, + 4.034877777099609, + 3.8518834114074707, + 3.923818826675415, + 3.944073438644409, + 3.908273458480835, + 4.0507683753967285, + 3.9514811038970947, + 3.9428741931915283, + 3.958681106567383, + 3.892235279083252, + 3.927875518798828, + 3.8114891052246094, + 3.8038203716278076, + 3.860494375228882, + 3.8671727180480957, + 3.921900510787964, + 3.8685169219970703, + 3.815171957015991, + 3.84080171585083, + 3.894885301589966, + 4.004189968109131, + 3.940119981765747, + 3.8817427158355713, + 3.93430757522583, + 3.955993890762329, + 3.7580690383911133, + 3.862086057662964, + 3.9857842922210693, + 4.004520416259766, + 3.882293701171875, + 3.9399263858795166, + 3.769292116165161, + 3.9555368423461914, + 3.9451563358306885, + 3.9461865425109863, + 3.849254846572876, + 3.8735458850860596, + 3.8974130153656006, + 3.8236262798309326, + 3.9049551486968994, + 3.869642496109009, + 3.7691352367401123, + 3.7994487285614014, + 3.858684778213501, + 3.887035369873047, + 3.7886388301849365, + 3.9229586124420166, + 3.882129430770874, + 3.815164566040039, + 3.921924352645874, + 3.916400194168091, + 3.8731377124786377, + 3.7516181468963623, + 3.8226797580718994, + 3.8468141555786133, + 3.891629219055176, + 3.956770896911621, + 3.689847946166992, + 3.8377108573913574, + 3.981879949569702, + 3.889333724975586, + 3.9509565830230713, + 3.719728708267212, + 3.935065746307373, + 3.7982394695281982, + 3.824537754058838, + 3.9397969245910645, + 3.799494504928589, + 3.825364351272583, + 3.6251883506774902, + 3.7655858993530273, + 3.7758500576019287, + 3.7141425609588623, + 3.5946826934814453, + 3.6947548389434814, + 3.5929157733917236, + 3.7204511165618896, + 3.7494449615478516, + 3.7604658603668213, + 3.844877004623413, + 3.8415844440460205, + 3.8961737155914307, + 3.906810760498047, + 3.8789970874786377, + 3.6893222332000732, + 3.727041721343994, + 3.6716222763061523, + 3.6618452072143555, + 3.761591911315918, + 3.6376888751983643, + 3.8086276054382324, + 3.7375707626342773, + 3.7338545322418213, + 3.7862589359283447, + 3.779496908187866, + 3.743140935897827, + 3.716708183288574, + 3.7044835090637207, + 3.6588404178619385, + 3.5751614570617676, + 3.8903043270111084, + 3.699963092803955, + 3.656158924102783, + 3.7595882415771484, + 3.786419153213501, + 3.79750919342041, + 3.865222215652466, + 3.804375410079956, + 3.6927995681762695, + 3.699458360671997, + 3.577537775039673, + 3.7001540660858154, + 3.7788612842559814, + 3.7907533645629883, + 3.64755916595459, + 3.8284943103790283, + 3.700690507888794, + 3.6641039848327637, + 3.64981746673584, + 3.6593880653381348, + 3.824692487716675, + 3.802788257598877, + 3.6893656253814697, + 3.6726162433624268, + 3.6440463066101074, + 3.697420358657837, + 3.72687029838562, + 3.7739663124084473, + 3.6731367111206055, + 3.7918097972869873, + 3.8607804775238037, + 3.771679639816284, + 3.8412582874298096, + 3.6746604442596436, + 3.6497280597686768, + 3.8073112964630127, + 3.579792022705078, + 3.7182700634002686, + 3.666485548019409, + 3.761935234069824, + 3.666808843612671, + 3.7309513092041016, + 3.760500907897949, + 3.7396340370178223, + 3.7198619842529297, + 3.755716562271118, + 3.6621904373168945, + 3.758481979370117, + 3.806381940841675, + 3.835543394088745, + 3.810584545135498, + 3.72452974319458, + 3.6558704376220703, + 3.691290855407715, + 3.636965274810791, + 3.6691527366638184, + 3.760770797729492, + 3.6594743728637695, + 3.6811342239379883, + 3.7885115146636963, + 3.8060302734375, + 3.75654935836792, + 3.623258590698242, + 3.7425894737243652, + 3.6048760414123535, + 3.7593579292297363, + 3.7397079467773438, + 3.6869940757751465, + 3.774780750274658, + 3.7261106967926025, + 3.6721980571746826, + 3.731485366821289, + 3.7203071117401123, + 3.7266323566436768, + 3.7161576747894287, + 3.7221057415008545, + 3.675224781036377, + 3.697767734527588, + 3.790729284286499, + 3.650785207748413, + 3.753094434738159, + 3.6673505306243896, + 3.671907901763916, + 3.5834927558898926, + 3.6792397499084473, + 3.7183101177215576, + 3.6887195110321045, + 3.7361581325531006, + 3.6464924812316895, + 3.583479404449463, + 3.6661288738250732, + 3.802464246749878, + 3.6661787033081055, + 3.7650535106658936, + 3.725410223007202, + 3.5981290340423584, + 3.590945243835449, + 3.6202404499053955, + 3.529048442840576, + 3.526369571685791, + 3.5734081268310547, + 3.5560812950134277, + 3.6371843814849854, + 3.5959174633026123, + 3.698472023010254, + 3.6864101886749268, + 3.7387070655822754, + 3.501382827758789, + 3.6053268909454346, + 3.5747323036193848, + 3.5654726028442383, + 3.5441136360168457, + 3.5318148136138916, + 3.649754524230957, + 3.6127710342407227, + 3.6204378604888916, + 3.668856620788574, + 3.6511614322662354, + 3.7247278690338135, + 3.591019630432129, + 3.5516445636749268, + 3.5468859672546387, + 3.67069411277771, + 3.6693317890167236, + 3.5381977558135986, + 3.572394847869873, + 3.6883156299591064, + 3.5797929763793945, + 3.6684787273406982, + 3.6913883686065674, + 3.5413014888763428, + 3.5319459438323975, + 3.590930223464966, + 3.6105844974517822, + 3.5129599571228027, + 3.626638650894165, + 3.642683744430542, + 3.6818206310272217, + 3.676504611968994, + 3.626537561416626, + 3.600297689437866, + 3.6756930351257324, + 3.610095977783203, + 3.483581304550171, + 3.5261528491973877, + 3.5809195041656494, + 3.592205286026001, + 3.5347628593444824, + 3.6179909706115723, + 3.5905685424804688, + 3.492525577545166, + 3.4729957580566406, + 3.5180110931396484, + 3.7181849479675293, + 3.562896728515625, + 3.609886646270752, + 3.600773334503174, + 3.614861011505127, + 3.507002115249634, + 3.5787980556488037, + 3.5450987815856934, + 3.578188180923462, + 3.608269453048706, + 3.661003828048706, + 3.6591615676879883, + 3.627495765686035, + 3.6979994773864746, + 3.687410354614258, + 3.5433926582336426, + 3.5685927867889404, + 3.6441664695739746, + 3.5511457920074463, + 3.610502243041992, + 3.626311779022217, + 3.7370245456695557, + 3.613036870956421, + 3.6771295070648193, + 3.6615195274353027, + 3.628997564315796, + 3.552626132965088, + 3.534257650375366, + 3.670865774154663, + 3.576979398727417, + 3.6775400638580322, + 3.6207306385040283, + 3.6338391304016113, + 3.5476155281066895, + 3.6049325466156006, + 3.53676700592041, + 3.595463991165161, + 3.5123982429504395, + 3.447573661804199, + 3.6136491298675537, + 3.5799858570098877, + 3.532949447631836, + 3.586390256881714, + 3.691234588623047, + 3.607839345932007, + 3.533810615539551, + 3.5976665019989014, + 3.6765711307525635, + 3.628058671951294, + 3.55987811088562, + 3.6086843013763428, + 3.523223876953125, + 3.574296474456787, + 3.6130638122558594, + 3.5678868293762207, + 3.5939688682556152, + 3.4727699756622314, + 3.551056146621704, + 3.5085878372192383, + 3.6396498680114746, + 3.65334153175354, + 3.555300235748291, + 3.5900890827178955, + 3.6624813079833984, + 3.6111032962799072, + 3.5241923332214355, + 3.764479875564575, + 3.478577136993408, + 3.489060401916504, + 3.3944365978240967, + 3.4851362705230713, + 3.3039069175720215, + 3.458601236343384, + 3.4555838108062744, + 3.3808205127716064, + 3.3318402767181396, + 3.563810110092163, + 3.4590024948120117, + 3.445098876953125, + 3.5910346508026123, + 3.5502567291259766, + 3.5027127265930176, + 3.5913047790527344, + 3.4876415729522705, + 3.4844846725463867, + 3.4108824729919434, + 3.4597959518432617, + 3.588407039642334, + 3.5097761154174805, + 3.626800060272217, + 3.4997403621673584, + 3.486643075942993, + 3.46016263961792, + 3.5016448497772217, + 3.5381100177764893, + 3.5492029190063477, + 3.5551252365112305, + 3.4686176776885986, + 3.346294403076172, + 3.4108505249023438, + 3.4998831748962402, + 3.436124801635742, + 3.4975383281707764, + 3.531369209289551, + 3.524155378341675, + 3.4328205585479736, + 3.5713462829589844, + 3.5421509742736816, + 3.4251708984375, + 3.3793206214904785, + 3.519915819168091, + 3.5623772144317627, + 3.308821201324463, + 3.5066378116607666, + 3.6018619537353516, + 3.3819172382354736, + 3.4113755226135254, + 3.5439088344573975, + 3.547010898590088, + 3.5130040645599365, + 3.4484832286834717, + 3.4676804542541504, + 3.48946213722229, + 3.485429286956787, + 3.343791961669922, + 3.5070157051086426, + 3.430152416229248, + 3.5122878551483154, + 3.5370845794677734, + 3.543505907058716, + 3.6177117824554443, + 3.475098133087158, + 3.5794517993927, + 3.4822537899017334, + 3.6462762355804443, + 3.4811573028564453, + 3.4544413089752197, + 3.3836162090301514, + 3.4270758628845215, + 3.4560794830322266, + 3.6014506816864014, + 3.4165749549865723, + 3.5456902980804443, + 3.510899543762207, + 3.4973669052124023, + 3.48172926902771, + 3.318429946899414, + 3.406010389328003, + 3.4923672676086426, + 3.487865447998047, + 3.424708127975464, + 3.467435598373413, + 3.4827842712402344, + 3.5671067237854004, + 3.480724573135376, + 3.525728940963745, + 3.4990122318267822, + 3.3917160034179688, + 3.4135091304779053, + 3.4852190017700195, + 3.5286788940429688, + 3.374610185623169, + 3.5731475353240967, + 3.4187211990356445, + 3.5411205291748047, + 3.544630527496338, + 3.5159404277801514, + 3.632321834564209, + 3.3869831562042236, + 3.527822732925415, + 3.480196237564087, + 3.4501800537109375, + 3.530092239379883, + 3.4393703937530518, + 3.3970675468444824, + 3.4794094562530518, + 3.4371542930603027, + 3.590148448944092, + 3.513408660888672, + 3.4878041744232178, + 3.545091390609741, + 3.5689098834991455, + 3.416395664215088, + 3.3969287872314453, + 3.524193525314331, + 3.519397020339966, + 3.2804946899414062, + 3.5020179748535156, + 3.5841410160064697, + 3.487933874130249, + 3.376009941101074, + 3.5020899772644043, + 3.3378829956054688, + 3.4436304569244385, + 3.456566333770752, + 3.5006861686706543, + 3.3728909492492676, + 3.3255040645599365, + 3.258937120437622, + 3.3532280921936035, + 3.350700616836548, + 3.428225517272949, + 3.4320883750915527, + 3.3664162158966064, + 3.366335868835449, + 3.3650219440460205, + 3.3090333938598633, + 3.3006365299224854, + 3.370441436767578, + 3.429080009460449, + 3.3846182823181152, + 3.4727957248687744, + 3.380648374557495, + 3.4440345764160156, + 3.3988773822784424, + 3.246243953704834, + 3.410391092300415, + 3.404186725616455, + 3.378674268722534, + 3.3181536197662354, + 3.34358549118042, + 3.3496105670928955, + 3.4832417964935303, + 3.391141653060913, + 3.3642067909240723, + 3.4211983680725098, + 3.4344687461853027, + 3.309457540512085, + 3.36991286277771, + 3.432156562805176, + 3.383073568344116, + 3.4271273612976074, + 3.4148991107940674, + 3.376969575881958, + 3.404811382293701, + 3.4176197052001953, + 3.3925716876983643, + 3.372263193130493, + 3.382046699523926, + 3.2545006275177, + 3.349943161010742, + 3.3798887729644775, + 3.433897018432617, + 3.302788496017456, + 3.4316468238830566, + 3.3931820392608643, + 3.349006175994873, + 3.4141664505004883, + 3.4143779277801514, + 3.3482816219329834, + 3.2545132637023926, + 3.4463860988616943, + 3.4243898391723633, + 3.360682487487793, + 3.408064365386963, + 3.4311773777008057, + 3.399550676345825, + 3.372138738632202, + 3.257676601409912, + 3.46510910987854, + 3.4078500270843506, + 3.374553918838501, + 3.4273481369018555, + 3.302751302719116, + 3.4286181926727295, + 3.341542959213257, + 3.2850749492645264, + 3.4429330825805664, + 3.381934404373169, + 3.2990753650665283, + 3.476314067840576, + 3.426640748977661, + 3.3670411109924316, + 3.2543487548828125, + 3.3376026153564453, + 3.5245394706726074, + 3.469590425491333, + 3.2761189937591553, + 3.3538143634796143, + 3.335909843444824, + 3.3726794719696045, + 3.4218757152557373, + 3.201484203338623, + 3.4245336055755615, + 3.407620906829834, + 3.4703683853149414, + 3.3193717002868652, + 3.316641330718994, + 3.341061592102051, + 3.441044807434082, + 3.331608772277832, + 3.2370738983154297, + 3.3805739879608154, + 3.399467706680298, + 3.4815003871917725, + 3.475719451904297, + 3.2192418575286865, + 3.434783697128296, + 3.155083417892456, + 3.442979097366333, + 3.4995176792144775, + 3.339130163192749, + 3.3530282974243164, + 3.4165639877319336, + 3.547816514968872, + 3.3455262184143066, + 3.448200225830078, + 3.3849332332611084, + 3.31884503364563, + 3.448779582977295, + 3.3715500831604004, + 3.2818872928619385, + 3.3645083904266357, + 3.369896411895752, + 3.388871192932129, + 3.4565813541412354, + 3.282909631729126, + 3.34027361869812, + 3.3426342010498047, + 3.3773717880249023, + 3.434504270553589, + 3.382544755935669, + 3.342252016067505, + 3.271064043045044, + 3.2813761234283447, + 3.4415769577026367, + 3.2755517959594727, + 3.3756260871887207, + 3.230538845062256, + 3.26413893699646, + 3.2769625186920166, + 3.3704099655151367, + 3.31683349609375, + 3.255873441696167, + 3.4050240516662598, + 3.369595527648926, + 3.2524452209472656, + 3.2771499156951904, + 3.2507309913635254, + 3.3794894218444824, + 3.2534561157226562, + 3.3516721725463867, + 3.3566272258758545, + 3.290029525756836, + 3.2370686531066895, + 3.2280890941619873, + 3.4534952640533447, + 3.322383403778076, + 3.348475456237793, + 3.42880916595459, + 3.346444845199585, + 3.1787045001983643, + 3.2442052364349365, + 3.269684076309204, + 3.3681178092956543, + 3.290776252746582, + 3.3282220363616943, + 3.3116137981414795, + 3.418344736099243, + 3.237260341644287, + 3.2624969482421875, + 3.192516803741455, + 3.354863405227661, + 3.228085517883301, + 3.3007590770721436, + 3.256455421447754, + 3.2801294326782227, + 3.384843587875366, + 3.2677414417266846, + 3.3834149837493896, + 3.2585535049438477, + 3.2220122814178467, + 3.2066221237182617, + 3.2762081623077393, + 3.306366443634033, + 3.3656625747680664, + 3.375748872756958, + 3.3221678733825684, + 3.314940929412842, + 3.3727355003356934, + 3.2880849838256836, + 3.2216567993164062, + 3.2993063926696777, + 3.3085739612579346, + 3.264897584915161, + 3.3165700435638428, + 3.3844847679138184, + 3.1649580001831055, + 3.3584163188934326, + 3.366790294647217, + 3.3920958042144775, + 3.2857723236083984, + 3.3158388137817383, + 3.2877323627471924, + 3.2140276432037354, + 3.3668088912963867, + 3.296436071395874, + 3.161900758743286, + 3.245404005050659, + 3.328695058822632, + 3.340507745742798, + 3.2966558933258057, + 3.2346370220184326, + 3.3287365436553955, + 3.271662950515747, + 3.3407437801361084, + 3.180899143218994, + 3.1434054374694824, + 3.112996816635132, + 3.166930913925171, + 3.1794302463531494, + 3.2917234897613525, + 3.3395371437072754, + 3.029384136199951, + 3.214843511581421, + 3.337320566177368, + 3.3245179653167725, + 3.2277274131774902, + 3.36312198638916, + 3.309166669845581, + 3.238896369934082, + 3.1398301124572754, + 3.336151599884033, + 3.4910457134246826, + 3.251000165939331, + 3.2883615493774414, + 3.42155122756958, + 3.2600362300872803, + 3.4132232666015625, + 3.3159728050231934, + 3.1956238746643066, + 3.3776707649230957, + 3.39898943901062, + 3.195645570755005, + 3.2929582595825195, + 3.3319573402404785, + 3.2201170921325684, + 3.3929076194763184, + 3.276353359222412, + 3.3220901489257812, + 3.355285406112671, + 3.3317744731903076, + 3.1902120113372803, + 3.3026041984558105, + 3.434429883956909, + 3.2565269470214844, + 3.2731494903564453, + 3.0848233699798584, + 3.285895586013794, + 3.2414023876190186, + 3.2251064777374268, + 3.0786478519439697, + 3.329586982727051, + 3.2737905979156494, + 3.2591919898986816, + 3.415743589401245, + 3.168147325515747, + 3.1419525146484375, + 3.2755770683288574, + 3.3441460132598877, + 3.3177900314331055, + 3.192981004714966, + 3.286008596420288, + 3.245826244354248, + 3.183466911315918, + 3.3815255165100098, + 3.388800859451294, + 3.2727909088134766, + 3.236671209335327, + 3.334707021713257, + 3.1790506839752197, + 3.2323834896087646, + 3.2920727729797363, + 3.233962297439575, + 3.259239673614502, + 3.239696502685547, + 3.278024911880493, + 3.2893056869506836, + 3.14611554145813, + 3.249260663986206, + 3.1186819076538086, + 3.3873302936553955, + 3.201664924621582, + 3.1807823181152344, + 3.3242454528808594, + 3.278508186340332, + 3.14572811126709, + 3.2019715309143066, + 3.1919238567352295, + 3.360610246658325, + 3.3300976753234863, + 3.2667407989501953, + 3.329585552215576, + 3.2741167545318604, + 3.2354280948638916, + 3.3796279430389404, + 3.2641234397888184, + 3.3913114070892334, + 3.2029294967651367, + 3.2523813247680664, + 3.293534994125366, + 3.20499587059021, + 3.221822500228882, + 3.248243570327759, + 3.279672145843506, + 3.292696714401245, + 3.3355534076690674, + 3.290783405303955, + 3.3648252487182617, + 3.166903018951416, + 3.2793352603912354, + 3.2317559719085693, + 3.1410605907440186, + 3.350327491760254, + 3.402092933654785, + 3.333085536956787, + 3.2426626682281494, + 3.291677951812744, + 3.2585842609405518, + 3.32513689994812, + 3.2384846210479736, + 3.117453098297119, + 3.2342700958251953, + 3.1892924308776855, + 3.3080668449401855, + 3.2251107692718506, + 3.257551431655884, + 3.240180015563965, + 3.2309329509735107, + 3.389737367630005, + 3.250641107559204, + 2.984440326690674, + 3.2072675228118896, + 3.2088027000427246, + 3.3288705348968506, + 3.2533297538757324, + 3.281416177749634, + 3.2844045162200928, + 3.2936511039733887, + 3.2127437591552734, + 3.185361623764038, + 3.3098015785217285, + 3.3341634273529053, + 3.1391761302948, + 3.212069511413574, + 3.2463788986206055, + 3.344780683517456, + 3.2985246181488037, + 3.186746835708618, + 3.211211919784546, + 3.1982269287109375, + 3.3931491374969482, + 3.3356449604034424, + 3.1401705741882324, + 3.0674922466278076, + 3.3116424083709717, + 3.23398756980896, + 3.215467691421509, + 3.3275773525238037, + 3.2926881313323975, + 3.26904034614563, + 3.240065813064575 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 93.13382019034653, + 60.85768256647037, + 51.506078427343034, + 46.29834125454488, + 43.27483533670639, + 41.01901609874112, + 39.18810316674485, + 37.599399155142166, + 36.17059263965323, + 34.999567666481575, + 34.295263539797205, + 33.16431105157259, + 32.70672145670586, + 31.949294147644256, + 31.254248755471135, + 31.076971714403864, + 30.716758957185146, + 30.612989824337564, + 30.54597878632734, + 30.521763432614147 + ], + "best_ppl": 30.521763432614147, + "train_time_s": 11322.3492436409 +} \ No newline at end of file diff --git a/063_sparse_dispatch/single_block_deltanet_results.json b/063_sparse_dispatch/single_block_deltanet_results.json new file mode 100644 index 0000000000000000000000000000000000000000..85ca4fe5a13ddec5a312ad9fa6a17c405c968e8c --- /dev/null +++ b/063_sparse_dispatch/single_block_deltanet_results.json @@ -0,0 +1,2057 @@ +{ + "label": "single_block_deltanet", + "seed": 42, + "attention_type": "deltanet", + "num_thin_blocks": 1, + "total_params": 43833920, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 18.986236572265625, + 15.88306713104248, + 8.034016609191895, + 7.601480007171631, + 7.5198073387146, + 7.512390613555908, + 7.420172691345215, + 7.24690580368042, + 7.187628269195557, + 6.8986616134643555, + 6.861200332641602, + 6.735040664672852, + 6.638885974884033, + 6.420592784881592, + 6.44752836227417, + 6.3326263427734375, + 6.2535400390625, + 6.183490753173828, + 5.917716979980469, + 5.977381229400635, + 5.8147292137146, + 5.72704553604126, + 5.668446063995361, + 5.599365234375, + 5.661370754241943, + 5.6440205574035645, + 5.516656398773193, + 5.404927730560303, + 5.501733779907227, + 5.4518632888793945, + 5.37692928314209, + 5.3355913162231445, + 5.252070426940918, + 5.290843486785889, + 5.282811164855957, + 5.2386064529418945, + 5.142518997192383, + 5.100065231323242, + 5.174398899078369, + 5.236088275909424, + 5.0477118492126465, + 5.171808242797852, + 5.011431694030762, + 5.036450386047363, + 5.068264961242676, + 5.067263603210449, + 5.050564765930176, + 5.129319190979004, + 5.103420734405518, + 4.943489074707031, + 5.096777439117432, + 5.062789440155029, + 4.9604668617248535, + 4.946126937866211, + 4.92174768447876, + 4.807257652282715, + 4.739697456359863, + 4.778863906860352, + 4.852664947509766, + 4.785428047180176, + 4.835392475128174, + 4.791952133178711, + 4.77969217300415, + 4.762716770172119, + 4.7563300132751465, + 4.844008922576904, + 4.749773979187012, + 4.807099342346191, + 4.829346656799316, + 4.756350994110107, + 4.75869607925415, + 4.677626132965088, + 4.744924068450928, + 4.6553568840026855, + 4.637032985687256, + 4.675136089324951, + 4.61014461517334, + 4.624022006988525, + 4.612794399261475, + 4.793307304382324, + 4.554707050323486, + 4.5203633308410645, + 4.4770097732543945, + 4.510761260986328, + 4.6921000480651855, + 4.551678657531738, + 4.564573764801025, + 4.646676063537598, + 4.518232345581055, + 4.7028303146362305, + 4.507776737213135, + 4.623928070068359, + 4.555524826049805, + 4.548391819000244, + 4.446454048156738, + 4.397367477416992, + 4.505640506744385, + 4.378288745880127, + 4.488551616668701, + 4.499248504638672, + 4.6214189529418945, + 4.55186653137207, + 4.467422962188721, + 4.510173320770264, + 4.584484577178955, + 4.472749710083008, + 4.437396049499512, + 4.343541145324707, + 4.431820869445801, + 4.491738319396973, + 4.353819370269775, + 4.4703874588012695, + 4.41220760345459, + 4.504807472229004, + 4.379476070404053, + 4.280885219573975, + 4.335453987121582, + 4.491897106170654, + 4.446074962615967, + 4.300631999969482, + 4.329628944396973, + 4.40028715133667, + 4.443387985229492, + 4.271405220031738, + 4.249364852905273, + 4.328045845031738, + 4.358150959014893, + 4.262266635894775, + 4.282383918762207, + 4.303577899932861, + 4.351038455963135, + 4.277433395385742, + 4.263005256652832, + 4.183679103851318, + 4.203700542449951, + 4.232616901397705, + 4.2374677658081055, + 4.305685997009277, + 4.317414283752441, + 4.2443671226501465, + 4.285706996917725, + 4.175291061401367, + 4.354269981384277, + 4.214541912078857, + 4.270571231842041, + 4.153130054473877, + 4.228610515594482, + 4.186272144317627, + 4.255548000335693, + 4.285386085510254, + 4.246230602264404, + 4.232132911682129, + 4.305041313171387, + 4.332159042358398, + 4.259730815887451, + 4.246347904205322, + 4.31276273727417, + 4.266134738922119, + 4.174798488616943, + 4.216351509094238, + 4.210314750671387, + 4.20986270904541, + 4.151453495025635, + 4.216012001037598, + 4.173055171966553, + 4.182376861572266, + 4.2237324714660645, + 4.112725257873535, + 4.21549129486084, + 4.178603172302246, + 4.191834926605225, + 4.120138645172119, + 4.086332321166992, + 4.2451605796813965, + 4.063835620880127, + 4.274438381195068, + 4.246479034423828, + 4.16621732711792, + 4.23756217956543, + 4.168667793273926, + 4.165210247039795, + 3.9325060844421387, + 4.103819847106934, + 4.134143829345703, + 4.206457138061523, + 4.048749923706055, + 4.0751423835754395, + 4.205057144165039, + 4.179512977600098, + 4.100666046142578, + 4.1619648933410645, + 4.148053169250488, + 4.014719009399414, + 4.239392280578613, + 4.178881645202637, + 4.078073024749756, + 4.106055736541748, + 4.059823513031006, + 4.095085620880127, + 4.1679253578186035, + 4.135420322418213, + 4.173664093017578, + 4.245480537414551, + 4.079254627227783, + 4.064966678619385, + 4.101781845092773, + 4.1219072341918945, + 4.167086124420166, + 4.008824348449707, + 4.114899635314941, + 4.091726779937744, + 4.059077262878418, + 4.145312786102295, + 4.207911014556885, + 4.094513416290283, + 4.095384120941162, + 4.091238975524902, + 4.124475955963135, + 4.210586071014404, + 4.168574810028076, + 4.049658298492432, + 4.008020877838135, + 3.9620261192321777, + 4.014838218688965, + 4.070882797241211, + 4.089173316955566, + 4.029113292694092, + 4.031198501586914, + 4.035125255584717, + 3.999835968017578, + 4.0114312171936035, + 4.0479817390441895, + 4.06632661819458, + 4.091841220855713, + 4.005026817321777, + 4.065392971038818, + 4.032188415527344, + 4.03250789642334, + 4.065145492553711, + 4.059332370758057, + 4.02625846862793, + 3.9494264125823975, + 4.041890621185303, + 3.93782114982605, + 3.958674430847168, + 4.107670783996582, + 4.129201889038086, + 3.973830461502075, + 4.007946968078613, + 4.148512840270996, + 3.9347610473632812, + 4.134629726409912, + 3.9736948013305664, + 4.004649639129639, + 3.9745233058929443, + 4.049681186676025, + 3.962296962738037, + 4.08925199508667, + 4.059412956237793, + 4.039823532104492, + 3.9948136806488037, + 3.960664987564087, + 3.96563720703125, + 4.063101768493652, + 3.9095776081085205, + 3.941939353942871, + 4.001118183135986, + 4.012603282928467, + 3.96519136428833, + 3.97406005859375, + 4.024648666381836, + 3.9637837409973145, + 4.011573314666748, + 3.9119203090667725, + 3.880028009414673, + 4.129704475402832, + 4.006293773651123, + 3.9797277450561523, + 4.052349090576172, + 3.944124698638916, + 3.9503931999206543, + 4.0246782302856445, + 3.9966273307800293, + 3.8571159839630127, + 4.093890190124512, + 3.947680711746216, + 4.0753326416015625, + 3.9879300594329834, + 3.8969967365264893, + 3.973177671432495, + 3.9747424125671387, + 3.853630542755127, + 4.093318462371826, + 4.03069543838501, + 4.037137508392334, + 3.931558609008789, + 3.9592530727386475, + 3.9700520038604736, + 4.064608097076416, + 3.9622607231140137, + 3.9461727142333984, + 3.9379594326019287, + 3.866910696029663, + 3.9540557861328125, + 4.040105819702148, + 3.9420125484466553, + 3.9387309551239014, + 3.931595802307129, + 3.8476264476776123, + 3.8463222980499268, + 3.792579174041748, + 3.8528025150299072, + 4.097377300262451, + 3.9426846504211426, + 4.087845325469971, + 3.934415817260742, + 3.96411395072937, + 3.9583427906036377, + 3.9253251552581787, + 3.8782594203948975, + 3.9333817958831787, + 3.982140064239502, + 3.8876101970672607, + 4.0055036544799805, + 3.92677903175354, + 3.9416191577911377, + 3.9089791774749756, + 3.9999241828918457, + 3.964343547821045, + 3.902432680130005, + 3.9392178058624268, + 3.8855698108673096, + 3.900034189224243, + 3.9260659217834473, + 3.860299587249756, + 3.941798210144043, + 3.7894809246063232, + 3.952967643737793, + 3.974371910095215, + 4.02437162399292, + 3.8628249168395996, + 3.97068452835083, + 3.8504886627197266, + 3.9934499263763428, + 3.964848041534424, + 3.9993503093719482, + 3.9364326000213623, + 3.9072117805480957, + 4.042708396911621, + 3.974551200866699, + 3.8608202934265137, + 3.890439510345459, + 3.7956085205078125, + 3.972724437713623, + 3.905886173248291, + 3.9163830280303955, + 3.9346022605895996, + 3.9352076053619385, + 3.885939121246338, + 3.97786021232605, + 3.8715054988861084, + 3.872596025466919, + 3.839188814163208, + 3.973506212234497, + 3.8466084003448486, + 3.9335720539093018, + 3.8988864421844482, + 3.830573320388794, + 3.920635223388672, + 3.8226478099823, + 3.7910847663879395, + 3.8585238456726074, + 3.9079320430755615, + 3.9026482105255127, + 3.8428738117218018, + 3.9318180084228516, + 3.9972987174987793, + 3.9408624172210693, + 3.86848521232605, + 3.738312244415283, + 3.7809579372406006, + 3.855332851409912, + 3.8253555297851562, + 3.7758426666259766, + 3.833390474319458, + 3.906761646270752, + 3.836655378341675, + 3.8028504848480225, + 3.8212790489196777, + 3.7592077255249023, + 3.8493616580963135, + 3.8023760318756104, + 3.6894631385803223, + 3.843820571899414, + 3.870211124420166, + 3.8374223709106445, + 3.778167724609375, + 3.793698310852051, + 3.835210084915161, + 3.8658742904663086, + 3.759563446044922, + 3.7623414993286133, + 3.8754990100860596, + 3.84390926361084, + 3.8901169300079346, + 3.8413538932800293, + 3.7497458457946777, + 3.7896840572357178, + 3.727095365524292, + 3.8109281063079834, + 3.8288090229034424, + 3.865823745727539, + 3.885669708251953, + 3.8332273960113525, + 3.6905431747436523, + 3.912646532058716, + 3.7878458499908447, + 3.649399518966675, + 3.9029135704040527, + 3.6342995166778564, + 3.7327959537506104, + 3.8682701587677, + 3.8991174697875977, + 3.760098934173584, + 3.8749332427978516, + 3.7246932983398438, + 3.7684972286224365, + 3.8178107738494873, + 3.9320595264434814, + 3.862825393676758, + 3.72975492477417, + 3.78031587600708, + 3.8076915740966797, + 3.8364157676696777, + 3.7700412273406982, + 3.8666117191314697, + 3.6892807483673096, + 3.8047475814819336, + 3.864828109741211, + 3.7191648483276367, + 3.8033223152160645, + 3.700695753097534, + 3.818937301635742, + 3.678900718688965, + 3.762148141860962, + 3.7216522693634033, + 3.745450735092163, + 3.879706621170044, + 3.76609468460083, + 3.802246570587158, + 3.7393171787261963, + 3.820207118988037, + 3.6854569911956787, + 3.7958598136901855, + 3.7903478145599365, + 3.742321729660034, + 3.799348831176758, + 3.720726251602173, + 3.785295009613037, + 3.7728941440582275, + 3.823831081390381, + 3.8169944286346436, + 3.665544271469116, + 3.918449640274048, + 3.7446553707122803, + 3.873774528503418, + 3.9167706966400146, + 3.832322120666504, + 3.790400505065918, + 3.7994608879089355, + 3.8586227893829346, + 3.758279323577881, + 3.7518749237060547, + 3.828005790710449, + 3.739234447479248, + 3.8754658699035645, + 3.8098392486572266, + 3.745009422302246, + 3.7232537269592285, + 3.902148485183716, + 3.8098058700561523, + 3.741471767425537, + 3.872396230697632, + 3.8440349102020264, + 3.8002586364746094, + 3.709829568862915, + 3.7574665546417236, + 3.7429628372192383, + 3.7937989234924316, + 3.670292377471924, + 3.5828239917755127, + 3.84032940864563, + 3.882396936416626, + 3.7957375049591064, + 3.7423183917999268, + 3.832141399383545, + 3.6659255027770996, + 3.8209707736968994, + 3.688850164413452, + 3.946892023086548, + 3.8208587169647217, + 3.811128616333008, + 3.8923916816711426, + 3.8047685623168945, + 3.7771568298339844, + 3.7059366703033447, + 3.521522045135498, + 3.6896090507507324, + 3.6987133026123047, + 3.610067367553711, + 3.7469801902770996, + 3.7378199100494385, + 3.7803008556365967, + 3.665388584136963, + 3.716895580291748, + 3.789802074432373, + 3.673088550567627, + 3.6666460037231445, + 3.751516819000244, + 3.75115966796875, + 3.7983505725860596, + 3.7772300243377686, + 3.829195976257324, + 3.6679368019104004, + 3.771296262741089, + 3.7245402336120605, + 3.7344870567321777, + 3.6741647720336914, + 3.7451229095458984, + 3.778046131134033, + 3.7505643367767334, + 3.8261122703552246, + 3.7537100315093994, + 3.594792366027832, + 3.7627627849578857, + 3.801736831665039, + 3.7871644496917725, + 3.6431548595428467, + 3.6647534370422363, + 3.819732666015625, + 3.7785542011260986, + 3.7179694175720215, + 3.6748831272125244, + 3.7604126930236816, + 3.745986223220825, + 3.6657543182373047, + 3.683412790298462, + 3.7564070224761963, + 3.653979539871216, + 3.6827707290649414, + 3.767106771469116, + 3.64794921875, + 3.722311496734619, + 3.6184282302856445, + 3.705770969390869, + 3.7667698860168457, + 3.687499761581421, + 3.5707528591156006, + 3.633683919906616, + 3.7858529090881348, + 3.6810784339904785, + 3.6896634101867676, + 3.6283791065216064, + 3.6640586853027344, + 3.673827886581421, + 3.644648790359497, + 3.6816658973693848, + 3.7183139324188232, + 3.761087417602539, + 3.645986557006836, + 3.6964738368988037, + 3.758737325668335, + 3.7641801834106445, + 3.8954195976257324, + 3.709258794784546, + 3.6150283813476562, + 3.7583062648773193, + 3.6616809368133545, + 3.695943832397461, + 3.7599406242370605, + 3.66339373588562, + 3.7932541370391846, + 3.7904887199401855, + 3.626505136489868, + 3.816884994506836, + 3.6376442909240723, + 3.7592201232910156, + 3.7209410667419434, + 3.605405330657959, + 3.7078421115875244, + 3.871955633163452, + 3.7410948276519775, + 3.7016069889068604, + 3.6824707984924316, + 3.7866647243499756, + 3.6778042316436768, + 3.8175911903381348, + 3.572200059890747, + 3.682751178741455, + 3.6944942474365234, + 3.7036349773406982, + 3.710646629333496, + 3.5760610103607178, + 3.67020320892334, + 3.6473581790924072, + 3.8094284534454346, + 3.520533561706543, + 3.765627384185791, + 3.695969581604004, + 3.62396502494812, + 3.7516555786132812, + 3.7003300189971924, + 3.6980507373809814, + 3.718859910964966, + 3.750243902206421, + 3.5154144763946533, + 3.885136127471924, + 3.763103485107422, + 3.6468923091888428, + 3.6993327140808105, + 3.8152389526367188, + 3.530635356903076, + 3.5997157096862793, + 3.7090375423431396, + 3.556335210800171, + 3.6869707107543945, + 3.7328808307647705, + 3.6113762855529785, + 3.8732309341430664, + 3.7323596477508545, + 3.765807867050171, + 3.679490804672241, + 3.627500534057617, + 3.735565662384033, + 3.6262381076812744, + 3.718190908432007, + 3.6276538372039795, + 3.6093506813049316, + 3.4148030281066895, + 3.623784303665161, + 3.7199738025665283, + 3.6705665588378906, + 3.532569408416748, + 3.650242567062378, + 3.6662778854370117, + 3.7119102478027344, + 3.669402599334717, + 3.59818959236145, + 3.587561845779419, + 3.6964223384857178, + 3.6757867336273193, + 3.5644495487213135, + 3.7907841205596924, + 3.6978626251220703, + 3.7294094562530518, + 3.593512773513794, + 3.7108116149902344, + 3.666973352432251, + 3.5507943630218506, + 3.5719120502471924, + 3.596083879470825, + 3.628962755203247, + 3.6353800296783447, + 3.701634407043457, + 3.740324020385742, + 3.727634906768799, + 3.6179566383361816, + 3.5358760356903076, + 3.7797672748565674, + 3.579440116882324, + 3.4948155879974365, + 3.6129820346832275, + 3.823667526245117, + 3.6534085273742676, + 3.4860639572143555, + 3.596996307373047, + 3.5926952362060547, + 3.7963380813598633, + 3.6461803913116455, + 3.6781082153320312, + 3.5506443977355957, + 3.5994038581848145, + 3.6486494541168213, + 3.5760130882263184, + 3.584845542907715, + 3.662600040435791, + 3.6524534225463867, + 3.6561472415924072, + 3.7774388790130615, + 3.6948232650756836, + 3.61517596244812, + 3.613158702850342, + 3.6074163913726807, + 3.4899542331695557, + 3.6168696880340576, + 3.672717809677124, + 3.6719448566436768, + 3.584880828857422, + 3.5771677494049072, + 3.5827829837799072, + 3.7294182777404785, + 3.7464113235473633, + 3.5573348999023438, + 3.6233320236206055, + 3.575618028640747, + 3.700363874435425, + 3.726654052734375, + 3.623103141784668, + 3.6649513244628906, + 3.733830213546753, + 3.558283805847168, + 3.662128210067749, + 3.5366628170013428, + 3.640162706375122, + 3.5698318481445312, + 3.549386739730835, + 3.552757501602173, + 3.5244455337524414, + 3.6004152297973633, + 3.5689589977264404, + 3.6240673065185547, + 3.6733176708221436, + 3.610119104385376, + 3.6057417392730713, + 3.651623010635376, + 3.6228742599487305, + 3.4820423126220703, + 3.6174466609954834, + 3.556246757507324, + 3.6774942874908447, + 3.6545701026916504, + 3.6971302032470703, + 3.5907528400421143, + 3.7268879413604736, + 3.5157454013824463, + 3.603001356124878, + 3.6588194370269775, + 3.545239210128784, + 3.718810558319092, + 3.6768109798431396, + 3.591566562652588, + 3.6801373958587646, + 3.5188236236572266, + 3.581815004348755, + 3.6033480167388916, + 3.5779519081115723, + 3.6689109802246094, + 3.623763084411621, + 3.629141092300415, + 3.613032579421997, + 3.6628525257110596, + 3.4295544624328613, + 3.5757322311401367, + 3.6658644676208496, + 3.6743886470794678, + 3.562262535095215, + 3.5744407176971436, + 3.642045497894287, + 3.67537260055542, + 3.4556374549865723, + 3.5693106651306152, + 3.5665791034698486, + 3.5239477157592773, + 3.5519585609436035, + 3.552445650100708, + 3.694115400314331, + 3.537144422531128, + 3.490180015563965, + 3.5646533966064453, + 3.6544175148010254, + 3.550797939300537, + 3.5462725162506104, + 3.4574451446533203, + 3.690957546234131, + 3.696014881134033, + 3.678016185760498, + 3.4819650650024414, + 3.426804304122925, + 3.6420364379882812, + 3.5162909030914307, + 3.6263651847839355, + 3.443309783935547, + 3.5458600521087646, + 3.615396738052368, + 3.6457934379577637, + 3.6876912117004395, + 3.599062919616699, + 3.4824719429016113, + 3.6531269550323486, + 3.6336209774017334, + 3.5655646324157715, + 3.651806592941284, + 3.68990421295166, + 3.556283473968506, + 3.500875234603882, + 3.6187398433685303, + 3.4798102378845215, + 3.5231497287750244, + 3.5467770099639893, + 3.625901699066162, + 3.5532948970794678, + 3.4343814849853516, + 3.6216557025909424, + 3.3768179416656494, + 3.702946901321411, + 3.57053279876709, + 3.450223445892334, + 3.508105516433716, + 3.4406871795654297, + 3.4621007442474365, + 3.5856215953826904, + 3.5670650005340576, + 3.6725635528564453, + 3.5446624755859375, + 3.498483896255493, + 3.684809684753418, + 3.6559314727783203, + 3.5736379623413086, + 3.656540632247925, + 3.6091558933258057, + 3.596689462661743, + 3.6966280937194824, + 3.5366928577423096, + 3.560046911239624, + 3.4449572563171387, + 3.412112236022949, + 3.6192495822906494, + 3.4516568183898926, + 3.5562171936035156, + 3.4685239791870117, + 3.674501895904541, + 3.53657603263855, + 3.5804738998413086, + 3.670305013656616, + 3.557936429977417, + 3.618065118789673, + 3.4956717491149902, + 3.572664260864258, + 3.475564479827881, + 3.626636028289795, + 3.5777084827423096, + 3.616602897644043, + 3.592941999435425, + 3.5524306297302246, + 3.597883701324463, + 3.654306173324585, + 3.521121025085449, + 3.6339969635009766, + 3.5523321628570557, + 3.5414555072784424, + 3.558828353881836, + 3.3768067359924316, + 3.548768997192383, + 3.5690245628356934, + 3.4884984493255615, + 3.554685592651367, + 3.562307119369507, + 3.440047025680542, + 3.609767436981201, + 3.469958782196045, + 3.618603467941284, + 3.5596184730529785, + 3.5604820251464844, + 3.5279581546783447, + 3.4609363079071045, + 3.5463788509368896, + 3.5819344520568848, + 3.5468297004699707, + 3.529895544052124, + 3.653585195541382, + 3.6463162899017334, + 3.540005683898926, + 3.631432294845581, + 3.631936550140381, + 3.5856521129608154, + 3.5171074867248535, + 3.5600829124450684, + 3.605588674545288, + 3.6139838695526123, + 3.487546920776367, + 3.6306827068328857, + 3.5526421070098877, + 3.611138343811035, + 3.626763105392456, + 3.5627498626708984, + 3.682241678237915, + 3.4952588081359863, + 3.536578416824341, + 3.570730447769165, + 3.509838342666626, + 3.455132007598877, + 3.6315553188323975, + 3.5451080799102783, + 3.651594638824463, + 3.6490180492401123, + 3.5889499187469482, + 3.4843533039093018, + 3.5523715019226074, + 3.484560489654541, + 3.619380474090576, + 3.566801071166992, + 3.438283681869507, + 3.419471263885498, + 3.590440034866333, + 3.57112979888916, + 3.7072532176971436, + 3.5711848735809326, + 3.457378625869751, + 3.439466953277588, + 3.6446173191070557, + 3.6054601669311523, + 3.4676456451416016, + 3.517399787902832, + 3.5794217586517334, + 3.5954787731170654, + 3.428565502166748, + 3.5411369800567627, + 3.4492502212524414, + 3.555419445037842, + 3.431767225265503, + 3.377821207046509, + 3.5629615783691406, + 3.5811750888824463, + 3.6676249504089355, + 3.3404905796051025, + 3.612919569015503, + 3.4564108848571777, + 3.495142698287964, + 3.400418758392334, + 3.6779232025146484, + 3.6304643154144287, + 3.518880605697632, + 3.3607325553894043, + 3.562434673309326, + 3.5794730186462402, + 3.594759225845337, + 3.5088891983032227, + 3.4729278087615967, + 3.4585535526275635, + 3.4985511302948, + 3.499603509902954, + 3.611227035522461, + 3.5834543704986572, + 3.582066535949707, + 3.544367551803589, + 3.54018497467041, + 3.443934440612793, + 3.639453172683716, + 3.568903923034668, + 3.5231411457061768, + 3.547100305557251, + 3.661076068878174, + 3.582918643951416, + 3.6705245971679688, + 3.4770658016204834, + 3.504185438156128, + 3.516697883605957, + 3.4729466438293457, + 3.6648356914520264, + 3.5796492099761963, + 3.513038158416748, + 3.5610947608947754, + 3.497185707092285, + 3.5091910362243652, + 3.586928129196167, + 3.419557571411133, + 3.6269853115081787, + 3.3584322929382324, + 3.407310724258423, + 3.507556676864624, + 3.552927255630493, + 3.39776349067688, + 3.5134711265563965, + 3.5099151134490967, + 3.5406174659729004, + 3.372736930847168, + 3.591682195663452, + 3.610107660293579, + 3.6297967433929443, + 3.6114158630371094, + 3.5268197059631348, + 3.4635071754455566, + 3.5039050579071045, + 3.5039970874786377, + 3.4844272136688232, + 3.5658769607543945, + 3.5618793964385986, + 3.653576135635376, + 3.5329794883728027, + 3.424314260482788, + 3.528225898742676, + 3.5332086086273193, + 3.524200201034546, + 3.4443159103393555, + 3.519071578979492, + 3.5394158363342285, + 3.5156431198120117, + 3.419664144515991, + 3.4997856616973877, + 3.6462438106536865 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 128.353210313958, + 80.99378271379825, + 64.65858075034895, + 56.48904265262075, + 51.869881432577536, + 48.65004086705287, + 46.70572063942119, + 44.615989205519085, + 43.2864227310968, + 41.34483903137775, + 40.39880021893856, + 39.50442407550911, + 38.930160637363485, + 37.843589309278556, + 37.23142708786259, + 36.92427064088659, + 36.534061772711205, + 36.377446987535706, + 36.30099883259372, + 36.27628780892663 + ], + "best_ppl": 36.27628780892663, + "train_time_s": 7526.138844013214 +} \ No newline at end of file diff --git a/063_sparse_dispatch/single_block_softmax_best.pt b/063_sparse_dispatch/single_block_softmax_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..f67850c36610870ea80079a307bbca921cfb6cf1 --- /dev/null +++ b/063_sparse_dispatch/single_block_softmax_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07c8470944ffa6944c2b23cd2b0a3c54536bcc7cd1fba946a54b0a7b2cca860e +size 173157531 diff --git a/063_sparse_dispatch/single_block_softmax_d1024_best.pt b/063_sparse_dispatch/single_block_softmax_d1024_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..2428c8457c52602fc1389a58433fbcd6536e5c90 --- /dev/null +++ b/063_sparse_dispatch/single_block_softmax_d1024_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75e9b074e429aac56e26f984666c80b72768b59c934f2661323f6e053bed4b50 +size 601124769 diff --git a/063_sparse_dispatch/single_block_softmax_d1024_results.json b/063_sparse_dispatch/single_block_softmax_d1024_results.json new file mode 100644 index 0000000000000000000000000000000000000000..c534a2c789b62c9f1b4c8be7f45ea294a2879204 --- /dev/null +++ b/063_sparse_dispatch/single_block_softmax_d1024_results.json @@ -0,0 +1,2057 @@ +{ + "label": "single_block_softmax_d1024", + "seed": 42, + "attention_type": "softmax", + "d_expert": 1024, + "total_params": 150267904, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 9.272101879119873, + 7.860560655593872, + 7.169002056121826, + 6.93440055847168, + 6.7241950035095215, + 6.578324317932129, + 6.361756324768066, + 6.2148377895355225, + 6.19636869430542, + 6.026845932006836, + 5.794589996337891, + 5.737905979156494, + 5.765958786010742, + 5.709526538848877, + 5.732246398925781, + 5.552887439727783, + 5.516457796096802, + 5.416062355041504, + 5.420290946960449, + 5.2768354415893555, + 5.19947624206543, + 5.087990999221802, + 5.228975057601929, + 5.059628963470459, + 5.017563581466675, + 4.90529465675354, + 4.9171552658081055, + 5.02549147605896, + 5.0134437084198, + 4.9882941246032715, + 4.987915515899658, + 4.80518651008606, + 4.877310514450073, + 4.7853968143463135, + 4.684232950210571, + 4.845152854919434, + 4.72679591178894, + 4.685853719711304, + 4.581024408340454, + 4.643207788467407, + 4.6510725021362305, + 4.546227693557739, + 4.528721332550049, + 4.69590950012207, + 4.60714054107666, + 4.552262306213379, + 4.542917013168335, + 4.528591156005859, + 4.493139982223511, + 4.468529462814331, + 4.521458387374878, + 4.488165378570557, + 4.391049385070801, + 4.458779335021973, + 4.496952772140503, + 4.54688024520874, + 4.356834888458252, + 4.346603870391846, + 4.3202736377716064, + 4.406907558441162, + 4.46726131439209, + 4.377482652664185, + 4.3324291706085205, + 4.345064878463745, + 4.301205158233643, + 4.307714462280273, + 4.2621331214904785, + 4.414929151535034, + 4.415709018707275, + 4.332139253616333, + 4.286692380905151, + 4.380311012268066, + 4.273466348648071, + 4.26711893081665, + 4.16317081451416, + 4.29379940032959, + 4.249238014221191, + 4.2555413246154785, + 4.296803951263428, + 4.29789924621582, + 4.262696266174316, + 4.220317363739014, + 4.236256837844849, + 4.201387643814087, + 4.176276206970215, + 4.071049213409424, + 4.244556903839111, + 4.129770994186401, + 4.189543724060059, + 4.312989950180054, + 4.231348037719727, + 4.145920515060425, + 4.15660548210144, + 4.1014721393585205, + 4.153640508651733, + 4.076190233230591, + 4.152151346206665, + 3.9466923475265503, + 4.2520904541015625, + 4.035136342048645, + 4.1620635986328125, + 4.169872760772705, + 4.169820308685303, + 4.22568678855896, + 4.176410675048828, + 4.084896087646484, + 4.020916819572449, + 4.1053385734558105, + 4.167127370834351, + 4.062385201454163, + 4.081047296524048, + 4.119703531265259, + 4.083644866943359, + 4.20529842376709, + 4.17113471031189, + 4.14654541015625, + 4.11666464805603, + 4.1294190883636475, + 4.126672983169556, + 4.01616096496582, + 4.069316387176514, + 4.129845142364502, + 4.051361083984375, + 4.009494781494141, + 4.072387337684631, + 4.149457931518555, + 3.980984926223755, + 3.919050931930542, + 4.052490711212158, + 3.8718769550323486, + 3.97653329372406, + 4.071184396743774, + 3.9192886352539062, + 3.937206745147705, + 4.0254738330841064, + 4.052783250808716, + 4.106449604034424, + 3.9229706525802612, + 3.903119683265686, + 3.960407853126526, + 3.938952922821045, + 3.874719500541687, + 4.01361358165741, + 3.9735939502716064, + 4.0060659646987915, + 3.9764246940612793, + 4.049669861793518, + 3.9621697664260864, + 4.031453847885132, + 3.865084409713745, + 4.012723445892334, + 3.9282559156417847, + 3.815226435661316, + 3.9943950176239014, + 3.9580310583114624, + 3.8528155088424683, + 3.7980250120162964, + 3.8763041496276855, + 3.960336208343506, + 3.9809459447860718, + 3.8876973390579224, + 3.970724582672119, + 3.8877464532852173, + 3.9340115785598755, + 3.8552613258361816, + 4.055204510688782, + 3.837889075279236, + 3.9564428329467773, + 3.971294641494751, + 3.904070258140564, + 3.7708104848861694, + 4.011729717254639, + 3.960753560066223, + 3.957700252532959, + 3.787307381629944, + 3.9593676328659058, + 3.9282004833221436, + 3.861387610435486, + 3.905324339866638, + 3.8777800798416138, + 3.8656095266342163, + 3.881413459777832, + 3.904518961906433, + 3.935953974723816, + 4.030388355255127, + 3.917925238609314, + 3.953961491584778, + 3.8835227489471436, + 3.865733027458191, + 4.068814039230347, + 3.937166213989258, + 3.844667911529541, + 3.900136709213257, + 3.799823522567749, + 3.8971755504608154, + 3.9729301929473877, + 3.8291962146759033, + 3.8400797843933105, + 3.806509017944336, + 3.99366295337677, + 3.879928469657898, + 3.9610294103622437, + 3.7980366945266724, + 3.759400248527527, + 3.758566379547119, + 3.7584710121154785, + 3.846250295639038, + 3.8149051666259766, + 3.9540677070617676, + 3.8065345287323, + 3.922642230987549, + 3.827336072921753, + 3.848344326019287, + 3.831353783607483, + 3.7704734802246094, + 3.9554272890090942, + 3.8989113569259644, + 3.8636475801467896, + 3.8986735343933105, + 3.79794442653656, + 3.75486958026886, + 4.044932842254639, + 3.8359291553497314, + 3.837294340133667, + 3.7961889505386353, + 3.8551619052886963, + 3.6365169286727905, + 3.814913511276245, + 3.7947839498519897, + 3.924357056617737, + 3.7635252475738525, + 3.822510242462158, + 3.945937156677246, + 3.8627524375915527, + 3.838129758834839, + 3.8670175075531006, + 3.8491233587265015, + 3.6500335931777954, + 3.8738266229629517, + 3.8918228149414062, + 3.8136751651763916, + 3.750253915786743, + 3.8419017791748047, + 3.73668897151947, + 3.747307538986206, + 3.902040719985962, + 3.904673218727112, + 3.9952504634857178, + 3.7603647708892822, + 3.689355731010437, + 3.8819659948349, + 3.767258405685425, + 3.863718032836914, + 3.6858835220336914, + 3.664267659187317, + 3.6523256301879883, + 3.811256170272827, + 3.7384992837905884, + 3.777205228805542, + 3.7750072479248047, + 3.7235394716262817, + 3.817634344100952, + 3.662123441696167, + 3.77934992313385, + 3.7229591608047485, + 3.7995142936706543, + 3.640865206718445, + 3.833022713661194, + 3.638184070587158, + 3.676672577857971, + 3.6193279027938843, + 3.8487741947174072, + 3.6710402965545654, + 3.70063316822052, + 3.725796937942505, + 3.730581760406494, + 3.6825873851776123, + 3.6103662252426147, + 3.64507794380188, + 3.6245694160461426, + 3.6315020322799683, + 3.635278820991516, + 3.7350263595581055, + 3.7494008541107178, + 3.746995210647583, + 3.7237976789474487, + 3.5927681922912598, + 3.80999755859375, + 3.7121453285217285, + 3.6091437339782715, + 3.718796491622925, + 3.625866174697876, + 3.6443711519241333, + 3.6629319190979004, + 3.85131573677063, + 3.866716742515564, + 3.7422584295272827, + 3.7694870233535767, + 3.6570775508880615, + 3.6333296298980713, + 3.6579734086990356, + 3.713480830192566, + 3.6699451208114624, + 3.658406138420105, + 3.7352386713027954, + 3.7731711864471436, + 3.8457725048065186, + 3.6396875381469727, + 3.706411600112915, + 3.591334819793701, + 3.8533700704574585, + 3.665010929107666, + 3.7117620706558228, + 3.629117488861084, + 3.7648065090179443, + 3.6812673807144165, + 3.726511597633362, + 3.7184488773345947, + 3.580386757850647, + 3.6516315937042236, + 3.7126694917678833, + 3.864526867866516, + 3.6859078407287598, + 3.7683584690093994, + 3.7422561645507812, + 3.5867356061935425, + 3.7150379419326782, + 3.740309953689575, + 3.67820942401886, + 3.6822588443756104, + 3.7025376558303833, + 3.6983487606048584, + 3.6828290224075317, + 3.748995542526245, + 3.6655341386795044, + 3.6183334589004517, + 3.7071433067321777, + 3.692036747932434, + 3.7126057147979736, + 3.563672661781311, + 3.610448956489563, + 3.741052985191345, + 3.5178463459014893, + 3.5079206228256226, + 3.681795120239258, + 3.671902894973755, + 3.60478138923645, + 3.649927258491516, + 3.647451162338257, + 3.7901999950408936, + 3.676923990249634, + 3.69764244556427, + 3.6779686212539673, + 3.747049331665039, + 3.7500652074813843, + 3.638860821723938, + 3.6794278621673584, + 3.577895402908325, + 3.608266830444336, + 3.7660547494888306, + 3.6425307989120483, + 3.787117123603821, + 3.7207032442092896, + 3.7153836488723755, + 3.6928462982177734, + 3.4767431020736694, + 3.59644877910614, + 3.7292076349258423, + 3.64709210395813, + 3.6239315271377563, + 3.615291953086853, + 3.5473712682724, + 3.585703730583191, + 3.7673734426498413, + 3.604311943054199, + 3.5420225858688354, + 3.700986862182617, + 3.6357815265655518, + 3.7642189264297485, + 3.496850848197937, + 3.558433771133423, + 3.5364413261413574, + 3.605795741081238, + 3.498337507247925, + 3.501926898956299, + 3.4616973400115967, + 3.5698978900909424, + 3.5374808311462402, + 3.5721596479415894, + 3.3682864904403687, + 3.5299086570739746, + 3.495510458946228, + 3.4728811979293823, + 3.575352430343628, + 3.606412887573242, + 3.435649275779724, + 3.587427258491516, + 3.5545542240142822, + 3.708107829093933, + 3.408703327178955, + 3.5178606510162354, + 3.5444881916046143, + 3.494866371154785, + 3.5934430360794067, + 3.5024945735931396, + 3.5071645975112915, + 3.497949481010437, + 3.5791540145874023, + 3.4455933570861816, + 3.597129464149475, + 3.535151243209839, + 3.6558237075805664, + 3.519215703010559, + 3.5033611059188843, + 3.4739638566970825, + 3.544298529624939, + 3.5481364727020264, + 3.5363306999206543, + 3.498144507408142, + 3.4930492639541626, + 3.5922679901123047, + 3.6117578744888306, + 3.5157140493392944, + 3.6641292572021484, + 3.5146639347076416, + 3.57042133808136, + 3.598854422569275, + 3.528545618057251, + 3.4703378677368164, + 3.6417369842529297, + 3.4578176736831665, + 3.4611021280288696, + 3.518831968307495, + 3.597840189933777, + 3.4821068048477173, + 3.613345980644226, + 3.5952552556991577, + 3.530089020729065, + 3.4300482273101807, + 3.6418752670288086, + 3.5065417289733887, + 3.5463379621505737, + 3.403946280479431, + 3.4651137590408325, + 3.4836288690567017, + 3.383508086204529, + 3.6089929342269897, + 3.5603721141815186, + 3.4961167573928833, + 3.6111719608306885, + 3.5679882764816284, + 3.5208113193511963, + 3.537649631500244, + 3.5948543548583984, + 3.5751761198043823, + 3.4895583391189575, + 3.4360965490341187, + 3.5341118574142456, + 3.600873827934265, + 3.5622137784957886, + 3.596688389778137, + 3.4771249294281006, + 3.6811134815216064, + 3.48045814037323, + 3.5738338232040405, + 3.4573888778686523, + 3.440732717514038, + 3.581368565559387, + 3.4877562522888184, + 3.491556763648987, + 3.650643825531006, + 3.5317819118499756, + 3.3739278316497803, + 3.4905850887298584, + 3.53669011592865, + 3.537393093109131, + 3.480767846107483, + 3.554098129272461, + 3.587851405143738, + 3.5646528005599976, + 3.439915180206299, + 3.6357085704803467, + 3.4921048879623413, + 3.429112434387207, + 3.4121525287628174, + 3.520266890525818, + 3.510532021522522, + 3.598973274230957, + 3.6596473455429077, + 3.6923136711120605, + 3.4797136783599854, + 3.5549172163009644, + 3.463269352912903, + 3.5692027807235718, + 3.531264066696167, + 3.425101399421692, + 3.5803943872451782, + 3.5155932903289795, + 3.5673967599868774, + 3.5623743534088135, + 3.6715428829193115, + 3.471966862678528, + 3.4971874952316284, + 3.7327624559402466, + 3.5141557455062866, + 3.4775294065475464, + 3.443053126335144, + 3.484304428100586, + 3.3401888608932495, + 3.35932719707489, + 3.371402382850647, + 3.2446396350860596, + 3.432255506515503, + 3.36638867855072, + 3.2777897119522095, + 3.3794692754745483, + 3.4912633895874023, + 3.3018412590026855, + 3.285965323448181, + 3.2187154293060303, + 3.4621556997299194, + 3.4654229879379272, + 3.4959821701049805, + 3.350814938545227, + 3.4645583629608154, + 3.381782650947571, + 3.5062150955200195, + 3.46975839138031, + 3.4150420427322388, + 3.5620027780532837, + 3.3702216148376465, + 3.504266381263733, + 3.5160810947418213, + 3.441877007484436, + 3.4055134057998657, + 3.3521727323532104, + 3.403165102005005, + 3.4406381845474243, + 3.315973401069641, + 3.3320478200912476, + 3.4525387287139893, + 3.2912977933883667, + 3.4490562677383423, + 3.252925395965576, + 3.4293277263641357, + 3.377442717552185, + 3.453391194343567, + 3.5134321451187134, + 3.4261045455932617, + 3.386834979057312, + 3.4790825843811035, + 3.476864218711853, + 3.5254122018814087, + 3.3050060272216797, + 3.406542181968689, + 3.4044668674468994, + 3.3856672048568726, + 3.4993399381637573, + 3.3728623390197754, + 3.4566763639450073, + 3.4417178630828857, + 3.3759886026382446, + 3.3916385173797607, + 3.457616925239563, + 3.3594411611557007, + 3.439324140548706, + 3.3191996812820435, + 3.3654067516326904, + 3.401143193244934, + 3.253166913986206, + 3.4592312574386597, + 3.434057593345642, + 3.4771454334259033, + 3.3924988508224487, + 3.393420696258545, + 3.4100711345672607, + 3.4712454080581665, + 3.456377148628235, + 3.382288098335266, + 3.362746477127075, + 3.4120280742645264, + 3.2795286178588867, + 3.3719319105148315, + 3.225929856300354, + 3.427635669708252, + 3.470383048057556, + 3.385180950164795, + 3.341413736343384, + 3.3904061317443848, + 3.3121925592422485, + 3.4164716005325317, + 3.463615298271179, + 3.3359456062316895, + 3.5109810829162598, + 3.5431928634643555, + 3.3587396144866943, + 3.5141983032226562, + 3.2693374156951904, + 3.3471314907073975, + 3.3071130514144897, + 3.3577117919921875, + 3.3589935302734375, + 3.3732045888900757, + 3.384437918663025, + 3.3424068689346313, + 3.3253982067108154, + 3.3180360794067383, + 3.33793842792511, + 3.350278854370117, + 3.3384764194488525, + 3.323012113571167, + 3.3773657083511353, + 3.4479122161865234, + 3.3784762620925903, + 3.256579875946045, + 3.460708737373352, + 3.3180928230285645, + 3.3025448322296143, + 3.433267116546631, + 3.2823656797409058, + 3.358648419380188, + 3.413204312324524, + 3.246384859085083, + 3.2707016468048096, + 3.3471027612686157, + 3.4371352195739746, + 3.246955990791321, + 3.443014144897461, + 3.378725051879883, + 3.3138004541397095, + 3.2117658853530884, + 3.469993829727173, + 3.2823086977005005, + 3.2822084426879883, + 3.2293156385421753, + 3.3165007829666138, + 3.214832901954651, + 3.213230848312378, + 3.241747498512268, + 3.2084121704101562, + 3.328490138053894, + 3.3432456254959106, + 3.3038069009780884, + 3.3115692138671875, + 3.280377149581909, + 3.3435115814208984, + 3.3197312355041504, + 3.2350821495056152, + 3.1298733949661255, + 3.298687219619751, + 3.3167089223861694, + 3.27663254737854, + 3.1700392961502075, + 3.341988444328308, + 3.372471332550049, + 3.3290274143218994, + 3.270723819732666, + 3.3018858432769775, + 3.4583805799484253, + 3.298813581466675, + 3.3088364601135254, + 3.2858184576034546, + 3.3579033613204956, + 3.2855693101882935, + 3.318469524383545, + 3.24946129322052, + 3.3211787939071655, + 3.2089314460754395, + 3.2722007036209106, + 3.2838644981384277, + 3.2407877445220947, + 3.2192777395248413, + 3.30304491519928, + 3.3243132829666138, + 3.213157892227173, + 3.3210736513137817, + 3.28638756275177, + 3.2081117630004883, + 3.277215003967285, + 3.2797521352767944, + 3.3522356748580933, + 3.2276101112365723, + 3.281652569770813, + 3.3312569856643677, + 3.170490026473999, + 3.2435277700424194, + 3.2815659046173096, + 3.273890972137451, + 3.2128485441207886, + 3.286474108695984, + 3.3191287517547607, + 3.141629695892334, + 3.4126696586608887, + 3.267844319343567, + 3.3207151889801025, + 3.2632230520248413, + 3.196613907814026, + 3.22122859954834, + 3.243148684501648, + 3.297507166862488, + 3.2127087116241455, + 3.2485058307647705, + 3.329517960548401, + 3.2896589040756226, + 3.2839027643203735, + 3.2438851594924927, + 3.1590492725372314, + 3.326238989830017, + 3.355956196784973, + 3.320167899131775, + 3.2762224674224854, + 3.3012553453445435, + 3.3180967569351196, + 3.2307530641555786, + 3.28564190864563, + 3.2891902923583984, + 3.3242748975753784, + 3.2604596614837646, + 3.224661707878113, + 3.1581801176071167, + 3.314448356628418, + 3.1442418098449707, + 3.22011399269104, + 3.259108543395996, + 3.244624972343445, + 3.176912546157837, + 3.3058921098709106, + 3.2810537815093994, + 3.211781859397888, + 3.3082960844039917, + 3.3181209564208984, + 3.2177520990371704, + 3.2144988775253296, + 3.229526996612549, + 3.304457187652588, + 3.276467442512512, + 3.2902467250823975, + 3.3318289518356323, + 3.2301746606826782, + 3.4631826877593994, + 3.2367312908172607, + 3.2352300882339478, + 3.202677845954895, + 3.3393797874450684, + 3.2158496379852295, + 3.18318510055542, + 3.369109272956848, + 3.2851192951202393, + 3.3479186296463013, + 3.253711223602295, + 3.2108548879623413, + 3.3214610815048218, + 3.177009701728821, + 3.1076773405075073, + 3.2164804935455322, + 3.3409159183502197, + 3.219139575958252, + 3.3046746253967285, + 3.2379602193832397, + 3.190929412841797, + 2.998679518699646, + 3.195688843727112, + 3.1439603567123413, + 3.2691562175750732, + 3.1628814935684204, + 3.188140034675598, + 3.2958455085754395, + 3.1891294717788696, + 3.3164923191070557, + 3.258016586303711, + 3.209334373474121, + 3.182674288749695, + 3.163386583328247, + 3.2801941633224487, + 3.2796125411987305, + 3.2396857738494873, + 3.1545300483703613, + 3.2140891551971436, + 3.330628514289856, + 3.2182353734970093, + 3.1542577743530273, + 3.1078094244003296, + 3.145974636077881, + 3.0943816900253296, + 3.1391379833221436, + 3.1535284519195557, + 3.1307339668273926, + 3.2101293802261353, + 3.1792510747909546, + 3.1888808012008667, + 3.0951056480407715, + 3.231232166290283, + 3.2490419149398804, + 3.3045735359191895, + 3.256238102912903, + 3.096139669418335, + 3.174986481666565, + 3.079956889152527, + 3.272608518600464, + 3.1331698894500732, + 3.2142971754074097, + 3.296389937400818, + 3.1298744678497314, + 3.1211718320846558, + 3.119587540626526, + 3.1719383001327515, + 3.138343930244446, + 3.1842435598373413, + 3.2977532148361206, + 3.09515118598938, + 3.2024011611938477, + 3.253076434135437, + 3.1760882139205933, + 3.0995306968688965, + 3.2317556142807007, + 3.2417943477630615, + 3.330265164375305, + 3.237827181816101, + 3.185707926750183, + 3.0314806699752808, + 3.1686097383499146, + 3.1065001487731934, + 3.1711947917938232, + 3.2726364135742188, + 3.1887892484664917, + 3.224980115890503, + 3.250487208366394, + 3.1445300579071045, + 3.1441041231155396, + 3.2340770959854126, + 3.1968510150909424, + 3.1465489864349365, + 3.1950618028640747, + 3.101857900619507, + 3.1293431520462036, + 3.324749231338501, + 3.169557571411133, + 3.1599576473236084, + 3.1265385150909424, + 3.2918078899383545, + 3.1773751974105835, + 3.1177549362182617, + 3.2629846334457397, + 3.199776768684387, + 3.214223265647888, + 3.210376024246216, + 3.2088102102279663, + 3.2316619157791138, + 3.05996310710907, + 3.1988223791122437, + 3.1509824991226196, + 3.1619662046432495, + 3.1403990983963013, + 3.1645140647888184, + 3.2420732975006104, + 3.2707754373550415, + 3.1689422130584717, + 3.222382664680481, + 3.148323893547058, + 3.215282440185547, + 3.277600646018982, + 3.179237484931946, + 3.309724807739258, + 3.1638917922973633, + 3.1909713745117188, + 3.1979202032089233, + 3.176062822341919, + 3.1404095888137817, + 3.1368002891540527, + 3.1195045709609985, + 3.057785153388977, + 3.2726775407791138, + 3.1859962940216064, + 3.081219792366028, + 3.2416051626205444, + 3.136331558227539, + 3.231503129005432, + 3.1920297145843506, + 3.154814600944519, + 3.120661497116089, + 3.2177997827529907, + 3.0767897367477417, + 3.258674383163452, + 3.084138035774231, + 3.2732332944869995, + 3.104963541030884, + 3.064531445503235, + 3.2045915126800537, + 3.1208391189575195, + 3.147502303123474, + 3.0926713943481445, + 3.1037187576293945, + 3.2111481428146362, + 3.2534862756729126, + 3.2645307779312134, + 3.1579108238220215, + 3.195694923400879, + 3.155674695968628, + 3.107440948486328, + 3.1422892808914185, + 3.0738126039505005, + 3.228188157081604, + 3.1580487489700317, + 3.1231590509414673, + 3.1864479780197144, + 3.090542197227478, + 3.161063551902771, + 3.049091339111328, + 3.105429172515869, + 3.1190766096115112, + 3.202234983444214, + 3.1214019060134888, + 3.0377026796340942, + 3.0496665239334106, + 3.1372616291046143, + 3.0927988290786743, + 3.196441411972046, + 3.2363834381103516, + 3.1532464027404785, + 3.107580780982971, + 3.174399256706238, + 3.092729091644287, + 3.1874290704727173, + 3.132480025291443, + 3.187875986099243, + 3.130661129951477, + 3.0866793394088745, + 2.990231990814209, + 3.130237579345703, + 3.1993629932403564, + 3.1272021532058716, + 3.1480417251586914, + 3.104355216026306, + 3.1233948469161987, + 3.1228654384613037, + 3.075302004814148, + 3.0990699529647827, + 3.135697364807129, + 3.0625959634780884, + 3.009199619293213, + 3.1375036239624023, + 3.163372755050659, + 3.0674248933792114, + 3.1567808389663696, + 3.011177659034729, + 3.056342601776123, + 2.9956445693969727, + 3.0665026903152466, + 3.00251305103302, + 3.043543577194214, + 3.0153807401657104, + 3.231416940689087, + 3.113492250442505, + 3.056036949157715, + 3.0529987812042236, + 3.191048502922058, + 3.14768648147583, + 3.158997058868408, + 3.161283850669861, + 3.0824296474456787, + 3.1583211421966553, + 3.0362966060638428, + 3.146466612815857, + 3.0611331462860107, + 3.1597983837127686, + 3.066794991493225, + 3.167993426322937, + 3.149856686592102, + 3.0116604566574097, + 3.0294551849365234, + 3.0254968404769897, + 3.1562286615371704, + 3.076094627380371, + 3.202810525894165, + 2.9910837411880493, + 3.215625286102295, + 3.1635833978652954, + 3.244921922683716, + 3.0972235202789307, + 3.063948392868042, + 3.0736091136932373, + 3.245096802711487, + 3.187090754508972, + 3.0897867679595947, + 3.181527614593506, + 3.1229686737060547, + 3.035651683807373, + 3.085693836212158, + 3.0755420923233032, + 3.1528804302215576, + 2.9847413301467896, + 3.1538513898849487, + 3.2450876235961914, + 3.104891777038574, + 3.153931140899658, + 3.192876696586609, + 3.2068597078323364, + 3.155495762825012, + 3.0622929334640503, + 3.1049447059631348 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 79.13968537847029, + 56.22907961701427, + 48.091771793820044, + 44.225533963643315, + 40.88455031449475, + 38.79686957196745, + 36.8794297312531, + 35.12611806669696, + 33.536829112481314, + 32.52492927991035, + 31.487795195138393, + 30.42388865282493, + 29.92885373221615, + 29.071943392843913, + 28.39560159569491, + 28.203561812880235, + 27.85664072660998, + 27.714083006919676, + 27.620789529599087, + 27.597695972613245 + ], + "best_ppl": 27.597695972613245, + "train_time_s": 20960.153661489487 +} \ No newline at end of file diff --git a/063_sparse_dispatch/single_block_softmax_d128_best.pt b/063_sparse_dispatch/single_block_softmax_d128_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9676c68e7f9db47190a056a988bf3e6e94822ce --- /dev/null +++ b/063_sparse_dispatch/single_block_softmax_d128_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62e2cbbfe3347f3b4c7069d848af61d4b478c8cf463c73e90161f2e2456ae72b +size 145870304 diff --git a/063_sparse_dispatch/single_block_softmax_d128_results.json b/063_sparse_dispatch/single_block_softmax_d128_results.json new file mode 100644 index 0000000000000000000000000000000000000000..df17828fcba54cea1268aa3a1f33292a434ffffa --- /dev/null +++ b/063_sparse_dispatch/single_block_softmax_d128_results.json @@ -0,0 +1,2057 @@ +{ + "label": "single_block_softmax_d128", + "seed": 42, + "attention_type": "softmax", + "d_expert": 128, + "total_params": 36454400, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.176332473754883, + 18.38493537902832, + 10.301492691040039, + 7.724294662475586, + 7.560311317443848, + 7.568721771240234, + 7.3805389404296875, + 7.160459518432617, + 7.123780727386475, + 6.98036527633667, + 6.896290302276611, + 6.799950122833252, + 6.694100856781006, + 6.618630886077881, + 6.632807731628418, + 6.5019850730896, + 6.444748878479004, + 6.318686008453369, + 6.257201194763184, + 6.195608615875244, + 6.147838592529297, + 6.151965618133545, + 6.009244441986084, + 5.994077205657959, + 5.881842136383057, + 5.902892112731934, + 5.810401916503906, + 5.841610908508301, + 5.933221817016602, + 5.860111236572266, + 5.834997177124023, + 5.806406021118164, + 5.659626007080078, + 5.695424556732178, + 5.703946590423584, + 5.563539505004883, + 5.60106897354126, + 5.631132125854492, + 5.491766929626465, + 5.484053611755371, + 5.507168292999268, + 5.500442028045654, + 5.425495147705078, + 5.53033447265625, + 5.328535556793213, + 5.397923469543457, + 5.422194480895996, + 5.374926567077637, + 5.435667991638184, + 5.408082008361816, + 5.181087493896484, + 5.319263935089111, + 5.360940933227539, + 5.410716533660889, + 5.253708839416504, + 5.270907878875732, + 5.268818378448486, + 5.2873616218566895, + 5.26251220703125, + 5.189105987548828, + 5.305906772613525, + 5.195435523986816, + 5.270017147064209, + 5.216676712036133, + 5.307044506072998, + 5.225730895996094, + 5.291917324066162, + 4.97044038772583, + 5.133973121643066, + 5.1219000816345215, + 5.14176607131958, + 5.252144813537598, + 5.166850566864014, + 5.2041826248168945, + 4.998137950897217, + 5.0751423835754395, + 5.073849201202393, + 5.134529113769531, + 5.053045749664307, + 5.063666343688965, + 5.047741413116455, + 5.081967830657959, + 5.137664318084717, + 5.033365726470947, + 5.052147388458252, + 5.1064133644104, + 5.057507514953613, + 4.984900951385498, + 4.981490135192871, + 4.970367431640625, + 5.056365489959717, + 5.050520420074463, + 4.903524398803711, + 4.953831195831299, + 4.981908321380615, + 4.988546371459961, + 5.0454182624816895, + 4.982699871063232, + 5.027095317840576, + 5.07418966293335, + 4.893226623535156, + 4.927728176116943, + 4.921019077301025, + 4.967068195343018, + 4.983798980712891, + 4.860747337341309, + 4.784634113311768, + 4.882040500640869, + 4.821338176727295, + 4.841310977935791, + 4.938424587249756, + 4.872479438781738, + 4.918280124664307, + 4.99717903137207, + 4.75425910949707, + 5.00127649307251, + 4.82274866104126, + 4.943767547607422, + 4.773256301879883, + 4.978766441345215, + 4.85919189453125, + 4.87211799621582, + 4.837236404418945, + 4.7845683097839355, + 4.854312896728516, + 4.786880970001221, + 4.719114303588867, + 4.770754337310791, + 4.8764848709106445, + 4.794676303863525, + 4.754076957702637, + 4.843699932098389, + 4.736432075500488, + 4.757781982421875, + 4.825713634490967, + 4.780594825744629, + 4.749108791351318, + 4.744709014892578, + 4.804887771606445, + 4.732357025146484, + 4.75112771987915, + 4.7992730140686035, + 4.796204090118408, + 4.760904788970947, + 4.801344394683838, + 4.799560546875, + 4.713302135467529, + 4.803776741027832, + 4.787086009979248, + 4.809532165527344, + 4.626336574554443, + 4.826074123382568, + 4.729110240936279, + 4.77116584777832, + 4.6826276779174805, + 4.63021993637085, + 4.7617926597595215, + 4.688449382781982, + 4.611555576324463, + 4.786966323852539, + 4.742098808288574, + 4.682907581329346, + 4.756590366363525, + 4.623534202575684, + 4.673096179962158, + 4.701512813568115, + 4.716947555541992, + 4.656827449798584, + 4.645505905151367, + 4.625742435455322, + 4.698864936828613, + 4.591511249542236, + 4.6562604904174805, + 4.627127647399902, + 4.603058338165283, + 4.629160404205322, + 4.754614353179932, + 4.639405727386475, + 4.641000270843506, + 4.6176958084106445, + 4.437778949737549, + 4.632902145385742, + 4.686023235321045, + 4.585952281951904, + 4.676535129547119, + 4.625016212463379, + 4.629488468170166, + 4.6090545654296875, + 4.659919261932373, + 4.581169605255127, + 4.6203179359436035, + 4.517242908477783, + 4.728363990783691, + 4.570070743560791, + 4.634684085845947, + 4.5149736404418945, + 4.65925407409668, + 4.497935771942139, + 4.489388465881348, + 4.700495719909668, + 4.683528423309326, + 4.572572708129883, + 4.54262113571167, + 4.644140243530273, + 4.537362098693848, + 4.687101364135742, + 4.698548793792725, + 4.465151309967041, + 4.567503452301025, + 4.497851371765137, + 4.53875207901001, + 4.421239376068115, + 4.672889232635498, + 4.594815254211426, + 4.494253158569336, + 4.571939468383789, + 4.672451019287109, + 4.541658401489258, + 4.5189642906188965, + 4.476364612579346, + 4.619054794311523, + 4.6305084228515625, + 4.667370796203613, + 4.673949718475342, + 4.507795333862305, + 4.446967601776123, + 4.608482360839844, + 4.60738468170166, + 4.606685638427734, + 4.608238220214844, + 4.541628360748291, + 4.431821346282959, + 4.565141677856445, + 4.481500148773193, + 4.460172653198242, + 4.47437047958374, + 4.471960067749023, + 4.634761810302734, + 4.497270107269287, + 4.633116245269775, + 4.501468181610107, + 4.614564895629883, + 4.65036678314209, + 4.5439772605896, + 4.438719749450684, + 4.564930438995361, + 4.482875347137451, + 4.375041484832764, + 4.43977689743042, + 4.580262660980225, + 4.467697620391846, + 4.519524097442627, + 4.518400192260742, + 4.412795543670654, + 4.365612030029297, + 4.45211935043335, + 4.5253167152404785, + 4.361785888671875, + 4.319389343261719, + 4.558004856109619, + 4.4728593826293945, + 4.263983726501465, + 4.4417009353637695, + 4.369678020477295, + 4.376847267150879, + 4.524572849273682, + 4.425839424133301, + 4.332033157348633, + 4.500314235687256, + 4.38722562789917, + 4.395471096038818, + 4.49371862411499, + 4.382561206817627, + 4.401780128479004, + 4.47048807144165, + 4.400230407714844, + 4.324878692626953, + 4.2676496505737305, + 4.430977821350098, + 4.332050323486328, + 4.390905857086182, + 4.369166374206543, + 4.430726051330566, + 4.4302754402160645, + 4.340541362762451, + 4.465597629547119, + 4.46022891998291, + 4.3324151039123535, + 4.42258358001709, + 4.351320743560791, + 4.350809574127197, + 4.37009334564209, + 4.515538692474365, + 4.267934322357178, + 4.420814037322998, + 4.368348598480225, + 4.366444110870361, + 4.452036380767822, + 4.446290969848633, + 4.395227432250977, + 4.4125590324401855, + 4.353158950805664, + 4.298429012298584, + 4.297497272491455, + 4.506746768951416, + 4.433642387390137, + 4.422295570373535, + 4.322796821594238, + 4.472319602966309, + 4.316206455230713, + 4.46773099899292, + 4.3359246253967285, + 4.408320426940918, + 4.370523929595947, + 4.410964488983154, + 4.455195903778076, + 4.40603494644165, + 4.279356956481934, + 4.372992038726807, + 4.425415515899658, + 4.403489112854004, + 4.416646480560303, + 4.394421100616455, + 4.4145402908325195, + 4.361048698425293, + 4.360531806945801, + 4.391055583953857, + 4.3576436042785645, + 4.477582931518555, + 4.492638111114502, + 4.4588942527771, + 4.247146129608154, + 4.228792667388916, + 4.1859588623046875, + 4.325067520141602, + 4.326719760894775, + 4.306540489196777, + 4.397283554077148, + 4.342705249786377, + 4.34782600402832, + 4.418398857116699, + 4.3620076179504395, + 4.393230438232422, + 4.424224853515625, + 4.346933364868164, + 4.402584552764893, + 4.377368927001953, + 4.418521881103516, + 4.340813636779785, + 4.3642578125, + 4.280690670013428, + 4.361579418182373, + 4.255397319793701, + 4.337101936340332, + 4.459866046905518, + 4.2900390625, + 4.361277103424072, + 4.31654167175293, + 4.330137729644775, + 4.354692459106445, + 4.486750602722168, + 4.246373176574707, + 4.348917484283447, + 4.3738203048706055, + 4.208148002624512, + 4.450684547424316, + 4.312341213226318, + 4.337599277496338, + 4.434365272521973, + 4.415511608123779, + 4.3376336097717285, + 4.275938987731934, + 4.271393299102783, + 4.176734447479248, + 4.293653964996338, + 4.281605243682861, + 4.3358564376831055, + 4.429258823394775, + 4.363598346710205, + 4.359732627868652, + 4.187207221984863, + 4.3598856925964355, + 4.20366907119751, + 4.191817760467529, + 4.298381805419922, + 4.1908416748046875, + 4.248565673828125, + 4.172898292541504, + 4.39590311050415, + 4.1381940841674805, + 4.253564834594727, + 4.317883014678955, + 4.245113849639893, + 4.2714104652404785, + 4.160620212554932, + 4.242184162139893, + 4.333864212036133, + 4.348459720611572, + 4.305259704589844, + 4.240100860595703, + 4.266660213470459, + 4.308670997619629, + 4.395303249359131, + 4.313893795013428, + 4.256931781768799, + 4.3708720207214355, + 4.093936920166016, + 4.202112674713135, + 4.274305820465088, + 4.217848300933838, + 4.155575275421143, + 4.241328239440918, + 4.155369758605957, + 4.216012001037598, + 4.2808837890625, + 4.229536533355713, + 4.30170202255249, + 4.213015556335449, + 4.158262252807617, + 4.232583999633789, + 4.234509468078613, + 4.241524696350098, + 4.239935398101807, + 4.272369384765625, + 4.266684055328369, + 4.269311428070068, + 4.238707542419434, + 4.243066310882568, + 4.253436088562012, + 4.149953842163086, + 4.13176155090332, + 4.343042373657227, + 4.354803085327148, + 4.148579120635986, + 4.468336582183838, + 4.281279563903809, + 4.212371826171875, + 4.279465198516846, + 4.105898380279541, + 4.142055034637451, + 4.301629066467285, + 4.266111373901367, + 4.319859981536865, + 4.214448928833008, + 4.252303600311279, + 4.210707664489746, + 4.1024489402771, + 4.097776889801025, + 4.134152412414551, + 4.20228385925293, + 4.424699783325195, + 4.1103291511535645, + 4.353491306304932, + 4.108160018920898, + 4.301358222961426, + 4.311038494110107, + 4.277657508850098, + 4.197139739990234, + 4.223202705383301, + 4.309672832489014, + 4.220980644226074, + 4.363130569458008, + 4.330185890197754, + 4.316912651062012, + 4.139510631561279, + 4.1691155433654785, + 4.39205265045166, + 4.202175140380859, + 4.236765384674072, + 4.349240303039551, + 4.2722296714782715, + 4.308323383331299, + 4.251796722412109, + 4.134993076324463, + 4.279397010803223, + 4.302524566650391, + 4.180596828460693, + 4.25620174407959, + 4.290757179260254, + 4.267865180969238, + 4.202410697937012, + 4.195535182952881, + 4.283775806427002, + 4.218504428863525, + 4.209053039550781, + 4.3176069259643555, + 4.3223042488098145, + 4.296865940093994, + 4.289254188537598, + 4.208504676818848, + 4.304782867431641, + 4.254759788513184, + 4.28043794631958, + 3.9332401752471924, + 4.16156530380249, + 4.206305027008057, + 4.196136474609375, + 4.253010272979736, + 4.167081832885742, + 4.192442893981934, + 4.1594929695129395, + 4.3184099197387695, + 4.22007942199707, + 4.225396633148193, + 4.170585632324219, + 4.1181511878967285, + 4.174577236175537, + 4.231995582580566, + 4.088967800140381, + 4.09423828125, + 4.071463108062744, + 4.1039838790893555, + 4.061471939086914, + 4.088828086853027, + 4.199707984924316, + 4.124685764312744, + 4.176821708679199, + 4.18607234954834, + 4.123698711395264, + 4.183473110198975, + 4.087954998016357, + 4.171304702758789, + 4.19730806350708, + 4.1328253746032715, + 4.257079124450684, + 4.077358722686768, + 4.168673992156982, + 4.0808796882629395, + 4.242679595947266, + 4.321089744567871, + 4.192817687988281, + 4.190121650695801, + 4.101876735687256, + 4.201162815093994, + 4.180306434631348, + 4.078887939453125, + 4.189963340759277, + 4.248907089233398, + 4.161370277404785, + 4.129108905792236, + 4.062371253967285, + 4.238724231719971, + 4.286348819732666, + 4.254344940185547, + 4.124649524688721, + 4.2990241050720215, + 4.112746715545654, + 4.097662448883057, + 4.138718128204346, + 4.212574481964111, + 4.089721202850342, + 4.176293849945068, + 4.044737815856934, + 4.19629430770874, + 4.228630542755127, + 4.146212100982666, + 4.169814109802246, + 4.261829376220703, + 4.146826267242432, + 4.176901340484619, + 4.202201843261719, + 4.078429222106934, + 4.108133316040039, + 4.203269004821777, + 4.121795654296875, + 4.224586009979248, + 4.104085445404053, + 4.082516193389893, + 4.098277568817139, + 4.070533752441406, + 4.111354827880859, + 4.11316442489624, + 4.1139984130859375, + 4.1716742515563965, + 4.165770053863525, + 4.199461936950684, + 4.16732120513916, + 4.142824172973633, + 4.094465255737305, + 4.072868824005127, + 4.25206184387207, + 4.104349613189697, + 4.029716968536377, + 4.071560859680176, + 4.164813041687012, + 4.14479923248291, + 4.084786891937256, + 4.1458539962768555, + 4.112954139709473, + 4.258428573608398, + 4.117847919464111, + 4.065824508666992, + 4.1355719566345215, + 4.165050506591797, + 4.339496612548828, + 4.17351770401001, + 4.129137992858887, + 4.182507514953613, + 4.224822044372559, + 4.245170593261719, + 4.148706912994385, + 4.233143329620361, + 4.051115036010742, + 4.186071872711182, + 4.0825581550598145, + 4.005885124206543, + 4.198692321777344, + 4.088901519775391, + 4.1425461769104, + 4.096429347991943, + 4.18516206741333, + 4.147459030151367, + 4.148946762084961, + 4.1360368728637695, + 4.1974053382873535, + 4.190610885620117, + 4.148863315582275, + 3.9937570095062256, + 4.234013080596924, + 4.07072114944458, + 4.251868724822998, + 4.030470848083496, + 4.099946022033691, + 3.932847738265991, + 4.144351005554199, + 4.230395317077637, + 4.253616809844971, + 4.143629550933838, + 3.947856903076172, + 4.096540451049805, + 4.102942943572998, + 4.161017894744873, + 4.04821252822876, + 4.23417329788208, + 3.9964213371276855, + 4.056456089019775, + 4.130592346191406, + 4.138591766357422, + 4.1332197189331055, + 4.037425994873047, + 4.044220924377441, + 4.060901165008545, + 4.039056777954102, + 4.012326717376709, + 4.102728366851807, + 4.191726207733154, + 4.139283180236816, + 4.019463062286377, + 4.098374366760254, + 4.086668014526367, + 4.0408477783203125, + 4.152194976806641, + 4.07780122756958, + 4.117466449737549, + 4.1147541999816895, + 4.028103828430176, + 4.045719146728516, + 4.071882724761963, + 4.080263614654541, + 4.042063236236572, + 4.043045997619629, + 4.111413478851318, + 4.0283026695251465, + 4.0698018074035645, + 4.123717784881592, + 4.127065181732178, + 4.1377410888671875, + 4.165176868438721, + 3.9999260902404785, + 4.103476524353027, + 4.163804054260254, + 4.157724857330322, + 4.100888729095459, + 4.103931903839111, + 4.0360307693481445, + 4.1726250648498535, + 4.090478897094727, + 4.0562920570373535, + 4.119004249572754, + 4.0541887283325195, + 4.0295515060424805, + 4.128571033477783, + 4.0747528076171875, + 4.072201728820801, + 4.022777557373047, + 4.051002025604248, + 4.160106182098389, + 4.038430690765381, + 4.10708475112915, + 3.989063024520874, + 4.08645486831665, + 4.160850524902344, + 4.049747467041016, + 4.024477005004883, + 4.064504146575928, + 4.224796772003174, + 4.039666175842285, + 4.1369242668151855, + 4.248992443084717, + 4.0151190757751465, + 4.044976711273193, + 4.177486419677734, + 3.977889060974121, + 4.035198211669922, + 4.049068927764893, + 4.148441791534424, + 4.186528205871582, + 4.045865535736084, + 4.023386001586914, + 4.065922737121582, + 4.118814945220947, + 4.095438480377197, + 4.0139079093933105, + 4.167926788330078, + 4.143614768981934, + 4.107815265655518, + 4.143046855926514, + 3.987266778945923, + 3.9559102058410645, + 4.185837745666504, + 4.047125339508057, + 4.1649932861328125, + 4.108109951019287, + 4.104365825653076, + 4.1170196533203125, + 4.067093849182129, + 3.9532618522644043, + 4.167660713195801, + 4.162572860717773, + 4.017630100250244, + 4.118612289428711, + 4.058819770812988, + 4.033334255218506, + 4.189464569091797, + 4.052477836608887, + 4.167822360992432, + 4.110771656036377, + 4.089543342590332, + 4.133146286010742, + 4.098072052001953, + 4.031911373138428, + 4.149018287658691, + 3.977532148361206, + 4.081054210662842, + 4.101276874542236, + 4.086462020874023, + 4.154049873352051, + 3.9980528354644775, + 4.093647480010986, + 4.222353458404541, + 4.113266944885254, + 4.068594455718994, + 3.994366407394409, + 4.140864372253418, + 4.166479110717773, + 4.1175665855407715, + 4.136966228485107, + 4.130038738250732, + 4.094533920288086, + 4.113109111785889, + 4.012477874755859, + 4.177511215209961, + 4.016365051269531, + 4.027101039886475, + 4.053489685058594, + 4.0540313720703125, + 3.928051471710205, + 3.9595465660095215, + 3.945315361022949, + 3.9311909675598145, + 4.097541332244873, + 4.037445068359375, + 4.015671253204346, + 4.159188270568848, + 4.037524700164795, + 3.9338114261627197, + 4.096473693847656, + 4.039182662963867, + 4.0875020027160645, + 4.013972759246826, + 4.23942232131958, + 4.100405216217041, + 4.07157564163208, + 4.151004791259766, + 4.014695167541504, + 4.065962791442871, + 4.15580940246582, + 3.985710382461548, + 4.044483184814453, + 3.9898877143859863, + 4.028340816497803, + 3.994556188583374, + 4.048777103424072, + 3.9175758361816406, + 4.082972049713135, + 4.03287935256958, + 4.031994342803955, + 4.063074588775635, + 4.050242900848389, + 3.9300715923309326, + 3.8984103202819824, + 4.0603203773498535, + 4.035257816314697, + 3.941099166870117, + 3.881052255630493, + 4.199827194213867, + 4.014410972595215, + 4.0515031814575195, + 4.109512805938721, + 4.0124311447143555, + 4.115096569061279, + 3.908013105392456, + 3.9406344890594482, + 3.999018907546997, + 4.052920341491699, + 4.030778884887695, + 3.970268964767456, + 4.027428150177002, + 3.9507052898406982, + 3.9499897956848145, + 4.036952495574951, + 4.0470099449157715, + 3.9311115741729736, + 4.068450450897217, + 3.947526693344116, + 3.9539334774017334, + 3.951904773712158, + 4.117744445800781, + 4.053983688354492, + 4.027003288269043, + 3.9969136714935303, + 4.048168182373047, + 4.011540412902832, + 3.968919038772583, + 4.067023277282715, + 4.087318420410156, + 4.092838287353516, + 4.123591899871826, + 4.023635387420654, + 4.080718517303467, + 4.019795894622803, + 4.004531383514404, + 4.060901641845703, + 4.041822910308838, + 3.9681878089904785, + 4.07095193862915, + 4.097644805908203, + 3.9610745906829834, + 4.175342559814453, + 4.002512454986572, + 4.061615943908691, + 4.010103225708008, + 4.031017780303955, + 4.100347995758057, + 4.0625457763671875, + 4.060278415679932, + 3.9321815967559814, + 3.9758267402648926, + 3.9591710567474365, + 4.10621976852417, + 3.9003143310546875, + 4.049101829528809, + 3.9059464931488037, + 4.163637161254883, + 4.124380588531494, + 4.109086513519287, + 3.9404914379119873, + 4.001163482666016, + 4.061090469360352, + 3.965567111968994, + 4.0059895515441895, + 4.103193759918213, + 4.0511016845703125, + 3.980811357498169, + 4.006762504577637, + 4.092925548553467, + 4.076976299285889, + 4.039494037628174, + 4.131969451904297, + 4.027904987335205, + 3.924935817718506, + 4.12670373916626, + 4.142152309417725, + 4.073067665100098, + 3.9534192085266113, + 3.887178897857666, + 4.021265506744385, + 4.140153884887695, + 4.093939781188965, + 3.8731415271759033, + 4.0428595542907715, + 4.088985919952393, + 3.985962390899658, + 4.126272201538086, + 4.063149929046631, + 4.154670715332031, + 3.9194791316986084, + 3.9998395442962646, + 4.00296688079834, + 4.0457024574279785, + 4.007960796356201, + 3.981834888458252, + 4.058630466461182, + 3.922335147857666, + 3.912952423095703, + 4.074065208435059, + 3.96647047996521, + 4.092591762542725, + 4.0896759033203125, + 4.039407253265381, + 4.000612735748291, + 3.967792272567749, + 4.023843288421631, + 4.057323455810547, + 3.950050115585327, + 4.0128655433654785, + 4.06371545791626, + 4.183388710021973, + 4.012699127197266, + 3.961324691772461, + 4.157004356384277, + 4.029828071594238, + 4.002869129180908, + 3.998157024383545, + 3.9951744079589844, + 4.022213459014893, + 4.093741416931152, + 3.991145610809326, + 4.082230567932129, + 4.159390926361084, + 4.063165187835693, + 4.072988510131836, + 3.9800357818603516, + 3.993562698364258, + 4.04395866394043, + 4.003382205963135, + 4.130507946014404, + 3.9358580112457275, + 3.991854190826416, + 3.918368101119995, + 3.9809634685516357, + 3.9539592266082764, + 4.019747734069824, + 4.116357803344727, + 4.081479549407959, + 3.9079697132110596, + 3.920184850692749, + 3.9583053588867188, + 4.065454959869385, + 4.17990779876709, + 4.001392364501953, + 4.040472507476807, + 4.095835208892822, + 4.009803771972656, + 4.035629749298096, + 4.035642147064209, + 4.00862979888916, + 4.101343154907227, + 3.8186697959899902, + 4.072710037231445, + 4.061707019805908, + 4.174251556396484, + 4.075819492340088, + 3.935734510421753, + 4.018671035766602, + 3.986034870147705, + 3.9697790145874023, + 4.012016296386719, + 4.016288757324219, + 4.050407409667969, + 4.057278633117676, + 3.9277544021606445, + 4.124890327453613, + 4.03425407409668, + 4.074867248535156, + 4.016211032867432, + 4.113704681396484, + 4.009227752685547, + 4.182784080505371, + 3.973836898803711, + 3.9836151599884033, + 4.122056007385254, + 4.108513832092285, + 4.00583553314209, + 3.8706209659576416, + 4.019254207611084, + 3.952216625213623, + 4.194624900817871, + 4.068050861358643, + 4.090786457061768, + 3.9334263801574707, + 4.035714149475098, + 4.009786128997803, + 3.9702086448669434, + 4.058384895324707, + 4.1071953773498535, + 3.97342848777771, + 3.9681203365325928, + 4.136559009552002, + 4.027153968811035 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 194.49789389088596, + 124.34328990420401, + 100.64060260706351, + 85.39694073638248, + 76.6770571041529, + 70.64697561651144, + 66.32531606135377, + 63.28905674894096, + 60.470898023702425, + 58.118450294927115, + 56.810928456857795, + 55.073873525576374, + 53.844720187256975, + 52.773929767181784, + 51.86701419402942, + 51.26957073878366, + 50.72602700382981, + 50.35070039073387, + 50.27182722042058, + 50.24720512965703 + ], + "best_ppl": 50.24720512965703, + "train_time_s": 5640.580499649048 +} \ No newline at end of file diff --git a/063_sparse_dispatch/single_block_softmax_d512_best.pt b/063_sparse_dispatch/single_block_softmax_d512_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..c22f0c4fafccb3dfc6610fae7b295348a8896ccd --- /dev/null +++ b/063_sparse_dispatch/single_block_softmax_d512_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e5761caab486e44336463f17b6903b8c3de90fe4d9dff94d9f0fb3b2ec4fecf +size 265482016 diff --git a/063_sparse_dispatch/single_block_softmax_d512_results.json b/063_sparse_dispatch/single_block_softmax_d512_results.json new file mode 100644 index 0000000000000000000000000000000000000000..92c351f532d4100cb30e220d22bd2b4296b712b7 --- /dev/null +++ b/063_sparse_dispatch/single_block_softmax_d512_results.json @@ -0,0 +1,2057 @@ +{ + "label": "single_block_softmax_d512", + "seed": 42, + "attention_type": "softmax", + "d_expert": 512, + "total_params": 66357248, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 15.813669204711914, + 8.440557479858398, + 7.47121000289917, + 7.239301681518555, + 7.047311305999756, + 6.85792350769043, + 6.766590595245361, + 6.550095558166504, + 6.484933853149414, + 6.3097825050354, + 6.273303031921387, + 6.1929802894592285, + 6.174386501312256, + 6.047475337982178, + 5.79181432723999, + 5.881019115447998, + 5.871752738952637, + 5.6910600662231445, + 5.70848274230957, + 5.6308698654174805, + 5.567530632019043, + 5.542713165283203, + 5.504654884338379, + 5.476783752441406, + 5.2415337562561035, + 5.425209999084473, + 5.391985893249512, + 5.277317047119141, + 5.2662577629089355, + 5.177986145019531, + 5.174928188323975, + 5.094598770141602, + 5.214468002319336, + 5.047663688659668, + 5.115211009979248, + 5.165629863739014, + 5.1327714920043945, + 4.918028831481934, + 5.136908531188965, + 5.037798881530762, + 4.874749660491943, + 4.917852878570557, + 4.859525680541992, + 4.903775215148926, + 4.881333827972412, + 4.884647846221924, + 4.757437705993652, + 4.860387325286865, + 4.790381908416748, + 4.78542423248291, + 4.850590705871582, + 4.707088470458984, + 4.856020450592041, + 4.607213020324707, + 4.716348648071289, + 4.680971145629883, + 4.650516510009766, + 4.702850818634033, + 4.691566467285156, + 4.64111852645874, + 4.651163101196289, + 4.707051753997803, + 4.621279716491699, + 4.629551887512207, + 4.595034599304199, + 4.601871490478516, + 4.494810581207275, + 4.601046562194824, + 4.598980903625488, + 4.5400390625, + 4.439642429351807, + 4.55434513092041, + 4.418798446655273, + 4.495662689208984, + 4.422118186950684, + 4.380860805511475, + 4.5261454582214355, + 4.4860663414001465, + 4.352931499481201, + 4.511445999145508, + 4.380867958068848, + 4.451148509979248, + 4.4042840003967285, + 4.4871087074279785, + 4.32619047164917, + 4.3950653076171875, + 4.460283279418945, + 4.398841381072998, + 4.443312168121338, + 4.323598861694336, + 4.296380996704102, + 4.3340959548950195, + 4.476419925689697, + 4.421837329864502, + 4.411654472351074, + 4.393030166625977, + 4.176390171051025, + 4.321396350860596, + 4.202417850494385, + 4.295412540435791, + 4.30095911026001, + 4.39825963973999, + 4.389321327209473, + 4.138681411743164, + 4.384230136871338, + 4.334469318389893, + 4.326939582824707, + 4.215909957885742, + 4.333635330200195, + 4.2942070960998535, + 4.141752243041992, + 4.212926387786865, + 4.305859088897705, + 4.252499103546143, + 4.239764213562012, + 4.27897834777832, + 4.356212615966797, + 4.2919464111328125, + 4.271398544311523, + 4.220156669616699, + 4.284317493438721, + 4.1829423904418945, + 4.24629020690918, + 4.184384346008301, + 4.246240615844727, + 4.300403118133545, + 4.040787220001221, + 4.248673439025879, + 4.1703267097473145, + 4.161165714263916, + 4.121463298797607, + 4.1316304206848145, + 4.201111793518066, + 4.145129680633545, + 4.169065475463867, + 4.125831127166748, + 4.200785160064697, + 4.069657325744629, + 4.165411949157715, + 3.9415574073791504, + 4.207895755767822, + 4.139024257659912, + 4.174060344696045, + 4.226902484893799, + 4.168089389801025, + 4.065567970275879, + 4.184290885925293, + 4.192387104034424, + 4.154361724853516, + 4.1936845779418945, + 4.134564399719238, + 4.074972152709961, + 4.097100734710693, + 4.140445232391357, + 4.225042343139648, + 4.0902581214904785, + 4.1879096031188965, + 4.214085102081299, + 4.1575398445129395, + 4.050432205200195, + 4.141121864318848, + 4.0825276374816895, + 4.109188556671143, + 3.9935240745544434, + 4.098076820373535, + 4.136686325073242, + 3.9861998558044434, + 4.064513683319092, + 4.053955078125, + 4.002506732940674, + 4.109365463256836, + 4.159247875213623, + 4.098912715911865, + 4.14232873916626, + 4.035067081451416, + 4.039890289306641, + 4.046959400177002, + 4.07852029800415, + 4.081326484680176, + 4.092146396636963, + 4.025160789489746, + 4.134200572967529, + 4.061306476593018, + 4.019951343536377, + 4.066192626953125, + 4.022829532623291, + 4.016316890716553, + 4.003079891204834, + 4.072341442108154, + 3.8984732627868652, + 3.9808342456817627, + 4.125952243804932, + 4.041411876678467, + 4.091626167297363, + 4.051382541656494, + 3.9755256175994873, + 4.039571285247803, + 4.090245246887207, + 4.085178375244141, + 4.123470783233643, + 3.981539487838745, + 3.9959826469421387, + 4.0579633712768555, + 4.089780807495117, + 4.093337059020996, + 4.037168502807617, + 3.970472812652588, + 4.025504112243652, + 3.9669063091278076, + 4.0348100662231445, + 3.866365432739258, + 4.055525302886963, + 4.001876354217529, + 4.041522979736328, + 3.9384288787841797, + 3.8782622814178467, + 4.030274868011475, + 3.9493229389190674, + 4.045159339904785, + 4.009214878082275, + 4.058200359344482, + 3.977945327758789, + 3.9297232627868652, + 3.8943262100219727, + 4.129798412322998, + 4.009725570678711, + 4.065953731536865, + 3.9939918518066406, + 3.9543533325195312, + 4.016005039215088, + 4.023250102996826, + 4.058923721313477, + 4.074881076812744, + 3.9040894508361816, + 4.037701606750488, + 3.916090488433838, + 4.041012287139893, + 4.041741847991943, + 3.9075024127960205, + 4.0923919677734375, + 3.892876386642456, + 3.921910524368286, + 3.995065927505493, + 3.88447904586792, + 3.9893312454223633, + 3.9984130859375, + 3.8876395225524902, + 4.004814624786377, + 3.9738314151763916, + 3.9724013805389404, + 4.027195453643799, + 3.923750162124634, + 3.8921890258789062, + 3.8967089653015137, + 3.7426719665527344, + 3.8358733654022217, + 3.8701350688934326, + 3.88918399810791, + 4.006918907165527, + 3.8819921016693115, + 3.9776124954223633, + 3.78094482421875, + 3.8116846084594727, + 3.8393428325653076, + 3.818082809448242, + 3.951510190963745, + 3.8849105834960938, + 3.87607741355896, + 3.873575448989868, + 3.9093918800354004, + 3.85010027885437, + 3.9215598106384277, + 4.000772953033447, + 3.8431153297424316, + 3.7413203716278076, + 3.8864965438842773, + 3.915759325027466, + 3.949582576751709, + 3.9252662658691406, + 3.9859182834625244, + 3.965358018875122, + 3.97417950630188, + 3.8533923625946045, + 3.7966136932373047, + 3.8279869556427, + 3.852292537689209, + 3.9220099449157715, + 3.782815456390381, + 3.985840082168579, + 3.923768997192383, + 3.8498618602752686, + 3.870795249938965, + 3.85756516456604, + 3.8523736000061035, + 3.916337728500366, + 3.9450948238372803, + 3.7798702716827393, + 3.9859986305236816, + 3.839392900466919, + 3.938044548034668, + 3.8468291759490967, + 3.912571907043457, + 3.7081499099731445, + 3.8348441123962402, + 4.058248043060303, + 3.9058449268341064, + 3.872197389602661, + 3.814157724380493, + 3.963343381881714, + 3.9520394802093506, + 3.9138357639312744, + 3.8589131832122803, + 3.868642807006836, + 3.8723130226135254, + 3.9237446784973145, + 3.823519706726074, + 3.913302421569824, + 3.8039743900299072, + 3.740597724914551, + 3.749483585357666, + 3.8544671535491943, + 3.8158528804779053, + 3.8110504150390625, + 3.779829978942871, + 3.7626113891601562, + 3.864835739135742, + 3.8102023601531982, + 3.787245512008667, + 3.8515069484710693, + 3.780341386795044, + 3.7293143272399902, + 3.9009926319122314, + 3.7608492374420166, + 3.9094369411468506, + 3.908407688140869, + 3.8859400749206543, + 3.9239330291748047, + 3.8348653316497803, + 3.8958117961883545, + 3.7286317348480225, + 3.808717966079712, + 3.890561580657959, + 3.8818118572235107, + 3.8653900623321533, + 3.82552170753479, + 3.7930116653442383, + 3.7689599990844727, + 3.9112205505371094, + 3.8273532390594482, + 3.8769569396972656, + 3.7840516567230225, + 3.873535394668579, + 3.795771837234497, + 3.868381977081299, + 3.792897939682007, + 3.837327480316162, + 3.8866994380950928, + 3.836665153503418, + 3.8561851978302, + 3.7910468578338623, + 3.755054473876953, + 3.806840658187866, + 3.779841661453247, + 3.875537157058716, + 3.791888952255249, + 3.862006425857544, + 3.8949670791625977, + 3.8379106521606445, + 3.79006028175354, + 3.7964696884155273, + 3.763519048690796, + 3.8718299865722656, + 3.8760647773742676, + 3.7842628955841064, + 3.6827893257141113, + 3.7949326038360596, + 3.859255790710449, + 3.7886507511138916, + 3.7758677005767822, + 3.593249797821045, + 3.840860366821289, + 3.7336134910583496, + 3.56607723236084, + 3.7312800884246826, + 3.6875851154327393, + 3.630167007446289, + 3.7070584297180176, + 3.829604387283325, + 3.6882264614105225, + 3.7352542877197266, + 3.6425764560699463, + 3.6651947498321533, + 3.68843412399292, + 3.8251640796661377, + 3.7894482612609863, + 3.7539587020874023, + 3.8061375617980957, + 3.7485909461975098, + 3.6904091835021973, + 3.819521188735962, + 3.7237887382507324, + 3.6732378005981445, + 3.7876393795013428, + 3.6885273456573486, + 3.844170570373535, + 3.599721670150757, + 3.585855484008789, + 3.769864320755005, + 3.6270604133605957, + 3.707408905029297, + 3.826970100402832, + 3.7004294395446777, + 3.801765203475952, + 3.7557218074798584, + 3.6728978157043457, + 3.692126989364624, + 3.6448721885681152, + 3.7338032722473145, + 3.7821903228759766, + 3.6549737453460693, + 3.718003988265991, + 3.8083386421203613, + 3.6920478343963623, + 3.5985727310180664, + 3.730618715286255, + 3.760678291320801, + 3.7131080627441406, + 3.725306987762451, + 3.731144905090332, + 3.720945119857788, + 3.6944234371185303, + 3.6941254138946533, + 3.7556960582733154, + 3.81079363822937, + 3.7720282077789307, + 3.7934677600860596, + 3.7020134925842285, + 3.684922695159912, + 3.705623149871826, + 3.8993067741394043, + 3.7502124309539795, + 3.7648744583129883, + 3.736509323120117, + 3.761267900466919, + 3.777205228805542, + 3.7900185585021973, + 3.741870164871216, + 3.7610714435577393, + 3.786386728286743, + 3.6546072959899902, + 3.8234899044036865, + 3.697120428085327, + 3.7295212745666504, + 3.7454075813293457, + 3.760016918182373, + 3.758849620819092, + 3.6955974102020264, + 3.8366551399230957, + 3.734776496887207, + 3.7438971996307373, + 3.7978293895721436, + 3.8111937046051025, + 3.682211399078369, + 3.635054349899292, + 3.696593999862671, + 3.6311841011047363, + 3.7309696674346924, + 3.6681902408599854, + 3.8058366775512695, + 3.749159812927246, + 3.6256768703460693, + 3.7593064308166504, + 3.714991331100464, + 3.7504470348358154, + 3.6330716609954834, + 3.7469253540039062, + 3.7402265071868896, + 3.625792980194092, + 3.881321430206299, + 3.7535462379455566, + 3.6357898712158203, + 3.7265398502349854, + 3.7483720779418945, + 3.739366054534912, + 3.742290735244751, + 3.580738067626953, + 3.6935677528381348, + 3.600229263305664, + 3.7845523357391357, + 3.7585020065307617, + 3.609584331512451, + 3.7027602195739746, + 3.4902639389038086, + 3.681547164916992, + 3.764214515686035, + 3.682863235473633, + 3.720144033432007, + 3.6716666221618652, + 3.7079756259918213, + 3.8121871948242188, + 3.745847225189209, + 3.6171228885650635, + 3.586897373199463, + 3.5203652381896973, + 3.731998920440674, + 3.758683443069458, + 3.4489970207214355, + 3.5402352809906006, + 3.566786050796509, + 3.521664619445801, + 3.676800012588501, + 3.734623670578003, + 3.533066511154175, + 3.6209089756011963, + 3.5550310611724854, + 3.5123658180236816, + 3.51257586479187, + 3.5384044647216797, + 3.6717824935913086, + 3.7211639881134033, + 3.5492019653320312, + 3.5925514698028564, + 3.7276854515075684, + 3.5562379360198975, + 3.6507606506347656, + 3.4797470569610596, + 3.6088011264801025, + 3.5805389881134033, + 3.7075557708740234, + 3.6796998977661133, + 3.6063930988311768, + 3.5583813190460205, + 3.5505776405334473, + 3.765955924987793, + 3.6544206142425537, + 3.654932975769043, + 3.6558279991149902, + 3.5872304439544678, + 3.589287519454956, + 3.541344404220581, + 3.5877296924591064, + 3.5820064544677734, + 3.576972484588623, + 3.541078567504883, + 3.463085174560547, + 3.7258265018463135, + 3.6132349967956543, + 3.5964913368225098, + 3.6678268909454346, + 3.6390316486358643, + 3.6017355918884277, + 3.6177923679351807, + 3.6565709114074707, + 3.8202688694000244, + 3.7169246673583984, + 3.6758339405059814, + 3.542783260345459, + 3.6899237632751465, + 3.6196701526641846, + 3.7109384536743164, + 3.5635862350463867, + 3.640777826309204, + 3.636260986328125, + 3.550401210784912, + 3.4780285358428955, + 3.675405740737915, + 3.6508281230926514, + 3.5440540313720703, + 3.566934108734131, + 3.6104023456573486, + 3.7629010677337646, + 3.7153728008270264, + 3.599384069442749, + 3.6796867847442627, + 3.5461394786834717, + 3.698577880859375, + 3.50593900680542, + 3.597400665283203, + 3.5805928707122803, + 3.488271951675415, + 3.4864349365234375, + 3.674868583679199, + 3.6634607315063477, + 3.618558883666992, + 3.6509766578674316, + 3.4349050521850586, + 3.7087435722351074, + 3.6635947227478027, + 3.791553258895874, + 3.586529016494751, + 3.6656641960144043, + 3.6959176063537598, + 3.5042881965637207, + 3.51365065574646, + 3.6173765659332275, + 3.575098991394043, + 3.4914159774780273, + 3.43843412399292, + 3.549431085586548, + 3.7645068168640137, + 3.692430019378662, + 3.5976479053497314, + 3.5791707038879395, + 3.5351972579956055, + 3.646843194961548, + 3.636854648590088, + 3.685589075088501, + 3.5988736152648926, + 3.586599588394165, + 3.6714298725128174, + 3.5540130138397217, + 3.7150373458862305, + 3.57553768157959, + 3.5307703018188477, + 3.642986297607422, + 3.5823400020599365, + 3.6788601875305176, + 3.6047518253326416, + 3.5155527591705322, + 3.612272262573242, + 3.6299197673797607, + 3.538890838623047, + 3.578289270401001, + 3.6614835262298584, + 3.5804600715637207, + 3.5037224292755127, + 3.5411105155944824, + 3.4576752185821533, + 3.537871837615967, + 3.6509008407592773, + 3.6385080814361572, + 3.6311094760894775, + 3.466310977935791, + 3.5004940032958984, + 3.3794188499450684, + 3.494323253631592, + 3.48844838142395, + 3.515941619873047, + 3.4463865756988525, + 3.4120209217071533, + 3.44580340385437, + 3.4170854091644287, + 3.3548078536987305, + 3.495795249938965, + 3.6010520458221436, + 3.4953577518463135, + 3.536294937133789, + 3.5912058353424072, + 3.5542662143707275, + 3.619687557220459, + 3.444516897201538, + 3.546330213546753, + 3.4311256408691406, + 3.580568313598633, + 3.482515573501587, + 3.572165012359619, + 3.562197685241699, + 3.544433832168579, + 3.534386396408081, + 3.416158676147461, + 3.6169772148132324, + 3.532069206237793, + 3.517428159713745, + 3.610775947570801, + 3.4934778213500977, + 3.4954683780670166, + 3.432523012161255, + 3.62092661857605, + 3.463074207305908, + 3.6511855125427246, + 3.4817261695861816, + 3.5090930461883545, + 3.337299346923828, + 3.5732154846191406, + 3.5308070182800293, + 3.4687998294830322, + 3.4320380687713623, + 3.5545849800109863, + 3.67946195602417, + 3.4306373596191406, + 3.465589761734009, + 3.643988609313965, + 3.454374074935913, + 3.529798984527588, + 3.472259759902954, + 3.4877705574035645, + 3.538466691970825, + 3.5761513710021973, + 3.664395332336426, + 3.590035915374756, + 3.449949026107788, + 3.464723587036133, + 3.3757081031799316, + 3.4690165519714355, + 3.5871336460113525, + 3.6202492713928223, + 3.458922863006592, + 3.401355504989624, + 3.4073469638824463, + 3.4841692447662354, + 3.6815671920776367, + 3.5513696670532227, + 3.4806270599365234, + 3.5223066806793213, + 3.552340030670166, + 3.528928279876709, + 3.469468832015991, + 3.5193886756896973, + 3.4451332092285156, + 3.511798143386841, + 3.534493923187256, + 3.6124117374420166, + 3.439725875854492, + 3.51444935798645, + 3.372886896133423, + 3.4656002521514893, + 3.56562876701355, + 3.4734201431274414, + 3.639596700668335, + 3.6058590412139893, + 3.5447535514831543, + 3.4436774253845215, + 3.4900453090667725, + 3.4106171131134033, + 3.5222537517547607, + 3.542116641998291, + 3.4614098072052, + 3.50166392326355, + 3.4745941162109375, + 3.4621288776397705, + 3.5516183376312256, + 3.43384051322937, + 3.5101141929626465, + 3.4616506099700928, + 3.4456746578216553, + 3.490180015563965, + 3.4484009742736816, + 3.6004674434661865, + 3.486330032348633, + 3.4833426475524902, + 3.3849377632141113, + 3.5764596462249756, + 3.492764711380005, + 3.450974941253662, + 3.5803914070129395, + 3.506857395172119, + 3.491793394088745, + 3.4361302852630615, + 3.474719524383545, + 3.5911788940429688, + 3.52152943611145, + 3.4606504440307617, + 3.4371464252471924, + 3.4613559246063232, + 3.625004529953003, + 3.5264763832092285, + 3.528546094894409, + 3.3480186462402344, + 3.5925800800323486, + 3.4383544921875, + 3.4544057846069336, + 3.3306005001068115, + 3.5143470764160156, + 3.518649101257324, + 3.514636754989624, + 3.5198161602020264, + 3.4392189979553223, + 3.3570094108581543, + 3.400369882583618, + 3.4655683040618896, + 3.372589111328125, + 3.565984010696411, + 3.2536227703094482, + 3.352531671524048, + 3.418996810913086, + 3.4759116172790527, + 3.3669466972351074, + 3.3413963317871094, + 3.3999743461608887, + 3.412240982055664, + 3.4114630222320557, + 3.4218480587005615, + 3.4244747161865234, + 3.4730498790740967, + 3.39542555809021, + 3.4426918029785156, + 3.3991687297821045, + 3.4165241718292236, + 3.3455307483673096, + 3.529766082763672, + 3.404024124145508, + 3.424222469329834, + 3.3904685974121094, + 3.416234016418457, + 3.4036648273468018, + 3.422107696533203, + 3.346722364425659, + 3.3528566360473633, + 3.459569215774536, + 3.4204111099243164, + 3.3656442165374756, + 3.3809974193573, + 3.52052903175354, + 3.467266082763672, + 3.5618038177490234, + 3.3730838298797607, + 3.338989019393921, + 3.3445258140563965, + 3.5040647983551025, + 3.354562997817993, + 3.3722429275512695, + 3.4431114196777344, + 3.427457571029663, + 3.5794296264648438, + 3.4666152000427246, + 3.502673387527466, + 3.429950475692749, + 3.3245294094085693, + 3.5101261138916016, + 3.478290557861328, + 3.340327501296997, + 3.474160671234131, + 3.5496842861175537, + 3.4706387519836426, + 3.3898894786834717, + 3.512287139892578, + 3.418160915374756, + 3.4032232761383057, + 3.3889243602752686, + 3.4005582332611084, + 3.5055038928985596, + 3.438368797302246, + 3.484656810760498, + 3.5422170162200928, + 3.445354461669922, + 3.3111796379089355, + 3.423015594482422, + 3.5393779277801514, + 3.6509451866149902, + 3.336338758468628, + 3.29670786857605, + 3.426818370819092, + 3.328566789627075, + 3.3844714164733887, + 3.4112517833709717, + 3.5636484622955322, + 3.425518274307251, + 3.4412295818328857, + 3.4094297885894775, + 3.3766372203826904, + 3.5139832496643066, + 3.458186149597168, + 3.507279872894287, + 3.496953010559082, + 3.377554178237915, + 3.469693899154663, + 3.454655647277832, + 3.5336875915527344, + 3.5273373126983643, + 3.37514328956604, + 3.448410987854004, + 3.5311834812164307, + 3.49183988571167, + 3.400547981262207, + 3.4755239486694336, + 3.381580114364624, + 3.382789373397827, + 3.452671766281128, + 3.535970449447632, + 3.38816499710083, + 3.357426404953003, + 3.446195602416992, + 3.4592156410217285, + 3.3027307987213135, + 3.4380147457122803, + 3.6057374477386475, + 3.455495834350586, + 3.2684643268585205, + 3.404775381088257, + 3.300382375717163, + 3.3669509887695312, + 3.5608067512512207, + 3.4401497840881348, + 3.3411705493927, + 3.5030086040496826, + 3.3482346534729004, + 3.3771090507507324, + 3.473952531814575, + 3.3847451210021973, + 3.3430540561676025, + 3.510042667388916, + 3.4735381603240967, + 3.3619256019592285, + 3.2675092220306396, + 3.346371650695801, + 3.3767616748809814, + 3.405639171600342, + 3.457167148590088, + 3.308530330657959, + 3.3113298416137695, + 3.278743028640747, + 3.413912773132324, + 3.4595634937286377, + 3.406541585922241, + 3.4868412017822266, + 3.3957931995391846, + 3.3562090396881104, + 3.2614026069641113, + 3.391357183456421, + 3.4283289909362793, + 3.4440786838531494, + 3.49099063873291, + 3.340778350830078, + 3.4155759811401367, + 3.4601404666900635, + 3.4512760639190674, + 3.3004491329193115, + 3.3862569332122803, + 3.351926565170288, + 3.4133527278900146, + 3.338780164718628, + 3.4869167804718018, + 3.3152191638946533, + 3.5136518478393555, + 3.3554773330688477, + 3.3867645263671875, + 3.3495306968688965, + 3.345984697341919, + 3.4308249950408936, + 3.4303150177001953, + 3.5199601650238037, + 3.4910316467285156, + 3.4649291038513184, + 3.4006149768829346, + 3.3626673221588135, + 3.4696028232574463, + 3.381633758544922, + 3.508836507797241, + 3.3436548709869385, + 3.4736037254333496, + 3.4183874130249023, + 3.3136816024780273, + 3.3422532081604004, + 3.471705436706543, + 3.35752534866333, + 3.404803514480591, + 3.3889365196228027, + 3.3430821895599365, + 3.3310184478759766, + 3.432032346725464, + 3.1841604709625244, + 3.4805727005004883, + 3.3702757358551025, + 3.305739641189575, + 3.4784467220306396, + 3.324897050857544, + 3.463225841522217, + 3.4366300106048584, + 3.3261799812316895, + 3.3776206970214844, + 3.524388313293457, + 3.522526979446411, + 3.3974432945251465, + 3.5196688175201416, + 3.350332498550415, + 3.403498649597168, + 3.2932376861572266, + 3.40388560295105, + 3.3832809925079346, + 3.419851541519165, + 3.45295786857605, + 3.4577462673187256, + 3.3081254959106445, + 3.384937047958374, + 3.4978034496307373, + 3.4245336055755615, + 3.34360933303833, + 3.3390755653381348, + 3.410154342651367, + 3.3624966144561768, + 3.3334970474243164, + 3.4322478771209717, + 3.3756558895111084, + 3.2180187702178955, + 3.4744367599487305, + 3.3762190341949463, + 3.338434934616089, + 3.4045588970184326, + 3.202160358428955, + 3.5432369709014893, + 3.4271857738494873, + 3.5006701946258545, + 3.5054078102111816, + 3.4046432971954346, + 3.5219736099243164, + 3.385183095932007, + 3.320009469985962, + 3.2764532566070557, + 3.4027318954467773, + 3.4795727729797363 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 106.2673075668327, + 67.16440084432014, + 55.549881286047906, + 50.03679269852684, + 46.18827609651844, + 43.71893257980089, + 41.83207281641715, + 39.7724206891784, + 38.47888366703438, + 36.94645818048185, + 35.94861098211236, + 34.91582676505028, + 34.1303770789132, + 33.36968790277621, + 32.65702597751621, + 32.30969670667967, + 31.904951424066155, + 31.748687721626485, + 31.652141775193797, + 31.637063231236375 + ], + "best_ppl": 31.637063231236375, + "train_time_s": 10766.378124713898 +} \ No newline at end of file diff --git a/063_sparse_dispatch/single_block_softmax_results.json b/063_sparse_dispatch/single_block_softmax_results.json new file mode 100644 index 0000000000000000000000000000000000000000..f7fcf5e75273c8ad12d1a9f37bac47379dd2cc3c --- /dev/null +++ b/063_sparse_dispatch/single_block_softmax_results.json @@ -0,0 +1,2057 @@ +{ + "label": "single_block_softmax", + "seed": 42, + "attention_type": "softmax", + "num_thin_blocks": 1, + "total_params": 43276288, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 18.950483322143555, + 11.67675495147705, + 7.801235198974609, + 7.4238739013671875, + 7.362990379333496, + 7.185711860656738, + 7.066243648529053, + 6.969537734985352, + 6.862404823303223, + 6.739349365234375, + 6.660433292388916, + 6.565223217010498, + 6.448904991149902, + 6.371743679046631, + 6.349189758300781, + 6.304277420043945, + 6.183341026306152, + 6.058931827545166, + 5.987085342407227, + 6.098825931549072, + 5.880980491638184, + 5.944774150848389, + 5.733438491821289, + 5.812276363372803, + 5.767115592956543, + 5.636913776397705, + 5.653101921081543, + 5.537007808685303, + 5.5294952392578125, + 5.5115275382995605, + 5.429083347320557, + 5.4543681144714355, + 5.4163899421691895, + 5.454217433929443, + 5.478633880615234, + 5.4650678634643555, + 5.366127967834473, + 5.349563121795654, + 5.364129543304443, + 5.3564276695251465, + 5.253610134124756, + 5.2167463302612305, + 5.223422050476074, + 5.219032287597656, + 5.275977611541748, + 5.1119256019592285, + 5.108026027679443, + 5.220982074737549, + 5.171204090118408, + 5.14819860458374, + 4.9795660972595215, + 5.118494987487793, + 4.941152095794678, + 5.105832576751709, + 5.0309672355651855, + 5.053302764892578, + 4.944222450256348, + 4.920672416687012, + 4.975588321685791, + 5.035074234008789, + 4.994085788726807, + 5.030338287353516, + 5.0169758796691895, + 4.946869373321533, + 4.923838138580322, + 4.9557647705078125, + 4.908802032470703, + 4.893853664398193, + 4.740291595458984, + 4.804003715515137, + 4.777996063232422, + 4.896176815032959, + 4.735986232757568, + 4.817660808563232, + 4.790355205535889, + 4.811056137084961, + 4.772539138793945, + 4.832962512969971, + 4.848730564117432, + 4.951208591461182, + 4.777863502502441, + 4.8795342445373535, + 4.728009223937988, + 4.796872615814209, + 4.611936092376709, + 4.7261576652526855, + 4.805990695953369, + 4.723201274871826, + 4.761190414428711, + 4.865216255187988, + 4.7210307121276855, + 4.683020114898682, + 4.678096771240234, + 4.699315547943115, + 4.677720546722412, + 4.732811450958252, + 4.741580009460449, + 4.632826328277588, + 4.6750102043151855, + 4.6944661140441895, + 4.696829319000244, + 4.737736701965332, + 4.6026482582092285, + 4.745224475860596, + 4.556089878082275, + 4.60914421081543, + 4.681727409362793, + 4.61018180847168, + 4.742677211761475, + 4.767246723175049, + 4.472999095916748, + 4.549196720123291, + 4.664861679077148, + 4.584484100341797, + 4.491100788116455, + 4.695133686065674, + 4.585545063018799, + 4.5925774574279785, + 4.536326885223389, + 4.595555782318115, + 4.578157424926758, + 4.5497727394104, + 4.5473313331604, + 4.4736504554748535, + 4.488591194152832, + 4.595966815948486, + 4.37197208404541, + 4.587179183959961, + 4.487050533294678, + 4.374406337738037, + 4.390039443969727, + 4.499502182006836, + 4.453010082244873, + 4.3252644538879395, + 4.316915988922119, + 4.324896335601807, + 4.399693965911865, + 4.5097527503967285, + 4.485695838928223, + 4.44575834274292, + 4.35549783706665, + 4.39393424987793, + 4.33527946472168, + 4.398732662200928, + 4.477471828460693, + 4.396413326263428, + 4.336357593536377, + 4.385487079620361, + 4.435326099395752, + 4.419784069061279, + 4.463594913482666, + 4.421060562133789, + 4.375502109527588, + 4.46051025390625, + 4.397774696350098, + 4.324289798736572, + 4.211759567260742, + 4.416100978851318, + 4.340286731719971, + 4.305440902709961, + 4.299349784851074, + 4.360773086547852, + 4.433974266052246, + 4.273390769958496, + 4.399841785430908, + 4.312375545501709, + 4.441518783569336, + 4.351953029632568, + 4.347447872161865, + 4.382948398590088, + 4.546870708465576, + 4.439817428588867, + 4.173521041870117, + 4.370694160461426, + 4.263620376586914, + 4.394693374633789, + 4.3729472160339355, + 4.326096534729004, + 4.463105201721191, + 4.302278518676758, + 4.368746757507324, + 4.239899635314941, + 4.319424629211426, + 4.3518571853637695, + 4.283227920532227, + 4.284939765930176, + 4.328218460083008, + 4.291415214538574, + 4.23237419128418, + 4.312056541442871, + 4.30198335647583, + 4.25791072845459, + 4.253021717071533, + 4.175677299499512, + 4.249844074249268, + 4.266523361206055, + 4.2835283279418945, + 4.299447536468506, + 4.171782493591309, + 4.241240501403809, + 4.29631233215332, + 4.237198829650879, + 4.357957363128662, + 4.301965713500977, + 4.290083885192871, + 4.158411502838135, + 4.387808322906494, + 4.280752658843994, + 4.311679363250732, + 4.219004154205322, + 4.306078910827637, + 4.2276530265808105, + 4.189144611358643, + 4.207414150238037, + 4.270448207855225, + 4.151742935180664, + 4.26265811920166, + 4.291469573974609, + 4.388403415679932, + 4.249612808227539, + 4.256258010864258, + 4.228755474090576, + 4.2462615966796875, + 4.388959884643555, + 4.275312900543213, + 4.319893836975098, + 4.226113319396973, + 4.209846019744873, + 4.174570560455322, + 4.342449188232422, + 4.309144496917725, + 4.299278259277344, + 4.268965244293213, + 4.265798568725586, + 4.165305137634277, + 4.280099868774414, + 4.1499810218811035, + 4.245143890380859, + 4.297135353088379, + 4.192325115203857, + 4.049238681793213, + 3.9973368644714355, + 4.2309041023254395, + 4.20125675201416, + 4.138364315032959, + 4.218252658843994, + 4.279767036437988, + 4.202349662780762, + 4.288081645965576, + 4.113224029541016, + 4.351134777069092, + 4.196523189544678, + 4.125522136688232, + 4.12955904006958, + 4.145206451416016, + 4.053562164306641, + 4.133852958679199, + 4.1112775802612305, + 4.230678558349609, + 4.162778377532959, + 4.085521697998047, + 4.107199668884277, + 4.170362949371338, + 4.037103652954102, + 4.066344261169434, + 4.161942958831787, + 4.2515106201171875, + 4.111747741699219, + 4.025495529174805, + 4.073416709899902, + 4.2005205154418945, + 4.033171653747559, + 4.046066761016846, + 4.102634429931641, + 4.142488479614258, + 4.139826774597168, + 4.134488582611084, + 4.1508073806762695, + 4.058198928833008, + 4.287525653839111, + 4.086806774139404, + 4.156717777252197, + 4.162199974060059, + 4.228703498840332, + 4.1027960777282715, + 4.170993328094482, + 4.062784194946289, + 3.997126579284668, + 4.143864631652832, + 4.205285549163818, + 4.013584613800049, + 4.048158168792725, + 4.1121978759765625, + 4.014550685882568, + 4.084421634674072, + 4.031282901763916, + 4.11932897567749, + 4.204909324645996, + 4.131067752838135, + 4.134816646575928, + 4.209792137145996, + 4.095869541168213, + 3.983381509780884, + 4.205753326416016, + 4.105737686157227, + 4.1121344566345215, + 4.154271125793457, + 4.057896137237549, + 4.20375919342041, + 4.079860687255859, + 4.0531439781188965, + 4.158442497253418, + 3.958033323287964, + 3.9717257022857666, + 4.126176357269287, + 4.009584426879883, + 3.9852845668792725, + 4.192967891693115, + 4.0467915534973145, + 4.043088436126709, + 3.968761682510376, + 4.154726982116699, + 4.105208873748779, + 4.090228080749512, + 4.194319248199463, + 4.0915350914001465, + 4.130237102508545, + 4.084076404571533, + 4.10639762878418, + 4.122158527374268, + 4.082856178283691, + 3.928710460662842, + 4.06343412399292, + 4.0496978759765625, + 4.065098762512207, + 4.0750346183776855, + 4.022726058959961, + 4.111993789672852, + 4.030699729919434, + 4.004004955291748, + 4.141550064086914, + 4.0210652351379395, + 4.109006881713867, + 4.119620323181152, + 4.141619682312012, + 4.158108711242676, + 4.005853176116943, + 4.061405181884766, + 4.148601055145264, + 4.021026611328125, + 4.135341644287109, + 4.099128246307373, + 4.039211750030518, + 4.00679874420166, + 4.098082542419434, + 4.020391941070557, + 4.068251132965088, + 3.9530813694000244, + 4.044948577880859, + 4.010799407958984, + 4.160805702209473, + 4.019221305847168, + 4.044930934906006, + 4.0458807945251465, + 3.9513704776763916, + 3.964496612548828, + 4.163477897644043, + 4.107147216796875, + 4.105037689208984, + 4.032041072845459, + 4.1067328453063965, + 4.04807710647583, + 3.953366756439209, + 4.125462055206299, + 4.12260103225708, + 3.8667259216308594, + 3.9384043216705322, + 3.94024920463562, + 4.001217842102051, + 3.9037322998046875, + 4.020877361297607, + 3.921207904815674, + 3.802927255630493, + 3.9327552318573, + 3.973306655883789, + 3.989556312561035, + 4.0990800857543945, + 3.877115488052368, + 4.026162624359131, + 3.924454689025879, + 3.9135005474090576, + 3.923680067062378, + 3.9545605182647705, + 3.9379587173461914, + 4.053014278411865, + 4.003915786743164, + 4.120864391326904, + 4.063857555389404, + 4.011681079864502, + 3.938612222671509, + 3.958339214324951, + 3.958754777908325, + 3.8726890087127686, + 3.90179181098938, + 3.8763551712036133, + 4.01529598236084, + 3.9995694160461426, + 3.933711051940918, + 3.8642313480377197, + 4.022371292114258, + 3.985079765319824, + 3.9130914211273193, + 4.060726165771484, + 3.9274678230285645, + 4.092141151428223, + 3.9622223377227783, + 3.892408609390259, + 3.9159657955169678, + 3.9718713760375977, + 4.015605449676514, + 3.83341121673584, + 3.9403529167175293, + 3.76450514793396, + 3.915649890899658, + 3.9205989837646484, + 4.003984451293945, + 4.0188307762146, + 4.004805088043213, + 4.024664878845215, + 3.9711835384368896, + 4.046971321105957, + 4.0785136222839355, + 3.940176248550415, + 3.87717604637146, + 3.8877546787261963, + 4.044753074645996, + 3.892911434173584, + 3.990975856781006, + 4.150217056274414, + 3.9087576866149902, + 3.9051122665405273, + 3.960043430328369, + 3.891169786453247, + 4.011144161224365, + 4.0064778327941895, + 4.003674507141113, + 3.924546957015991, + 4.050793170928955, + 3.9218125343322754, + 3.932589054107666, + 3.966135025024414, + 3.933237314224243, + 3.947518825531006, + 3.8816795349121094, + 4.007580757141113, + 3.963700532913208, + 3.944959878921509, + 3.9433646202087402, + 3.8259098529815674, + 3.9359757900238037, + 3.9518673419952393, + 3.912324905395508, + 3.9258675575256348, + 3.9904625415802, + 3.898085117340088, + 4.073469161987305, + 3.9918155670166016, + 4.003251552581787, + 3.950653076171875, + 3.931232213973999, + 3.9908971786499023, + 3.9689714908599854, + 3.9384701251983643, + 3.9166150093078613, + 3.9865593910217285, + 4.011183261871338, + 3.994109869003296, + 3.83948016166687, + 4.028087139129639, + 3.899587392807007, + 3.897564172744751, + 4.063424110412598, + 3.981106758117676, + 4.013476371765137, + 4.004257678985596, + 4.054891586303711, + 3.9513137340545654, + 3.9399900436401367, + 3.9373486042022705, + 4.028065204620361, + 3.8423352241516113, + 3.9765801429748535, + 3.8266472816467285, + 3.8623344898223877, + 4.003678798675537, + 3.989609956741333, + 3.940091371536255, + 3.8760201930999756, + 3.8952512741088867, + 4.029074668884277, + 4.014760494232178, + 3.924715280532837, + 3.965733289718628, + 3.932142496109009, + 3.9645895957946777, + 3.8117427825927734, + 3.8522591590881348, + 3.859795093536377, + 3.8465399742126465, + 4.023390769958496, + 3.900315999984741, + 3.9661478996276855, + 3.751953363418579, + 3.762110948562622, + 3.9333479404449463, + 3.925215721130371, + 3.850569486618042, + 3.905437707901001, + 3.8772923946380615, + 3.8742454051971436, + 3.8861145973205566, + 4.008763313293457, + 3.946051836013794, + 3.9880826473236084, + 3.8263351917266846, + 3.7801406383514404, + 3.909385919570923, + 3.880171060562134, + 3.7997453212738037, + 3.8034355640411377, + 3.929192304611206, + 4.020009994506836, + 3.8365821838378906, + 3.858659505844116, + 3.8526506423950195, + 3.8925178050994873, + 3.8817636966705322, + 3.9069061279296875, + 3.824958324432373, + 3.7657470703125, + 3.980156660079956, + 3.9155046939849854, + 3.8689076900482178, + 3.9387733936309814, + 3.881807327270508, + 3.9724228382110596, + 3.7867345809936523, + 3.930488348007202, + 3.9428274631500244, + 3.9135642051696777, + 3.877722978591919, + 3.910334825515747, + 3.8369884490966797, + 3.861724853515625, + 3.889186382293701, + 3.957108497619629, + 3.887181282043457, + 3.9201107025146484, + 3.7880659103393555, + 3.940288543701172, + 3.7941927909851074, + 3.9112837314605713, + 3.9911251068115234, + 3.875417709350586, + 3.8729312419891357, + 3.939657688140869, + 3.8774380683898926, + 3.8947720527648926, + 3.8879518508911133, + 3.8483738899230957, + 3.910518169403076, + 3.8039116859436035, + 3.797755002975464, + 3.949206829071045, + 3.7909886837005615, + 3.8959784507751465, + 3.8253865242004395, + 3.8371217250823975, + 3.7370588779449463, + 3.9562740325927734, + 3.981719970703125, + 3.9297971725463867, + 3.834547996520996, + 3.972832202911377, + 3.679783582687378, + 3.816314935684204, + 3.9664859771728516, + 3.8879246711730957, + 3.9142770767211914, + 3.6367595195770264, + 3.9335010051727295, + 3.8310704231262207, + 3.8721365928649902, + 3.908050775527954, + 3.7614171504974365, + 3.910585403442383, + 3.8989384174346924, + 3.9752120971679688, + 3.737285614013672, + 3.8182928562164307, + 3.842865467071533, + 3.8614213466644287, + 3.837566614151001, + 3.717714548110962, + 3.692047595977783, + 3.917330741882324, + 3.8650267124176025, + 3.8770556449890137, + 3.74717116355896, + 3.8194692134857178, + 3.891803741455078, + 3.909700393676758, + 3.928351879119873, + 3.9218409061431885, + 3.839691400527954, + 3.9744913578033447, + 3.9385645389556885, + 3.774235248565674, + 3.7037723064422607, + 3.9477338790893555, + 3.803666830062866, + 3.7502832412719727, + 4.030383586883545, + 3.79972767829895, + 3.78256893157959, + 3.817772388458252, + 3.7851688861846924, + 3.8904364109039307, + 3.826129913330078, + 3.833627223968506, + 3.8022522926330566, + 3.80548095703125, + 3.7146480083465576, + 3.9127156734466553, + 3.650886058807373, + 3.7717819213867188, + 3.6992440223693848, + 3.7869186401367188, + 3.854332208633423, + 3.664557695388794, + 3.864947557449341, + 3.7594926357269287, + 3.8027074337005615, + 3.8232100009918213, + 3.721264362335205, + 3.896383047103882, + 3.934980869293213, + 3.7507221698760986, + 3.8016598224639893, + 3.882373571395874, + 3.7723023891448975, + 3.7977943420410156, + 3.944270610809326, + 3.7446742057800293, + 3.8244740962982178, + 3.748905658721924, + 3.802785634994507, + 3.7781636714935303, + 3.650390386581421, + 3.7736377716064453, + 3.7108066082000732, + 3.8488781452178955, + 3.771616220474243, + 3.812952995300293, + 3.788275957107544, + 3.7725865840911865, + 3.6856460571289062, + 3.7620394229888916, + 3.9856321811676025, + 3.8060355186462402, + 3.8435559272766113, + 3.844554901123047, + 3.8242697715759277, + 3.721468210220337, + 3.766179084777832, + 3.811587333679199, + 3.803854465484619, + 3.891697883605957, + 3.8113670349121094, + 3.774960517883301, + 3.8566019535064697, + 3.8693675994873047, + 3.8722968101501465, + 3.756486177444458, + 3.798752784729004, + 3.88215970993042, + 3.761211395263672, + 3.7907655239105225, + 3.7354187965393066, + 3.7967841625213623, + 3.7539005279541016, + 3.8297722339630127, + 3.8168277740478516, + 3.8824727535247803, + 3.7315478324890137, + 3.8228187561035156, + 3.856675148010254, + 3.917656660079956, + 3.739640951156616, + 3.812755823135376, + 3.7760703563690186, + 3.6982533931732178, + 3.8077285289764404, + 3.8919436931610107, + 3.8139829635620117, + 3.7685117721557617, + 3.799201011657715, + 3.607483148574829, + 3.83086895942688, + 3.8583896160125732, + 3.831878662109375, + 3.7239246368408203, + 3.756270408630371, + 3.7420787811279297, + 3.837968349456787, + 3.939107894897461, + 3.869856595993042, + 3.7867393493652344, + 3.883479595184326, + 3.616891622543335, + 3.6705222129821777, + 3.9260807037353516, + 3.7293457984924316, + 3.7818970680236816, + 3.894968032836914, + 3.7964065074920654, + 3.837463855743408, + 3.854091167449951, + 3.631913423538208, + 3.81949520111084, + 3.823535203933716, + 3.7715001106262207, + 3.806344985961914, + 3.6470906734466553, + 3.844491958618164, + 3.7888243198394775, + 3.8963799476623535, + 3.717313051223755, + 3.7726879119873047, + 3.7289857864379883, + 3.6613128185272217, + 3.757850170135498, + 3.8359153270721436, + 3.713264226913452, + 3.841022253036499, + 3.807058572769165, + 3.7566347122192383, + 3.837775230407715, + 3.8150439262390137, + 3.7913098335266113, + 3.8072400093078613, + 3.7737772464752197, + 3.8426809310913086, + 3.7065179347991943, + 3.7919323444366455, + 3.755836248397827, + 3.7930867671966553, + 3.8800363540649414, + 3.8432319164276123, + 3.7507426738739014, + 3.8310136795043945, + 3.7485592365264893, + 3.7014267444610596, + 3.7377755641937256, + 3.6202597618103027, + 3.76729154586792, + 3.681687593460083, + 3.703049659729004, + 3.6212847232818604, + 3.6876635551452637, + 3.5281777381896973, + 3.701296091079712, + 3.666363000869751, + 3.805131673812866, + 3.830793619155884, + 3.79860520362854, + 3.7070770263671875, + 3.7450010776519775, + 3.7629382610321045, + 3.810513496398926, + 3.7562406063079834, + 3.7743639945983887, + 3.79301381111145, + 3.8856730461120605, + 3.786557674407959, + 3.8688740730285645, + 3.6754844188690186, + 3.6998331546783447, + 3.6676435470581055, + 3.664551019668579, + 3.764082193374634, + 3.705434560775757, + 3.8207380771636963, + 3.71576189994812, + 3.8391029834747314, + 3.6864161491394043, + 3.710007667541504, + 3.7664899826049805, + 3.787871837615967, + 3.718609571456909, + 3.681097984313965, + 3.6884517669677734, + 3.713188648223877, + 3.730006217956543, + 3.6575849056243896, + 3.557605028152466, + 3.772960662841797, + 3.7286155223846436, + 3.756500482559204, + 3.7017533779144287, + 3.669320583343506, + 3.6980597972869873, + 3.6057960987091064, + 3.8002161979675293, + 3.6872925758361816, + 3.742007255554199, + 3.7822911739349365, + 3.6711361408233643, + 3.779385566711426, + 3.720327377319336, + 3.7796428203582764, + 3.6616690158843994, + 3.768519878387451, + 3.7989253997802734, + 3.801490306854248, + 3.6576197147369385, + 3.6563615798950195, + 3.694831371307373, + 3.672037124633789, + 3.5689585208892822, + 3.658757448196411, + 3.8330907821655273, + 3.6810355186462402, + 3.6783180236816406, + 3.743563175201416, + 3.8288023471832275, + 3.768003463745117, + 3.761592388153076, + 3.702737331390381, + 3.664194345474243, + 3.7658073902130127, + 3.7738852500915527, + 3.798574209213257, + 3.66668701171875, + 3.7421703338623047, + 3.535794496536255, + 3.793060302734375, + 3.7560081481933594, + 3.8084867000579834, + 3.7065553665161133, + 3.55182147026062, + 3.8447108268737793, + 3.837662935256958, + 3.762880563735962, + 3.719613790512085, + 3.839508533477783, + 3.6921703815460205, + 3.725283622741699, + 3.7355988025665283, + 3.739459991455078, + 3.6781952381134033, + 3.659903049468994, + 3.7553293704986572, + 3.7951929569244385, + 3.743237018585205, + 3.7445003986358643, + 3.6738879680633545, + 3.7885823249816895, + 3.7383992671966553, + 3.808155059814453, + 3.6609253883361816, + 3.7182118892669678, + 3.7941346168518066, + 3.6493170261383057, + 3.752800703048706, + 3.8117926120758057, + 3.7002174854278564, + 3.731961488723755, + 3.6126620769500732, + 3.628770112991333, + 3.843240976333618, + 3.5476489067077637, + 3.745260000228882, + 3.77480149269104, + 3.869443655014038, + 3.7645647525787354, + 3.7392868995666504, + 3.7317936420440674, + 3.5994787216186523, + 3.6849257946014404, + 3.5995662212371826, + 3.6058032512664795, + 3.699857234954834, + 3.789530038833618, + 3.7452752590179443, + 3.7282748222351074, + 3.6865553855895996, + 3.6514642238616943, + 3.5939810276031494, + 3.773181438446045, + 3.6700587272644043, + 3.72332763671875, + 3.6410820484161377, + 3.7721996307373047, + 3.5910842418670654, + 3.6633105278015137, + 3.5815203189849854, + 3.7802085876464844, + 3.7024261951446533, + 3.7293999195098877, + 3.733342409133911, + 3.6105356216430664, + 3.8358120918273926, + 3.6791791915893555, + 3.800497531890869, + 3.7870099544525146, + 3.644848346710205, + 3.7565042972564697, + 3.658454418182373, + 3.656222105026245, + 3.675899028778076, + 3.8019747734069824, + 3.6517081260681152, + 3.7532949447631836, + 3.617708921432495, + 3.66072154045105, + 3.634697437286377, + 3.644645929336548, + 3.7847931385040283, + 3.7320070266723633, + 3.718867540359497, + 3.59537935256958, + 3.7520294189453125, + 3.677774667739868, + 3.6788012981414795, + 3.6659698486328125, + 3.7671167850494385, + 3.6817519664764404, + 3.811164379119873, + 3.7432520389556885, + 3.670424222946167, + 3.73931884765625, + 3.6015915870666504, + 3.615633249282837, + 3.6335818767547607, + 3.647176742553711, + 3.781618595123291, + 3.7273600101470947, + 3.6435208320617676, + 3.685133695602417, + 3.7067360877990723, + 3.7971243858337402, + 3.634087324142456, + 3.605259418487549, + 3.572497606277466, + 3.7476401329040527, + 3.6139798164367676, + 3.7490904331207275, + 3.739250898361206, + 3.8281090259552, + 3.754079818725586, + 3.6421761512756348, + 3.7836105823516846, + 3.653491258621216, + 3.7947025299072266, + 3.701159954071045, + 3.7340338230133057, + 3.664579153060913, + 3.6212713718414307, + 3.4838523864746094, + 3.7053351402282715, + 3.6333119869232178, + 3.5496506690979004, + 3.727212429046631, + 3.630732774734497, + 3.6453592777252197, + 3.57543683052063, + 3.5974299907684326, + 3.7159597873687744, + 3.6830713748931885, + 3.7521655559539795, + 3.7027547359466553, + 3.6782431602478027, + 3.784303903579712, + 3.758408546447754, + 3.5873286724090576, + 3.788872718811035, + 3.509946823120117, + 3.6890616416931152, + 3.8560078144073486, + 3.7374327182769775, + 3.776792526245117, + 3.7680158615112305, + 3.6902544498443604, + 3.6658132076263428, + 3.778712272644043, + 3.6250481605529785, + 3.600534439086914, + 3.6504454612731934, + 3.6708924770355225 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 147.55903943306862, + 92.13781294471386, + 71.86103053204391, + 62.77059011200154, + 57.46928384281052, + 54.008081814626784, + 51.02353837008834, + 48.86681694912601, + 46.79757780520936, + 45.206052514101984, + 43.767516991354114, + 42.61593058781018, + 41.74635124867182, + 40.872169210045655, + 40.095870488460825, + 39.642903522297715, + 39.202021625010595, + 38.99872499969655, + 38.85597140011268, + 38.82829121156178 + ], + "best_ppl": 38.82829121156178, + "train_time_s": 7206.096261739731 +} \ No newline at end of file diff --git a/063_sparse_dispatch/softmax_1plus2of5_isoparam_best.pt b/063_sparse_dispatch/softmax_1plus2of5_isoparam_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..4469cd9ed57eca4120ee0c4c2274f20e13d621cd --- /dev/null +++ b/063_sparse_dispatch/softmax_1plus2of5_isoparam_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab31fa17f9a525739321f0c4abc900cb475ceaaca382a87d9c55816361ddd7d4 +size 350342369 diff --git a/063_sparse_dispatch/softmax_1plus2of5_isoparam_results.json b/063_sparse_dispatch/softmax_1plus2of5_isoparam_results.json new file mode 100644 index 0000000000000000000000000000000000000000..84956a192ee88f270f869c109fdebe918ef18798 --- /dev/null +++ b/063_sparse_dispatch/softmax_1plus2of5_isoparam_results.json @@ -0,0 +1,1245 @@ +{ + "label": "softmax_1plus2of5_isoparam", + "seed": 42, + "attention_mode": "sparse", + "routed_attn_type": "softmax", + "d_ff_thin": 1133, + "top_k_blocks": 2, + "num_thin_blocks": 5, + "num_shared_blocks": 1, + "total_params": 87525376, + "steps": [ + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 3.6795778274536133, + 3.697526693344116, + 3.7053492069244385, + 3.718477964401245, + 3.7866674661636353, + 3.6081098318099976, + 3.704477310180664, + 3.829926013946533, + 3.7251933813095093, + 3.8073281049728394, + 3.7871320247650146, + 3.786872148513794, + 3.7736047506332397, + 3.7217025756835938, + 3.6873844861984253, + 3.6924742460250854, + 3.7323840856552124, + 3.7570608854293823, + 3.6805890798568726, + 3.6996971368789673, + 3.766104817390442, + 3.6201752424240112, + 3.644681930541992, + 3.6239471435546875, + 3.7755268812179565, + 3.8123464584350586, + 3.767045497894287, + 3.6828285455703735, + 3.754228711128235, + 3.76906955242157, + 3.644185423851013, + 3.774750828742981, + 3.835475206375122, + 3.711121439933777, + 3.7427576780319214, + 3.655916452407837, + 3.762338399887085, + 3.8193639516830444, + 3.710065722465515, + 3.7111986875534058, + 3.817992925643921, + 3.798783540725708, + 3.7399154901504517, + 3.7991206645965576, + 3.7295517921447754, + 3.8028383255004883, + 3.6812922954559326, + 3.7051256895065308, + 3.6106083393096924, + 3.74478018283844, + 3.6379483938217163, + 3.6702725887298584, + 3.7511396408081055, + 3.721366286277771, + 3.7844913005828857, + 3.62894070148468, + 3.6829322576522827, + 3.7084527015686035, + 3.8089599609375, + 3.763788938522339, + 3.6556020975112915, + 3.7836464643478394, + 3.8557345867156982, + 3.69294810295105, + 3.723309636116028, + 3.770991563796997, + 3.6660773754119873, + 3.8311524391174316, + 3.866625666618347, + 3.782828688621521, + 3.725653052330017, + 3.732008457183838, + 3.758989930152893, + 3.722555160522461, + 3.7458553314208984, + 3.6871156692504883, + 3.762923002243042, + 3.799298405647278, + 3.7115726470947266, + 3.703971028327942, + 3.6969891786575317, + 3.725032925605774, + 3.843701124191284, + 3.7289159297943115, + 3.758110284805298, + 3.7578604221343994, + 3.5960768461227417, + 3.6871169805526733, + 3.703706979751587, + 3.7215405702590942, + 3.7847307920455933, + 3.6708943843841553, + 3.621226191520691, + 3.635392189025879, + 3.5860224962234497, + 3.6964439153671265, + 3.7528858184814453, + 3.6957212686538696, + 3.840804934501648, + 3.7205796241760254, + 3.8309866189956665, + 3.538601517677307, + 3.7344861030578613, + 3.6749496459960938, + 3.6572585105895996, + 3.6701409816741943, + 3.736455798149109, + 3.715731382369995, + 3.6266589164733887, + 3.6630163192749023, + 3.6318674087524414, + 3.576840877532959, + 3.7404452562332153, + 3.581531286239624, + 3.7848575115203857, + 3.6094744205474854, + 3.777257204055786, + 3.62459135055542, + 3.5626460313796997, + 3.714028477668762, + 3.5902390480041504, + 3.69442617893219, + 3.7107290029525757, + 3.669816255569458, + 3.679268002510071, + 3.7951250076293945, + 3.707628607749939, + 3.612892508506775, + 3.5330313444137573, + 3.5611977577209473, + 3.6428648233413696, + 3.700288772583008, + 3.5873422622680664, + 3.5990676879882812, + 3.596433162689209, + 3.716039299964905, + 3.622051477432251, + 3.491443395614624, + 3.513680934906006, + 3.5380001068115234, + 3.602882981300354, + 3.5445542335510254, + 3.6226149797439575, + 3.627338171005249, + 3.5205607414245605, + 3.5762553215026855, + 3.545136570930481, + 3.614107608795166, + 3.6360843181610107, + 3.659416079521179, + 3.487824320793152, + 3.5572426319122314, + 3.5979206562042236, + 3.644197702407837, + 3.654781222343445, + 3.6324843168258667, + 3.6607773303985596, + 3.522329568862915, + 3.6871129274368286, + 3.614904999732971, + 3.488986849784851, + 3.542314291000366, + 3.6484897136688232, + 3.6464580297470093, + 3.5874547958374023, + 3.524870991706848, + 3.674425959587097, + 3.6085243225097656, + 3.5658750534057617, + 3.7042503356933594, + 3.553512930870056, + 3.548550248146057, + 3.6706188917160034, + 3.645095467567444, + 3.55668044090271, + 3.5822378396987915, + 3.6016440391540527, + 3.556225538253784, + 3.63279128074646, + 3.5699894428253174, + 3.6355512142181396, + 3.630388617515564, + 3.612561821937561, + 3.4846739768981934, + 3.6369446516036987, + 3.5919374227523804, + 3.577948212623596, + 3.5337042808532715, + 3.5658562183380127, + 3.5345356464385986, + 3.572452187538147, + 3.56125271320343, + 3.5005342960357666, + 3.4810186624526978, + 3.6204116344451904, + 3.5691040754318237, + 3.539470076560974, + 3.6445990800857544, + 3.4639087915420532, + 3.5663182735443115, + 3.6824673414230347, + 3.599485754966736, + 3.5147403478622437, + 3.5945241451263428, + 3.6005771160125732, + 3.7005542516708374, + 3.504379987716675, + 3.6621443033218384, + 3.326747179031372, + 3.5012015104293823, + 3.563285708427429, + 3.667009115219116, + 3.545304298400879, + 3.628610372543335, + 3.7021230459213257, + 3.6503788232803345, + 3.5608593225479126, + 3.506203532218933, + 3.5235681533813477, + 3.460500717163086, + 3.4973747730255127, + 3.570344924926758, + 3.528477668762207, + 3.6254193782806396, + 3.562020778656006, + 3.482665777206421, + 3.5021356344223022, + 3.535904288291931, + 3.5064347982406616, + 3.64792263507843, + 3.554574966430664, + 3.6481330394744873, + 3.5406230688095093, + 3.5698267221450806, + 3.691638469696045, + 3.5300323963165283, + 3.597609519958496, + 3.6108938455581665, + 3.579294443130493, + 3.4440683126449585, + 3.5080240964889526, + 3.533715605735779, + 3.625132203102112, + 3.6367831230163574, + 3.5544899702072144, + 3.3636146783828735, + 3.6443897485733032, + 3.4711865186691284, + 3.5561052560806274, + 3.4984019994735718, + 3.4981842041015625, + 3.4947370290756226, + 3.5168116092681885, + 3.4508293867111206, + 3.4365720748901367, + 3.4471408128738403, + 3.4906235933303833, + 3.508253812789917, + 3.4255049228668213, + 3.420639991760254, + 3.3567440509796143, + 3.575275182723999, + 3.4569203853607178, + 3.5868563652038574, + 3.5721248388290405, + 3.4659719467163086, + 3.561330199241638, + 3.429133176803589, + 3.515749454498291, + 3.5182881355285645, + 3.5700128078460693, + 3.5410985946655273, + 3.383112907409668, + 3.5921117067337036, + 3.431297540664673, + 3.4455736875534058, + 3.497403860092163, + 3.4082192182540894, + 3.5580674409866333, + 3.4998576641082764, + 3.4610027074813843, + 3.5824499130249023, + 3.542527198791504, + 3.5239222049713135, + 3.572029709815979, + 3.543145179748535, + 3.4554425477981567, + 3.540600299835205, + 3.4416226148605347, + 3.3869214057922363, + 3.46581768989563, + 3.5979000329971313, + 3.570884108543396, + 3.4392027854919434, + 3.4125359058380127, + 3.5496268272399902, + 3.4899327754974365, + 3.520007610321045, + 3.429367184638977, + 3.468578338623047, + 3.565095067024231, + 3.418676972389221, + 3.4972423315048218, + 3.3780373334884644, + 3.485535144805908, + 3.496354341506958, + 3.4166370630264282, + 3.4363722801208496, + 3.412049412727356, + 3.4623416662216187, + 3.5808123350143433, + 3.4638571739196777, + 3.403816342353821, + 3.516021251678467, + 3.4802098274230957, + 3.410767912864685, + 3.4987375736236572, + 3.4998929500579834, + 3.557802438735962, + 3.5029804706573486, + 3.4324740171432495, + 3.5043834447860718, + 3.5740374326705933, + 3.6616352796554565, + 3.496820092201233, + 3.6463361978530884, + 3.56973397731781, + 3.4234389066696167, + 3.5504932403564453, + 3.455176830291748, + 3.5391829013824463, + 3.489363193511963, + 3.4027464389801025, + 3.5758544206619263, + 3.4476840496063232, + 3.526025891304016, + 3.401647448539734, + 3.416362762451172, + 3.3481801748275757, + 3.5426292419433594, + 3.460348963737488, + 3.4588500261306763, + 3.3335108757019043, + 3.6072468757629395, + 3.5262175798416138, + 3.519671678543091, + 3.441166877746582, + 3.474781632423401, + 3.5801810026168823, + 3.457207441329956, + 3.479654908180237, + 3.531905174255371, + 3.4020888805389404, + 3.425872802734375, + 3.4882463216781616, + 3.5902018547058105, + 3.3968416452407837, + 3.4411253929138184, + 3.5645008087158203, + 3.489092230796814, + 3.5700818300247192, + 3.465645670890808, + 3.407264471054077, + 3.596999406814575, + 3.5408990383148193, + 3.497345447540283, + 3.459367513656616, + 3.5594969987869263, + 3.55993115901947, + 3.5836949348449707, + 3.5434157848358154, + 3.5420433282852173, + 3.4959208965301514, + 3.3613606691360474, + 3.448341965675354, + 3.54560387134552, + 3.596921443939209, + 3.5869951248168945, + 3.3712698221206665, + 3.308046817779541, + 3.37571120262146, + 3.389791488647461, + 3.337800979614258, + 3.4388303756713867, + 3.4621334075927734, + 3.3809734582901, + 3.3962124586105347, + 3.4597151279449463, + 3.379981279373169, + 3.3347833156585693, + 3.348116636276245, + 3.3747897148132324, + 3.3848694562911987, + 3.481292724609375, + 3.3324496746063232, + 3.4067212343215942, + 3.468776822090149, + 3.320321202278137, + 3.3341251611709595, + 3.251630663871765, + 3.440803289413452, + 3.4470489025115967, + 3.404591917991638, + 3.4297327995300293, + 3.3693329095840454, + 3.3203957080841064, + 3.510250210762024, + 3.4158403873443604, + 3.3806605339050293, + 3.482473134994507, + 3.415072798728943, + 3.47298800945282, + 3.480868697166443, + 3.4347089529037476, + 3.4081926345825195, + 3.421848773956299, + 3.4699379205703735, + 3.397038459777832, + 3.420883893966675, + 3.3363622426986694, + 3.4170243740081787, + 3.3376227617263794, + 3.5184837579727173, + 3.450385332107544, + 3.5086255073547363, + 3.496058225631714, + 3.4330594539642334, + 3.4542367458343506, + 3.287795901298523, + 3.4420443773269653, + 3.418677806854248, + 3.346487045288086, + 3.3680862188339233, + 3.320886492729187, + 3.4374438524246216, + 3.412964701652527, + 3.347928524017334, + 3.504520535469055, + 3.394792318344116, + 3.39081871509552, + 3.3627172708511353, + 3.4066070318222046, + 3.3899080753326416, + 3.4650217294692993, + 3.438991904258728, + 3.4248948097229004, + 3.4316375255584717, + 3.446777105331421, + 3.308859348297119, + 3.3701573610305786, + 3.40520179271698, + 3.510816216468811, + 3.3559999465942383, + 3.4511547088623047, + 3.6002317667007446, + 3.5258415937423706, + 3.45327091217041, + 3.4995752573013306, + 3.379241943359375, + 3.464755415916443, + 3.4942822456359863, + 3.3858143091201782, + 3.3139976263046265, + 3.3063331842422485, + 3.358971118927002, + 3.4014700651168823, + 3.360451340675354, + 3.3461172580718994, + 3.431864857673645, + 3.385540723800659, + 3.3281480073928833, + 3.4278790950775146, + 3.3388779163360596, + 3.4048988819122314, + 3.5369584560394287, + 3.472854256629944, + 3.3370258808135986, + 3.38387668132782, + 3.4203898906707764, + 3.4621587991714478, + 3.397251009941101, + 3.479302167892456, + 3.6035306453704834, + 3.4556398391723633, + 3.448507785797119, + 3.4083526134490967, + 3.4532275199890137, + 3.361177921295166, + 3.2911654710769653, + 3.4382611513137817, + 3.4670538902282715, + 3.4168920516967773, + 3.511977791786194, + 3.4773463010787964, + 3.4049710035324097, + 3.3440905809402466, + 3.4151792526245117, + 3.4503477811813354, + 3.289017915725708, + 3.336042642593384, + 3.5523338317871094, + 3.4840012788772583, + 3.4183040857315063, + 3.3875426054000854, + 3.485413074493408, + 3.4826362133026123, + 3.3469966650009155, + 3.419284462928772, + 3.418288230895996, + 3.3949015140533447, + 3.3248231410980225, + 3.3631397485733032, + 3.4677414894104004, + 3.300929069519043, + 3.4042211771011353, + 3.3933422565460205, + 3.4120384454727173, + 3.3251655101776123, + 3.3875129222869873, + 3.370638370513916, + 3.4465891122817993, + 3.436778426170349, + 3.435518741607666, + 3.5497233867645264, + 3.377534866333008, + 3.4133485555648804, + 3.3372970819473267, + 3.328748941421509, + 3.4128741025924683, + 3.34800922870636, + 3.3247209787368774, + 3.3216828107833862, + 3.373457431793213, + 3.3958133459091187, + 3.1877468824386597, + 3.377853751182556, + 3.351359724998474, + 3.2406320571899414, + 3.38814640045166, + 3.337508797645569, + 3.3328685760498047, + 3.346353054046631, + 3.308436393737793, + 3.2905495166778564, + 3.3205646276474, + 3.4502962827682495, + 3.473746657371521, + 3.3885356187820435, + 3.326636791229248, + 3.430145263671875, + 3.393003821372986, + 3.325126886367798, + 3.3884934186935425, + 3.368176817893982, + 3.356015205383301, + 3.4074792861938477, + 3.4053927659988403, + 3.5102670192718506, + 3.4332103729248047, + 3.450658440589905, + 3.3961682319641113, + 3.383711099624634, + 3.4405062198638916, + 3.5686984062194824, + 3.4917306900024414, + 3.2371299266815186, + 3.362844944000244, + 3.378070116043091, + 3.3240257501602173, + 3.2589298486709595, + 3.47115695476532, + 3.3988969326019287, + 3.2744603157043457, + 3.3916581869125366, + 3.348422050476074, + 3.4210407733917236, + 3.308065414428711, + 3.3476678133010864, + 3.341929793357849, + 3.4323389530181885, + 3.589371681213379, + 3.4208425283432007, + 3.393231511116028, + 3.413000226020813, + 3.4675322771072388, + 3.4166876077651978, + 3.262590765953064, + 3.301022529602051, + 3.3850667476654053, + 3.44814133644104, + 3.3999937772750854, + 3.3228288888931274, + 3.3426231145858765, + 3.518585443496704, + 3.404793620109558, + 3.342239737510681, + 3.2992067337036133, + 3.3825806379318237, + 3.3971471786499023, + 3.362206816673279 + ], + "val_steps": [ + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 39.28121846747502, + 37.81624379543061, + 36.46840781046551, + 35.31164322969326, + 34.3493612658387, + 33.62958230122116, + 32.974784969138284, + 32.62008524868141, + 32.162445162632004, + 31.88693227959521, + 31.848502329216814, + 31.817747270764205 + ], + "best_ppl": 31.817747270764205, + "train_time_s": 12577.746372938156 +} \ No newline at end of file diff --git a/063_sparse_dispatch/softmax_1plus2of5_isoparam_s137_best.pt b/063_sparse_dispatch/softmax_1plus2of5_isoparam_s137_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f07ba5bd4f6bbabb873872dd8d6ccaf805a7133 --- /dev/null +++ b/063_sparse_dispatch/softmax_1plus2of5_isoparam_s137_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:338a94badb3c152d3a28ae613869112b347d2d4304a46e3f6a92cbf8f328b1d5 +size 350345254 diff --git a/063_sparse_dispatch/softmax_1plus2of5_isoparam_s137_results.json b/063_sparse_dispatch/softmax_1plus2of5_isoparam_s137_results.json new file mode 100644 index 0000000000000000000000000000000000000000..365f847bc113da1dccb96c38d9655c01b4516cd7 --- /dev/null +++ b/063_sparse_dispatch/softmax_1plus2of5_isoparam_s137_results.json @@ -0,0 +1,2056 @@ +{ + "label": "softmax_1plus2of5_isoparam_s137", + "seed": 137, + "d_ff_thin": 1133, + "total_params": 87525376, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 18.908164978027344, + 13.085579872131348, + 7.822511434555054, + 7.281003713607788, + 7.060247898101807, + 6.991706848144531, + 6.8858256340026855, + 6.8389739990234375, + 6.663103818893433, + 6.5918967723846436, + 6.57911229133606, + 6.322447299957275, + 6.3316121101379395, + 6.28877067565918, + 6.220341444015503, + 6.184633731842041, + 5.983234405517578, + 5.869335651397705, + 5.97298264503479, + 5.789786338806152, + 5.721731901168823, + 5.729383230209351, + 5.612488031387329, + 5.572798728942871, + 5.550717353820801, + 5.4498679637908936, + 5.445030689239502, + 5.439234495162964, + 5.393899202346802, + 5.391282558441162, + 5.329241752624512, + 5.3700995445251465, + 5.351703882217407, + 5.143979072570801, + 5.187432765960693, + 5.22903037071228, + 5.091721773147583, + 5.296994924545288, + 5.142261266708374, + 5.117093324661255, + 5.217156171798706, + 5.0784995555877686, + 4.997483015060425, + 5.047545433044434, + 4.89588475227356, + 5.0195512771606445, + 4.978024005889893, + 4.841375112533569, + 4.967408657073975, + 5.035736322402954, + 4.960970640182495, + 4.953664541244507, + 4.9694695472717285, + 4.982593774795532, + 4.829094171524048, + 4.761146783828735, + 4.867039680480957, + 4.786496877670288, + 4.897637367248535, + 4.879412412643433, + 4.824340343475342, + 4.735453844070435, + 4.716433525085449, + 4.810197830200195, + 4.810604572296143, + 4.647815465927124, + 4.68097448348999, + 4.812811613082886, + 4.580227613449097, + 4.652504920959473, + 4.6049964427948, + 4.794703483581543, + 4.770648956298828, + 4.751653671264648, + 4.702939987182617, + 4.680419921875, + 4.68550705909729, + 4.615073204040527, + 4.650238513946533, + 4.647583484649658, + 4.674779891967773, + 4.5366904735565186, + 4.491217851638794, + 4.574831485748291, + 4.554784536361694, + 4.719508409500122, + 4.575759172439575, + 4.532731056213379, + 4.456543207168579, + 4.618383407592773, + 4.555837869644165, + 4.516757488250732, + 4.5639588832855225, + 4.584312677383423, + 4.5911009311676025, + 4.5163867473602295, + 4.498837471008301, + 4.488189220428467, + 4.490353584289551, + 4.459977388381958, + 4.443003416061401, + 4.47085976600647, + 4.642090559005737, + 4.450477123260498, + 4.420819044113159, + 4.544927597045898, + 4.5134828090667725, + 4.498741149902344, + 4.389774560928345, + 4.452660799026489, + 4.434351205825806, + 4.370956659317017, + 4.494096994400024, + 4.342342138290405, + 4.400418996810913, + 4.3757219314575195, + 4.350195646286011, + 4.4460508823394775, + 4.284417629241943, + 4.3481457233428955, + 4.332949876785278, + 4.325083017349243, + 4.436793088912964, + 4.415987968444824, + 4.255106687545776, + 4.383281946182251, + 4.193180084228516, + 4.177347421646118, + 4.3193581104278564, + 4.292854070663452, + 4.22937273979187, + 4.26111626625061, + 4.253042459487915, + 4.258862733840942, + 4.300230503082275, + 4.305810213088989, + 4.317568302154541, + 4.310664415359497, + 4.40825891494751, + 4.1915788650512695, + 4.249022006988525, + 4.086025357246399, + 4.160298585891724, + 4.199846982955933, + 4.232187986373901, + 4.3195273876190186, + 4.288490056991577, + 4.2976930141448975, + 4.396856784820557, + 4.311121702194214, + 4.272506475448608, + 4.253288745880127, + 4.16972541809082, + 4.2532947063446045, + 4.079798936843872, + 4.195306301116943, + 4.341918230056763, + 4.1184375286102295, + 4.36646580696106, + 4.262754201889038, + 4.286304473876953, + 4.36140251159668, + 4.132350444793701, + 4.176380157470703, + 4.323575258255005, + 4.245307922363281, + 4.185797452926636, + 4.205713272094727, + 4.276248455047607, + 4.256350755691528, + 4.129642963409424, + 4.219253778457642, + 4.111011028289795, + 4.127743482589722, + 4.222842216491699, + 4.154832363128662, + 4.220425128936768, + 4.220592260360718, + 4.112103223800659, + 4.18463397026062, + 4.094326972961426, + 4.081420660018921, + 4.1444244384765625, + 4.115433692932129, + 4.022534132003784, + 4.100844144821167, + 4.004475235939026, + 4.3128721714019775, + 4.176812648773193, + 4.0068279504776, + 4.200674057006836, + 4.111189365386963, + 4.140975475311279, + 4.101790428161621, + 4.118133306503296, + 4.055137634277344, + 4.2104151248931885, + 4.258645534515381, + 4.109823226928711, + 4.211456775665283, + 4.141435861587524, + 4.105766296386719, + 4.112382888793945, + 4.0779736042022705, + 4.068097352981567, + 4.164934396743774, + 4.018582582473755, + 4.056536436080933, + 4.0925023555755615, + 4.033008456230164, + 4.0269869565963745, + 3.9344595670700073, + 4.104901075363159, + 4.144994020462036, + 4.165896415710449, + 4.038588523864746, + 4.016256809234619, + 4.069590449333191, + 4.02390730381012, + 4.0970107316970825, + 4.103824138641357, + 3.9852592945098877, + 4.081346750259399, + 4.092799067497253, + 4.001003623008728, + 4.038859128952026, + 4.133469581604004, + 3.938841938972473, + 4.108682155609131, + 3.936378836631775, + 4.165898084640503, + 4.0400474071502686, + 4.082904934883118, + 4.061576008796692, + 4.012383580207825, + 4.122205018997192, + 4.038534641265869, + 4.113759756088257, + 3.8871008157730103, + 4.128990411758423, + 4.009798884391785, + 3.9846150875091553, + 4.05653190612793, + 4.0250325202941895, + 4.1765406131744385, + 4.098313808441162, + 3.9892382621765137, + 4.085582733154297, + 4.026671409606934, + 3.954818606376648, + 3.9434906244277954, + 3.9800360202789307, + 4.030682563781738, + 3.9451619386672974, + 3.9019113779067993, + 3.9223450422286987, + 3.8236968517303467, + 4.059241771697998, + 3.949742078781128, + 3.961297035217285, + 3.956691026687622, + 3.9287099838256836, + 3.869903326034546, + 3.9549121856689453, + 3.9665610790252686, + 3.924416184425354, + 4.028702974319458, + 3.8829712867736816, + 3.923418164253235, + 3.9114187955856323, + 3.983283758163452, + 3.8593506813049316, + 3.944161295890808, + 3.815315842628479, + 3.976295232772827, + 3.860858678817749, + 4.002341270446777, + 3.8358237743377686, + 3.9936972856521606, + 3.7843528985977173, + 3.924237608909607, + 3.8557475805282593, + 3.9220513105392456, + 3.893330693244934, + 3.9314751625061035, + 3.9250365495681763, + 3.762658476829529, + 3.9009474515914917, + 3.907091975212097, + 4.041457176208496, + 3.8946404457092285, + 3.9357798099517822, + 3.9209495782852173, + 3.8583874702453613, + 3.798043966293335, + 3.9251108169555664, + 3.8273041248321533, + 3.9479304552078247, + 3.9944785833358765, + 3.9352864027023315, + 3.838820457458496, + 4.114758253097534, + 4.072821378707886, + 3.8527027368545532, + 4.0264599323272705, + 3.783095359802246, + 3.9376132488250732, + 3.995251774787903, + 3.836144208908081, + 3.831853985786438, + 3.810583233833313, + 3.957505226135254, + 3.735299587249756, + 4.010387897491455, + 3.847371220588684, + 4.016918897628784, + 3.7971506118774414, + 3.8999786376953125, + 4.03769052028656, + 3.8522528409957886, + 3.882016897201538, + 3.823309063911438, + 3.8630943298339844, + 3.947695016860962, + 3.868089199066162, + 3.8870826959609985, + 3.9626740217208862, + 3.8980493545532227, + 3.9038840532302856, + 3.8935776948928833, + 3.854085922241211, + 3.805997133255005, + 3.9076792001724243, + 3.879209876060486, + 3.907136559486389, + 3.904096245765686, + 3.9064871072769165, + 3.841651678085327, + 3.7490824460983276, + 3.8200424909591675, + 3.954687714576721, + 3.9178764820098877, + 4.009877681732178, + 3.8612585067749023, + 3.915524482727051, + 3.9398813247680664, + 3.84390127658844, + 4.003106236457825, + 3.852410078048706, + 3.887058734893799, + 3.814720630645752, + 3.8520132303237915, + 3.880050301551819, + 3.788390040397644, + 3.8307589292526245, + 4.0275843143463135, + 3.8104031085968018, + 3.882116675376892, + 3.8298323154449463, + 3.8533328771591187, + 3.880424737930298, + 3.922685980796814, + 3.932114839553833, + 3.755545973777771, + 3.8541570901870728, + 3.844083309173584, + 3.846494436264038, + 3.8250073194503784, + 3.990297794342041, + 3.8308732509613037, + 3.8946306705474854, + 3.8766520023345947, + 3.716951608657837, + 3.9248329401016235, + 3.806157946586609, + 3.838073253631592, + 3.864199638366699, + 3.8080579042434692, + 3.8243463039398193, + 3.796638011932373, + 3.6664352416992188, + 3.8023648262023926, + 3.728816509246826, + 3.8045456409454346, + 3.821454167366028, + 3.7300424575805664, + 3.992907762527466, + 3.8500782251358032, + 3.8186832666397095, + 3.8547844886779785, + 3.7063286304473877, + 3.7459917068481445, + 3.848002076148987, + 3.6871204376220703, + 3.8670661449432373, + 3.710232138633728, + 3.635068655014038, + 3.639712333679199, + 3.7166701555252075, + 3.7032583951950073, + 3.7586408853530884, + 3.8582465648651123, + 3.69434916973114, + 3.652403473854065, + 3.7072088718414307, + 3.778829336166382, + 3.715742826461792, + 3.719598174095154, + 3.74590802192688, + 3.6628862619400024, + 3.6005072593688965, + 3.6474374532699585, + 3.8126535415649414, + 3.781015992164612, + 3.7943789958953857, + 3.7781842947006226, + 3.8568663597106934, + 3.8469088077545166, + 3.8093631267547607, + 3.869779944419861, + 3.750242590904236, + 3.715791702270508, + 3.8222445249557495, + 3.646081566810608, + 3.8463138341903687, + 3.684933304786682, + 3.7321689128875732, + 3.6941301822662354, + 3.855734705924988, + 3.764644980430603, + 3.7185498476028442, + 3.669888496398926, + 3.662033796310425, + 3.810997486114502, + 3.662911057472229, + 3.774791955947876, + 3.861498236656189, + 3.659204840660095, + 3.864023447036743, + 3.734022855758667, + 3.7086496353149414, + 3.6773468255996704, + 3.8345932960510254, + 3.7765116691589355, + 3.9254053831100464, + 3.738884925842285, + 3.8377909660339355, + 3.7676236629486084, + 3.491852045059204, + 3.689021110534668, + 3.8250832557678223, + 3.738207697868347, + 3.7669583559036255, + 3.624351978302002, + 3.7498100996017456, + 3.724193572998047, + 3.7976213693618774, + 3.769168257713318, + 3.6327754259109497, + 3.828339099884033, + 3.746858835220337, + 3.825870156288147, + 3.6824042797088623, + 3.6210131645202637, + 3.7692453861236572, + 3.6825824975967407, + 3.7166061401367188, + 3.7072936296463013, + 3.789191484451294, + 3.6660962104797363, + 3.687317371368408, + 3.6813541650772095, + 3.7359405755996704, + 3.6440058946609497, + 3.7221816778182983, + 3.7021656036376953, + 3.845473527908325, + 3.704718828201294, + 3.784412980079651, + 3.8206104040145874, + 3.5658211708068848, + 3.8406065702438354, + 3.6537933349609375, + 3.807155728340149, + 3.70072865486145, + 3.6797382831573486, + 3.776837468147278, + 3.641917109489441, + 3.7114193439483643, + 3.730123519897461, + 3.737324357032776, + 3.6520562171936035, + 3.6528149843215942, + 3.667359232902527, + 3.7783849239349365, + 3.674153447151184, + 3.7914390563964844, + 3.6675761938095093, + 3.6898680925369263, + 3.789597749710083, + 3.6547170877456665, + 3.7327204942703247, + 3.6955777406692505, + 3.778642177581787, + 3.7741893529891968, + 3.7004318237304688, + 3.713475823402405, + 3.5284368991851807, + 3.6733932495117188, + 3.642926573753357, + 3.565206527709961, + 3.484064817428589, + 3.619466185569763, + 3.6499651670455933, + 3.6218771934509277, + 3.7699681520462036, + 3.5502777099609375, + 3.5981359481811523, + 3.5500086545944214, + 3.4673675298690796, + 3.6711552143096924, + 3.569137454032898, + 3.632708191871643, + 3.587306261062622, + 3.467307448387146, + 3.620454430580139, + 3.5104641914367676, + 3.5556070804595947, + 3.65308678150177, + 3.5580209493637085, + 3.681069493293762, + 3.651206851005554, + 3.7154204845428467, + 3.5005228519439697, + 3.647536039352417, + 3.53875732421875, + 3.686051607131958, + 3.6764410734176636, + 3.5070008039474487, + 3.660656213760376, + 3.5898948907852173, + 3.5806931257247925, + 3.6081117391586304, + 3.68921160697937, + 3.5076558589935303, + 3.6075143814086914, + 3.6485332250595093, + 3.589386224746704, + 3.681233763694763, + 3.8179235458374023, + 3.6157546043395996, + 3.6606355905532837, + 3.6274949312210083, + 3.644957423210144, + 3.661368250846863, + 3.5728050470352173, + 3.5714293718338013, + 3.583855628967285, + 3.6844847202301025, + 3.602144241333008, + 3.6074377298355103, + 3.5676668882369995, + 3.6022496223449707, + 3.7212201356887817, + 3.5749101638793945, + 3.6746374368667603, + 3.562968611717224, + 3.6707022190093994, + 3.582651734352112, + 3.57731556892395, + 3.731279730796814, + 3.6309107542037964, + 3.624101161956787, + 3.6301517486572266, + 3.5093199014663696, + 3.7451342344284058, + 3.612139105796814, + 3.5928268432617188, + 3.4617602825164795, + 3.636423110961914, + 3.6027411222457886, + 3.558045268058777, + 3.592359781265259, + 3.6563720703125, + 3.6830888986587524, + 3.497232437133789, + 3.6513091325759888, + 3.707461953163147, + 3.6258926391601562, + 3.6761075258255005, + 3.6272107362747192, + 3.5655791759490967, + 3.372810125350952, + 3.5523442029953003, + 3.490195870399475, + 3.5918344259262085, + 3.645796060562134, + 3.628056049346924, + 3.672108769416809, + 3.587510108947754, + 3.650253176689148, + 3.6289771795272827, + 3.677140474319458, + 3.66145920753479, + 3.5668472051620483, + 3.658389925956726, + 3.5894408226013184, + 3.4992003440856934, + 3.642266273498535, + 3.4782410860061646, + 3.5475451946258545, + 3.517322301864624, + 3.6105841398239136, + 3.610467314720154, + 3.648183226585388, + 3.489064931869507, + 3.626651644706726, + 3.494924545288086, + 3.583156704902649, + 3.5484368801116943, + 3.663372278213501, + 3.6091712713241577, + 3.598716616630554, + 3.4653384685516357, + 3.596005916595459, + 3.54508638381958, + 3.6660783290863037, + 3.4991016387939453, + 3.564267635345459, + 3.6976795196533203, + 3.601197123527527, + 3.671025514602661, + 3.43451464176178, + 3.533596158027649, + 3.3756083250045776, + 3.435622453689575, + 3.580625057220459, + 3.592581272125244, + 3.575192093849182, + 3.630511164665222, + 3.4337658882141113, + 3.620019793510437, + 3.4653621912002563, + 3.542278289794922, + 3.5707839727401733, + 3.5492316484451294, + 3.4450098276138306, + 3.5499753952026367, + 3.4476640224456787, + 3.4120194911956787, + 3.5440053939819336, + 3.501845359802246, + 3.449272632598877, + 3.5293586254119873, + 3.54767906665802, + 3.539071202278137, + 3.5167794227600098, + 3.4775620698928833, + 3.5365623235702515, + 3.4835933446884155, + 3.5482473373413086, + 3.454312562942505, + 3.5407378673553467, + 3.5163588523864746, + 3.4826385974884033, + 3.4753615856170654, + 3.434453248977661, + 3.441286087036133, + 3.4450771808624268, + 3.573984980583191, + 3.603846311569214, + 3.5699373483657837, + 3.429956078529358, + 3.44334876537323, + 3.4783172607421875, + 3.5632630586624146, + 3.5206621885299683, + 3.6198389530181885, + 3.465908408164978, + 3.5386128425598145, + 3.516706347465515, + 3.457421660423279, + 3.490179181098938, + 3.452946901321411, + 3.413069009780884, + 3.495206594467163, + 3.489719867706299, + 3.561746597290039, + 3.5386468172073364, + 3.415006995201111, + 3.5586553812026978, + 3.612594485282898, + 3.4628742933273315, + 3.5270012617111206, + 3.4289348125457764, + 3.4715497493743896, + 3.5230852365493774, + 3.5550252199172974, + 3.5519418716430664, + 3.5334551334381104, + 3.533593535423279, + 3.496798872947693, + 3.5817569494247437, + 3.506361722946167, + 3.4062191247940063, + 3.6102399826049805, + 3.533489942550659, + 3.505582332611084, + 3.494002103805542, + 3.491011142730713, + 3.527984857559204, + 3.5154013633728027, + 3.598130941390991, + 3.4298412799835205, + 3.4930745363235474, + 3.453132748603821, + 3.4180890321731567, + 3.502563238143921, + 3.586733818054199, + 3.356955409049988, + 3.4898600578308105, + 3.529292345046997, + 3.603745698928833, + 3.543882727622986, + 3.417468786239624, + 3.401060104370117, + 3.56908643245697, + 3.3595502376556396, + 3.473295569419861, + 3.559152126312256, + 3.4976255893707275, + 3.5210927724838257, + 3.4220995903015137, + 3.5539849996566772, + 3.559217095375061, + 3.4767098426818848, + 3.5045496225357056, + 3.4989776611328125, + 3.5501022338867188, + 3.4312551021575928, + 3.487759232521057, + 3.4883720874786377, + 3.464101791381836, + 3.415608286857605, + 3.5301120281219482, + 3.661103129386902, + 3.471666097640991, + 3.631515860557556, + 3.545357823371887, + 3.39468777179718, + 3.3822596073150635, + 3.467798590660095, + 3.339062452316284, + 3.4896368980407715, + 3.355475902557373, + 3.4539685249328613, + 3.5425976514816284, + 3.4845153093338013, + 3.5643739700317383, + 3.462122678756714, + 3.2836575508117676, + 3.4941565990448, + 3.254448413848877, + 3.318727493286133, + 3.454464554786682, + 3.3275322914123535, + 3.470037341117859, + 3.478376626968384, + 3.5281213521957397, + 3.4556984901428223, + 3.4081908464431763, + 3.470047950744629, + 3.2482213973999023, + 3.492891550064087, + 3.3225300312042236, + 3.428272008895874, + 3.384861469268799, + 3.410061240196228, + 3.3817577362060547, + 3.4225480556488037, + 3.5270323753356934, + 3.411038398742676, + 3.5098754167556763, + 3.3924254179000854, + 3.3883801698684692, + 3.4512895345687866, + 3.4527506828308105, + 3.4101169109344482, + 3.444472551345825, + 3.4910160303115845, + 3.330901265144348, + 3.4958198070526123, + 3.4949408769607544, + 3.3576226234436035, + 3.4126425981521606, + 3.4755043983459473, + 3.4475228786468506, + 3.525486469268799, + 3.360380172729492, + 3.40559983253479, + 3.4097018241882324, + 3.4056116342544556, + 3.39879834651947, + 3.3996230363845825, + 3.458195924758911, + 3.4796314239501953, + 3.5184457302093506, + 3.5186492204666138, + 3.3959591388702393, + 3.465575695037842, + 3.3747721910476685, + 3.4962189197540283, + 3.455775260925293, + 3.511079788208008, + 3.4251517057418823, + 3.348210334777832, + 3.3536709547042847, + 3.4677940607070923, + 3.4482275247573853, + 3.560396194458008, + 3.3294469118118286, + 3.384635090827942, + 3.4053999185562134, + 3.4238102436065674, + 3.3682836294174194, + 3.501447081565857, + 3.4315844774246216, + 3.5477182865142822, + 3.3229269981384277, + 3.4240896701812744, + 3.4790401458740234, + 3.3909997940063477, + 3.3621866703033447, + 3.474637031555176, + 3.4134212732315063, + 3.367099404335022, + 3.3551801443099976, + 3.4982444047927856, + 3.4940264225006104, + 3.39206063747406, + 3.435909152030945, + 3.4227453470230103, + 3.2917802333831787, + 3.4655165672302246, + 3.414072632789612, + 3.322830319404602, + 3.327800750732422, + 3.4279037714004517, + 3.3533376455307007, + 3.501397967338562, + 3.488200306892395, + 3.4294309616088867, + 3.4273104667663574, + 3.439136028289795, + 3.4682886600494385, + 3.455795407295227, + 3.5191354751586914, + 3.401058554649353, + 3.2326366901397705, + 3.5089510679244995, + 3.427541732788086, + 3.314754843711853, + 3.4145491123199463, + 3.410659909248352, + 3.4244816303253174, + 3.505912661552429, + 3.460736870765686, + 3.389113187789917, + 3.4863505363464355, + 3.3327367305755615, + 3.4791117906570435, + 3.466469168663025, + 3.430152177810669, + 3.527582287788391, + 3.451097011566162, + 3.354289174079895, + 3.431986689567566, + 3.4535473585128784, + 3.5068756341934204, + 3.4203048944473267, + 3.290991425514221, + 3.4283818006515503, + 3.323682427406311, + 3.3724628686904907, + 3.426533818244934, + 3.3725839853286743, + 3.2690166234970093, + 3.5922948122024536, + 3.3872005939483643, + 3.415865898132324, + 3.3468302488327026, + 3.248116612434387, + 3.4982945919036865, + 3.5015335083007812, + 3.387032985687256, + 3.3404370546340942, + 3.418530225753784, + 3.3137696981430054, + 3.3989474773406982, + 3.3922338485717773, + 3.5096213817596436, + 3.3285900354385376, + 3.4172314405441284, + 3.488658905029297, + 3.430356740951538, + 3.3671900033950806, + 3.438143253326416, + 3.503026843070984, + 3.4407464265823364, + 3.381321668624878, + 3.4174402952194214, + 3.359718084335327, + 3.3809894323349, + 3.3494848012924194, + 3.4138097763061523, + 3.396571636199951, + 3.290373921394348, + 3.305882453918457, + 3.4101110696792603, + 3.4435960054397583, + 3.4061636924743652, + 3.420854926109314, + 3.4051685333251953, + 3.400737762451172, + 3.339311718940735, + 3.420093297958374, + 3.3561094999313354, + 3.3597265481948853, + 3.356716752052307, + 3.3722234964370728, + 3.437764048576355, + 3.4656814336776733, + 3.4660940170288086, + 3.3378148078918457, + 3.5053317546844482, + 3.286959171295166, + 3.4989280700683594, + 3.466231107711792, + 3.3519957065582275, + 3.3700127601623535, + 3.4792405366897583, + 3.5075466632843018, + 3.3616777658462524, + 3.3215006589889526, + 3.4254415035247803, + 3.389620304107666, + 3.2170615196228027, + 3.416069984436035, + 3.324831962585449, + 3.4411234855651855, + 3.1925947666168213, + 3.484488368034363, + 3.3833996057510376, + 3.3753461837768555, + 3.4004606008529663, + 3.4848225116729736, + 3.4485480785369873, + 3.3323031663894653, + 3.4905072450637817, + 3.4651890993118286, + 3.3470754623413086, + 3.403535842895508, + 3.3733930587768555, + 3.345599412918091, + 3.418495535850525, + 3.3234223127365112, + 3.3752622604370117, + 3.4119309186935425, + 3.4666757583618164, + 3.502256393432617, + 3.3483726978302, + 3.438513159751892, + 3.393196702003479, + 3.1777377128601074, + 3.2888870239257812, + 3.172056555747986, + 3.4294865131378174, + 3.4049121141433716, + 3.378464460372925, + 3.44167697429657, + 3.3330050706863403, + 3.39383065700531, + 3.457123875617981, + 3.3594175577163696, + 3.5274198055267334, + 3.3328638076782227, + 3.364720582962036, + 3.327828049659729, + 3.4126418828964233, + 3.5674185752868652, + 3.5087188482284546, + 3.2987334728240967, + 3.3836910724639893, + 3.395641326904297, + 3.3263083696365356, + 3.351095676422119, + 3.4128774404525757, + 3.4201323986053467, + 3.4896613359451294, + 3.284456253051758 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 126.51674859264594, + 79.16189447075308, + 62.56998244256721, + 54.924797080914, + 49.46562760783205, + 46.19064825463921, + 43.577729766857956, + 41.14660881796902, + 39.58189120407699, + 38.11500333002334, + 36.93955003848509, + 35.64398648135135, + 34.76666158875481, + 34.07614306785335, + 33.12746938255666, + 32.80011073814345, + 32.388274158382075, + 32.23568688243296, + 32.145041526441986, + 32.11144064734876 + ], + "best_ppl": 32.11144064734876, + "train_time_s": 19461.512393712997 +} \ No newline at end of file diff --git a/063_sparse_dispatch/softmax_1plus2of5_isoparam_s256_best.pt b/063_sparse_dispatch/softmax_1plus2of5_isoparam_s256_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..2501e31816b43d23df05659cd3e8be1ee6f7a1ad --- /dev/null +++ b/063_sparse_dispatch/softmax_1plus2of5_isoparam_s256_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d71556b1336efe919b62db7b1cec8a7834d2fbbaf4b9a682634166b00c6fdca5 +size 350345254 diff --git a/063_sparse_dispatch/softmax_1plus2of5_isoparam_s256_results.json b/063_sparse_dispatch/softmax_1plus2of5_isoparam_s256_results.json new file mode 100644 index 0000000000000000000000000000000000000000..d3430e80002ed33599695fd567f6e95976f6a03e --- /dev/null +++ b/063_sparse_dispatch/softmax_1plus2of5_isoparam_s256_results.json @@ -0,0 +1,2056 @@ +{ + "label": "softmax_1plus2of5_isoparam_s256", + "seed": 256, + "d_ff_thin": 1133, + "total_params": 87525376, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 18.996585845947266, + 12.790808200836182, + 7.852917194366455, + 7.265927076339722, + 7.066587209701538, + 6.932797193527222, + 6.869969367980957, + 6.761896371841431, + 6.691022872924805, + 6.584529161453247, + 6.48747181892395, + 6.3582375049591064, + 6.264127969741821, + 6.207859754562378, + 6.100029706954956, + 5.979037761688232, + 6.1099114418029785, + 5.9189722537994385, + 5.8936848640441895, + 5.792679786682129, + 5.7506492137908936, + 5.595245599746704, + 5.754106521606445, + 5.555257081985474, + 5.556506156921387, + 5.563337802886963, + 5.424635648727417, + 5.464148283004761, + 5.322368860244751, + 5.388973236083984, + 5.455282211303711, + 5.108644008636475, + 5.271996021270752, + 5.29251766204834, + 5.264651775360107, + 5.242269992828369, + 5.187849044799805, + 5.197559118270874, + 5.161268472671509, + 5.05175256729126, + 5.188881874084473, + 5.0933661460876465, + 5.100140810012817, + 4.9763476848602295, + 4.952924728393555, + 4.961355924606323, + 4.951645374298096, + 4.986440181732178, + 4.9087536334991455, + 4.932084083557129, + 4.810742139816284, + 4.898251533508301, + 4.96938157081604, + 4.8165812492370605, + 4.754106044769287, + 4.857502698898315, + 4.8770225048065186, + 4.919317960739136, + 4.845205545425415, + 4.846625089645386, + 4.942672252655029, + 4.818507671356201, + 4.770862817764282, + 4.727515935897827, + 4.779951095581055, + 4.780208349227905, + 4.675248622894287, + 4.740875720977783, + 4.709129095077515, + 4.662522554397583, + 4.703934192657471, + 4.740832567214966, + 4.517131328582764, + 4.737159013748169, + 4.700090169906616, + 4.6712634563446045, + 4.7848029136657715, + 4.708108901977539, + 4.6712706089019775, + 4.427627325057983, + 4.568897247314453, + 4.638171434402466, + 4.579901218414307, + 4.477832794189453, + 4.532165050506592, + 4.497214317321777, + 4.591289758682251, + 4.648005723953247, + 4.4354424476623535, + 4.573447942733765, + 4.558868408203125, + 4.525363445281982, + 4.551769971847534, + 4.582542657852173, + 4.5754759311676025, + 4.367791414260864, + 4.5141823291778564, + 4.653369188308716, + 4.535067558288574, + 4.499703407287598, + 4.568346738815308, + 4.527987718582153, + 4.53109073638916, + 4.583508253097534, + 4.499494552612305, + 4.419757843017578, + 4.4780662059783936, + 4.466100215911865, + 4.565115928649902, + 4.3761515617370605, + 4.4808855056762695, + 4.452519655227661, + 4.377331256866455, + 4.410012245178223, + 4.371577262878418, + 4.393189430236816, + 4.393511533737183, + 4.443571090698242, + 4.42426323890686, + 4.3468358516693115, + 4.453749895095825, + 4.30833101272583, + 4.25227952003479, + 4.391167879104614, + 4.2920753955841064, + 4.301004886627197, + 4.328061103820801, + 4.213786840438843, + 4.330401659011841, + 4.369869947433472, + 4.251475095748901, + 4.319890737533569, + 4.365111589431763, + 4.338959455490112, + 4.341843128204346, + 4.358703374862671, + 4.2602219581604, + 4.334563493728638, + 4.355693340301514, + 4.317841053009033, + 4.196675539016724, + 4.235781908035278, + 4.197746753692627, + 4.296336650848389, + 4.280948162078857, + 4.226383924484253, + 4.2779858112335205, + 4.243221759796143, + 4.223191499710083, + 4.132203102111816, + 4.377793550491333, + 4.255021095275879, + 4.127948999404907, + 4.24165415763855, + 4.394593238830566, + 4.219817399978638, + 4.196745157241821, + 4.277215003967285, + 4.269118070602417, + 4.212177276611328, + 4.182599306106567, + 4.283613681793213, + 4.159308433532715, + 4.077913522720337, + 4.181678771972656, + 4.214595556259155, + 4.162173271179199, + 4.158539295196533, + 4.244253635406494, + 4.1204211711883545, + 4.302537441253662, + 4.196422100067139, + 4.255773305892944, + 4.1641459465026855, + 4.338314056396484, + 4.078838348388672, + 4.086266756057739, + 4.268347501754761, + 4.155869722366333, + 4.222769498825073, + 4.1635496616363525, + 4.235562086105347, + 4.092837572097778, + 4.136035919189453, + 4.2083141803741455, + 4.147154331207275, + 4.0759594440460205, + 4.205975770950317, + 4.139629125595093, + 4.231108903884888, + 4.105016231536865, + 4.249250173568726, + 4.1555421352386475, + 4.221874475479126, + 4.135931491851807, + 4.2041473388671875, + 4.104116678237915, + 4.129604816436768, + 4.181751251220703, + 4.041579365730286, + 4.074423789978027, + 4.046825528144836, + 4.133923292160034, + 4.099331855773926, + 4.1203508377075195, + 4.144665718078613, + 4.158179521560669, + 4.292837381362915, + 3.941128730773926, + 4.080390453338623, + 4.03357207775116, + 4.035247564315796, + 4.209361553192139, + 4.107364654541016, + 4.163816928863525, + 4.052049517631531, + 4.019321322441101, + 4.058345556259155, + 3.9206637144088745, + 4.115259647369385, + 4.107456684112549, + 4.152280569076538, + 4.1732940673828125, + 4.040177583694458, + 3.9481829404830933, + 4.005730986595154, + 4.03543484210968, + 4.069719314575195, + 4.040468454360962, + 4.050686001777649, + 4.015464544296265, + 3.9683120250701904, + 4.179596424102783, + 4.050566911697388, + 4.145672798156738, + 3.907194495201111, + 4.032275319099426, + 4.143158197402954, + 4.028458833694458, + 4.050150632858276, + 3.8751882314682007, + 4.106178522109985, + 4.056585669517517, + 4.0403735637664795, + 3.9987924098968506, + 4.162807464599609, + 3.967141032218933, + 4.031254768371582, + 4.051376819610596, + 4.033841967582703, + 3.998705506324768, + 4.1356565952301025, + 4.039388060569763, + 3.977799415588379, + 3.8911473751068115, + 3.885058641433716, + 3.937186360359192, + 3.9943262338638306, + 3.927172899246216, + 3.9356600046157837, + 3.9339929819107056, + 3.917086601257324, + 3.9063860177993774, + 3.947843909263611, + 4.018292665481567, + 4.02665376663208, + 4.016907572746277, + 3.9012686014175415, + 3.9670846462249756, + 3.9418411254882812, + 4.011869788169861, + 3.9911786317825317, + 3.7855831384658813, + 3.994307518005371, + 3.8988550901412964, + 3.8904508352279663, + 3.9419702291488647, + 3.92188823223114, + 3.9400503635406494, + 4.024947762489319, + 3.9173598289489746, + 3.902989625930786, + 4.005592107772827, + 4.005211591720581, + 3.861047148704529, + 3.9009008407592773, + 3.877737283706665, + 3.9537192583084106, + 3.907166361808777, + 3.9905030727386475, + 4.017235994338989, + 3.8979058265686035, + 3.9760860204696655, + 3.8939614295959473, + 3.8631309270858765, + 3.8479329347610474, + 4.016940593719482, + 3.9338685274124146, + 3.9129985570907593, + 4.059300184249878, + 4.056445360183716, + 3.8671538829803467, + 3.802793860435486, + 3.853200078010559, + 3.828214406967163, + 3.9096498489379883, + 3.943311929702759, + 3.8421337604522705, + 3.9108296632766724, + 3.853805422782898, + 3.8275790214538574, + 3.976651430130005, + 3.9731818437576294, + 3.866340160369873, + 3.9924381971359253, + 3.8499300479888916, + 4.001147031784058, + 3.8561031818389893, + 3.859761953353882, + 3.9578511714935303, + 3.9612960815429688, + 3.903782844543457, + 3.886783003807068, + 3.9958595037460327, + 4.005700349807739, + 3.8909205198287964, + 3.959542989730835, + 3.9072128534317017, + 3.8582839965820312, + 3.945460796356201, + 3.73609459400177, + 3.923567056655884, + 3.78183913230896, + 3.848800778388977, + 3.8912346363067627, + 3.9567850828170776, + 3.8749836683273315, + 3.9941463470458984, + 3.834706664085388, + 3.8129767179489136, + 3.9559260606765747, + 3.928673028945923, + 3.9209922552108765, + 3.831739664077759, + 3.866078734397888, + 3.674846053123474, + 3.790944457054138, + 3.912809729576111, + 3.852501630783081, + 3.806154251098633, + 4.058220148086548, + 3.8055895566940308, + 3.823029041290283, + 3.8502339124679565, + 3.977512836456299, + 3.7798080444335938, + 3.8854724168777466, + 3.785431385040283, + 3.8478353023529053, + 3.8160765171051025, + 4.001203775405884, + 3.892290711402893, + 3.8480875492095947, + 3.926247239112854, + 3.85882568359375, + 3.878760814666748, + 3.7426925897598267, + 3.803401827812195, + 3.943985342979431, + 4.018345236778259, + 3.9366625547409058, + 3.850035309791565, + 3.861790895462036, + 3.8720797300338745, + 3.90906298160553, + 3.8954315185546875, + 3.7923550605773926, + 3.8840566873550415, + 3.9350110292434692, + 3.7890347242355347, + 3.724107027053833, + 3.72489070892334, + 3.809815526008606, + 3.864729166030884, + 3.8067322969436646, + 3.739191174507141, + 3.7458359003067017, + 3.840434193611145, + 3.6798012256622314, + 3.8084796667099, + 3.82317578792572, + 3.7784663438796997, + 3.676089286804199, + 3.7658944129943848, + 3.722536325454712, + 3.7006677389144897, + 3.814270496368408, + 3.7622036933898926, + 3.754884719848633, + 3.7892366647720337, + 3.858474373817444, + 3.789023995399475, + 3.796609878540039, + 3.7215245962142944, + 3.7054359912872314, + 3.8158146142959595, + 3.741047501564026, + 3.7710577249526978, + 3.8342114686965942, + 3.8377314805984497, + 3.753567695617676, + 3.6329123973846436, + 3.8212345838546753, + 3.695158839225769, + 3.7080310583114624, + 3.7093703746795654, + 3.820905327796936, + 3.7302955389022827, + 3.7540284395217896, + 3.802507758140564, + 3.7682671546936035, + 3.765089511871338, + 3.6370028257369995, + 3.7328654527664185, + 3.820825934410095, + 3.7996954917907715, + 3.843430519104004, + 3.642945885658264, + 3.818955183029175, + 3.7603691816329956, + 3.78557550907135, + 3.718542218208313, + 3.7958730459213257, + 3.834464192390442, + 3.8407957553863525, + 3.7651329040527344, + 3.7332924604415894, + 3.7218509912490845, + 3.745659351348877, + 3.8231769800186157, + 3.7391515970230103, + 3.77878475189209, + 3.8581920862197876, + 3.7412350177764893, + 3.759040951728821, + 3.7346115112304688, + 3.755424380302429, + 3.7076183557510376, + 3.7620131969451904, + 3.7792978286743164, + 3.793572425842285, + 3.791137933731079, + 3.7903972864151, + 3.636022210121155, + 3.731171131134033, + 3.6430423259735107, + 3.663398861885071, + 3.679134964942932, + 3.6412711143493652, + 3.803465723991394, + 3.7684398889541626, + 3.6874680519104004, + 3.752733826637268, + 3.6694992780685425, + 3.701590061187744, + 3.6983840465545654, + 3.757888913154602, + 3.784773349761963, + 3.820653557777405, + 3.823335886001587, + 3.6154513359069824, + 3.7844001054763794, + 3.7630549669265747, + 3.688613533973694, + 3.645370841026306, + 3.797512412071228, + 3.822898030281067, + 3.697781562805176, + 3.761814832687378, + 3.6946117877960205, + 3.742809534072876, + 3.755538821220398, + 3.7946661710739136, + 3.7294068336486816, + 3.71914279460907, + 3.685026168823242, + 3.840822458267212, + 3.733314037322998, + 3.765875220298767, + 3.776510000228882, + 3.5516624450683594, + 3.773267149925232, + 3.7562516927719116, + 3.7451913356781006, + 3.600743055343628, + 3.7476953268051147, + 3.72951340675354, + 3.740791082382202, + 3.625733733177185, + 3.6862624883651733, + 3.693201780319214, + 3.6641706228256226, + 3.6863632202148438, + 3.8217711448669434, + 3.7838122844696045, + 3.755353331565857, + 3.747612237930298, + 3.5573002099990845, + 3.6884257793426514, + 3.5722228288650513, + 3.461688280105591, + 3.6945347785949707, + 3.7033634185791016, + 3.7478036880493164, + 3.6435834169387817, + 3.5673046112060547, + 3.6732897758483887, + 3.659074306488037, + 3.7024658918380737, + 3.5924347639083862, + 3.5269659757614136, + 3.6846266984939575, + 3.6061512231826782, + 3.704885959625244, + 3.6360186338424683, + 3.6649351119995117, + 3.6647789478302, + 3.625019907951355, + 3.5552700757980347, + 3.7122126817703247, + 3.669029474258423, + 3.7232940196990967, + 3.5744224786758423, + 3.541049003601074, + 3.6619971990585327, + 3.5830209255218506, + 3.7236489057540894, + 3.6010560989379883, + 3.5536125898361206, + 3.632046103477478, + 3.613369941711426, + 3.6648035049438477, + 3.57495653629303, + 3.7460702657699585, + 3.632974624633789, + 3.7339165210723877, + 3.6196558475494385, + 3.720224380493164, + 3.709269881248474, + 3.664262890815735, + 3.630728244781494, + 3.5722482204437256, + 3.5568490028381348, + 3.7186689376831055, + 3.731020212173462, + 3.7195581197738647, + 3.7191414833068848, + 3.680234670639038, + 3.6650086641311646, + 3.7141298055648804, + 3.7100120782852173, + 3.6669548749923706, + 3.656369924545288, + 3.593923807144165, + 3.6650885343551636, + 3.6381136178970337, + 3.515026569366455, + 3.6161675453186035, + 3.677573323249817, + 3.7707438468933105, + 3.6876678466796875, + 3.5957568883895874, + 3.5912927389144897, + 3.785304069519043, + 3.5989819765090942, + 3.7408193349838257, + 3.617565155029297, + 3.718282461166382, + 3.5214561223983765, + 3.6204358339309692, + 3.6380386352539062, + 3.738951563835144, + 3.6207852363586426, + 3.6871918439865112, + 3.8257243633270264, + 3.732261061668396, + 3.5599093437194824, + 3.6451207399368286, + 3.599893569946289, + 3.644791603088379, + 3.6213860511779785, + 3.658517599105835, + 3.5440033674240112, + 3.4161972999572754, + 3.6204100847244263, + 3.5995789766311646, + 3.639158844947815, + 3.7467265129089355, + 3.5769671201705933, + 3.718552827835083, + 3.60866117477417, + 3.6882728338241577, + 3.604343056678772, + 3.625191569328308, + 3.656622290611267, + 3.693013310432434, + 3.6038575172424316, + 3.6833382844924927, + 3.641294240951538, + 3.57114040851593, + 3.5484288930892944, + 3.6427948474884033, + 3.750434994697571, + 3.7252936363220215, + 3.562527060508728, + 3.687047839164734, + 3.438297152519226, + 3.6245853900909424, + 3.3786802291870117, + 3.6247516870498657, + 3.675538659095764, + 3.7146763801574707, + 3.5988398790359497, + 3.5799314975738525, + 3.458760619163513, + 3.471737265586853, + 3.630144715309143, + 3.753320336341858, + 3.6173906326293945, + 3.602554678916931, + 3.595063328742981, + 3.768523335456848, + 3.4873828887939453, + 3.580256223678589, + 3.458156943321228, + 3.5249295234680176, + 3.437831997871399, + 3.621194362640381, + 3.4567970037460327, + 3.5694397687911987, + 3.501057982444763, + 3.506202220916748, + 3.5625048875808716, + 3.5030397176742554, + 3.642902374267578, + 3.5641322135925293, + 3.6291452646255493, + 3.4332393407821655, + 3.4662362337112427, + 3.492757201194763, + 3.5427842140197754, + 3.5298267602920532, + 3.4774458408355713, + 3.641744017601013, + 3.4890103340148926, + 3.6049076318740845, + 3.4252511262893677, + 3.5800929069519043, + 3.5287768840789795, + 3.524112820625305, + 3.7176607847213745, + 3.628117561340332, + 3.566564202308655, + 3.4878129959106445, + 3.465453028678894, + 3.5100830793380737, + 3.56454598903656, + 3.601239323616028, + 3.4556045532226562, + 3.491948962211609, + 3.595636487007141, + 3.5296318531036377, + 3.4749948978424072, + 3.552797555923462, + 3.586135983467102, + 3.5277167558670044, + 3.585023522377014, + 3.450371503829956, + 3.48833429813385, + 3.5526375770568848, + 3.54483699798584, + 3.513988494873047, + 3.4074361324310303, + 3.516233444213867, + 3.518089532852173, + 3.432288885116577, + 3.622015953063965, + 3.5167261362075806, + 3.5213316679000854, + 3.620633602142334, + 3.6236695051193237, + 3.579097628593445, + 3.559017300605774, + 3.6249271631240845, + 3.595313549041748, + 3.5446733236312866, + 3.407493233680725, + 3.6115639209747314, + 3.565369963645935, + 3.3695677518844604, + 3.5716168880462646, + 3.51643443107605, + 3.52323579788208, + 3.5413477420806885, + 3.633926510810852, + 3.578120470046997, + 3.616967558860779, + 3.440501570701599, + 3.5787676572799683, + 3.504366636276245, + 3.498806357383728, + 3.5589300394058228, + 3.544061779975891, + 3.363706588745117, + 3.502626657485962, + 3.544754147529602, + 3.5202484130859375, + 3.467752456665039, + 3.4261921644210815, + 3.4371843338012695, + 3.5082896947860718, + 3.4893226623535156, + 3.650885820388794, + 3.5038846731185913, + 3.422385096549988, + 3.579330801963806, + 3.4509440660476685, + 3.4562538862228394, + 3.536002993583679, + 3.58513081073761, + 3.531031847000122, + 3.581947922706604, + 3.5881468057632446, + 3.4774603843688965, + 3.621742844581604, + 3.657712697982788, + 3.4937926530838013, + 3.5198386907577515, + 3.5753233432769775, + 3.5242605209350586, + 3.5657947063446045, + 3.4395086765289307, + 3.560415744781494, + 3.4920586347579956, + 3.4854495525360107, + 3.4524059295654297, + 3.612464666366577, + 3.477235794067383, + 3.5544837713241577, + 3.5530518293380737, + 3.3612587451934814, + 3.410357117652893, + 3.595518469810486, + 3.3797013759613037, + 3.5323617458343506, + 3.368678092956543, + 3.4855796098709106, + 3.445473074913025, + 3.6840492486953735, + 3.373956799507141, + 3.485231637954712, + 3.2827006578445435, + 3.4091880321502686, + 3.3482354879379272, + 3.45745849609375, + 3.4288593530654907, + 3.4654988050460815, + 3.300683379173279, + 3.4566715955734253, + 3.4655007123947144, + 3.430750608444214, + 3.440971255302429, + 3.424616813659668, + 3.5086216926574707, + 3.4379533529281616, + 3.3439927101135254, + 3.5911643505096436, + 3.443607449531555, + 3.5531468391418457, + 3.4902149438858032, + 3.4803372621536255, + 3.506407380104065, + 3.584885597229004, + 3.523403763771057, + 3.6153290271759033, + 3.3185330629348755, + 3.5286022424697876, + 3.4943796396255493, + 3.4941012859344482, + 3.3680953979492188, + 3.597672462463379, + 3.4698197841644287, + 3.4447274208068848, + 3.4125903844833374, + 3.3253557682037354, + 3.6077276468276978, + 3.5863641500473022, + 3.57261860370636, + 3.347332239151001, + 3.4338293075561523, + 3.471882462501526, + 3.4145950078964233, + 3.5268328189849854, + 3.48494815826416, + 3.491869330406189, + 3.456360697746277, + 3.3429698944091797, + 3.366899609565735, + 3.4765207767486572, + 3.420232653617859, + 3.4119207859039307, + 3.4059442281723022, + 3.460302472114563, + 3.5331859588623047, + 3.4864754676818848, + 3.425132393836975, + 3.5142719745635986, + 3.2884217500686646, + 3.5136409997940063, + 3.548388719558716, + 3.3753262758255005, + 3.5596038103103638, + 3.4679471254348755, + 3.528017520904541, + 3.4763731956481934, + 3.3759018182754517, + 3.4914630651474, + 3.540102958679199, + 3.4228341579437256, + 3.3265936374664307, + 3.511116623878479, + 3.3508380651474, + 3.514572262763977, + 3.4721386432647705, + 3.4041424989700317, + 3.4282639026641846, + 3.3840218782424927, + 3.4387439489364624, + 3.491403818130493, + 3.5475049018859863, + 3.4057892560958862, + 3.6322875022888184, + 3.475285768508911, + 3.368410587310791, + 3.470045804977417, + 3.4228864908218384, + 3.540490746498108, + 3.4707003831863403, + 3.4345508813858032, + 3.439663887023926, + 3.3982049226760864, + 3.4633060693740845, + 3.455387592315674, + 3.4874820709228516, + 3.3984984159469604, + 3.437292218208313, + 3.5321905612945557, + 3.297015070915222, + 3.464772343635559, + 3.383669137954712, + 3.394339442253113, + 3.4264352321624756, + 3.3763500452041626, + 3.410096287727356, + 3.6322869062423706, + 3.375940203666687, + 3.430155873298645, + 3.486590027809143, + 3.3775163888931274, + 3.5975024700164795, + 3.4356695413589478, + 3.480181574821472, + 3.4400664567947388, + 3.5755159854888916, + 3.4390971660614014, + 3.3862650394439697, + 3.452602744102478, + 3.392094612121582, + 3.4775948524475098, + 3.431047558784485, + 3.4205832481384277, + 3.552154064178467, + 3.3471256494522095, + 3.421730637550354, + 3.41759717464447, + 3.392373561859131, + 3.4643748998641968, + 3.5058517456054688, + 3.4382660388946533, + 3.4318318367004395, + 3.2880245447158813, + 3.347637891769409, + 3.2632776498794556, + 3.476746678352356, + 3.4308578968048096, + 3.301843285560608, + 3.489686131477356, + 3.418591260910034, + 3.40832257270813, + 3.302896738052368, + 3.3853994607925415, + 3.4447869062423706, + 3.499746561050415, + 3.4048755168914795, + 3.3433724641799927, + 3.4330252408981323, + 3.28904128074646, + 3.519600987434387, + 3.5423965454101562, + 3.481878161430359, + 3.436702013015747, + 3.3454307317733765, + 3.346499800682068, + 3.327932119369507, + 3.440942883491516, + 3.503689408302307, + 3.3209469318389893, + 3.328973174095154, + 3.4757790565490723, + 3.371597647666931, + 3.4614598751068115, + 3.4307843446731567, + 3.405323028564453, + 3.4279403686523438, + 3.386488676071167, + 3.3897849321365356, + 3.372130036354065, + 3.4908783435821533, + 3.4973455667495728, + 3.4339277744293213, + 3.438485026359558, + 3.476231336593628, + 3.34391188621521, + 3.3719574213027954, + 3.462036967277527, + 3.472382426261902, + 3.26435649394989, + 3.3669371604919434, + 3.475517749786377, + 3.309110641479492, + 3.531097650527954, + 3.4099925756454468, + 3.3874841928482056, + 3.49387264251709, + 3.3636032342910767, + 3.3858214616775513, + 3.38047194480896, + 3.4131680727005005, + 3.261356830596924, + 3.3028818368911743, + 3.425816774368286, + 3.3242193460464478, + 3.372853398323059, + 3.2742457389831543, + 3.3585760593414307, + 3.4836742877960205, + 3.403416156768799, + 3.389237403869629, + 3.467726707458496, + 3.403491973876953, + 3.414479613304138, + 3.3737977743148804, + 3.4325108528137207, + 3.3818161487579346, + 3.365140914916992, + 3.4700961112976074, + 3.403863787651062, + 3.2800925970077515, + 3.3693588972091675, + 3.2687944173812866, + 3.374752163887024, + 3.4219268560409546, + 3.419626832008362, + 3.438323497772217, + 3.37154483795166, + 3.4353219270706177, + 3.474433660507202, + 3.3380481004714966, + 3.3989979028701782, + 3.410887360572815, + 3.3580890893936157, + 3.5196590423583984, + 3.5597625970840454, + 3.5467811822891235, + 3.5219770669937134, + 3.4104249477386475, + 3.4456835985183716, + 3.349156618118286, + 3.316148042678833, + 3.43691086769104, + 3.394672393798828, + 3.4695812463760376, + 3.4924659729003906, + 3.37119460105896, + 3.403945207595825, + 3.4071038961410522, + 3.466108560562134, + 3.4458082914352417, + 3.4207884073257446, + 3.426546573638916 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 126.17411083721215, + 79.49078817074312, + 63.868678915972, + 55.62856995692453, + 50.46677738171901, + 47.08767595273256, + 44.4070334585528, + 42.135456550043656, + 40.36133232703819, + 38.526016779528646, + 37.2890602275401, + 35.979607295935686, + 35.05798733656483, + 34.15193688020731, + 33.48560924471844, + 33.11315010874752, + 32.690939530480996, + 32.45161237546835, + 32.385938776774424, + 32.34499231086322 + ], + "best_ppl": 32.34499231086322, + "train_time_s": 19254.48855137825 +} \ No newline at end of file diff --git a/063_sparse_dispatch/sparse_3of15_best.pt b/063_sparse_dispatch/sparse_3of15_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..fec435f1c7bf87ac62739d612bd94be6faf782aa --- /dev/null +++ b/063_sparse_dispatch/sparse_3of15_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:384d64bfe1d5cfa1d3fa1c3a6caa27f227ab6a52071dbca5e142a7e55a628f42 +size 762143430 diff --git a/063_sparse_dispatch/sparse_3of15_results.json b/063_sparse_dispatch/sparse_3of15_results.json new file mode 100644 index 0000000000000000000000000000000000000000..7edb1b551aeb3175eff4884630081b3e1f87c53b --- /dev/null +++ b/063_sparse_dispatch/sparse_3of15_results.json @@ -0,0 +1,2057 @@ +{ + "label": "sparse_3of15", + "attention_mode": "sparse", + "top_k_blocks": 3, + "num_thin_blocks": 15, + "total_params": 190363648, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.003977298736572, + 15.391336441040039, + 7.984594106674194, + 7.184508323669434, + 7.0980260372161865, + 7.022733092308044, + 6.847659349441528, + 6.710046410560608, + 6.57315981388092, + 6.568584203720093, + 6.409439921379089, + 6.419973015785217, + 6.331320762634277, + 6.256755232810974, + 6.248715162277222, + 6.052494049072266, + 5.946696162223816, + 6.083134293556213, + 6.006249904632568, + 5.891277551651001, + 5.761326313018799, + 5.825458765029907, + 5.704015374183655, + 5.716663360595703, + 5.660764575004578, + 5.633748292922974, + 5.597256660461426, + 5.571169376373291, + 5.465669751167297, + 5.595702767372131, + 5.3548424243927, + 5.510397791862488, + 5.225245952606201, + 5.268363118171692, + 5.404284596443176, + 5.35852324962616, + 5.2683188915252686, + 5.24091649055481, + 5.252418160438538, + 5.281603813171387, + 5.069544076919556, + 5.202057480812073, + 5.328661799430847, + 5.105193734169006, + 5.183677911758423, + 5.177465438842773, + 5.098245739936829, + 5.185310125350952, + 5.164160370826721, + 5.000414609909058, + 5.1827545166015625, + 4.9315184354782104, + 5.122311353683472, + 5.05445659160614, + 4.961262941360474, + 5.014511346817017, + 4.982876300811768, + 4.9061408042907715, + 4.913482189178467, + 4.935028314590454, + 5.001381993293762, + 4.93001914024353, + 4.943150520324707, + 4.889400124549866, + 4.951772570610046, + 4.779729127883911, + 4.764078259468079, + 4.8890039920806885, + 4.690137982368469, + 4.659477353096008, + 4.859571933746338, + 4.823531746864319, + 4.851894378662109, + 4.80341899394989, + 4.775662064552307, + 4.843946576118469, + 4.710065484046936, + 4.717471957206726, + 4.7534602880477905, + 4.548163294792175, + 4.719534873962402, + 4.71203076839447, + 4.802356123924255, + 4.680018901824951, + 4.636545419692993, + 4.762752294540405, + 4.621143460273743, + 4.669782280921936, + 4.6299145221710205, + 4.567765116691589, + 4.618712306022644, + 4.691151857376099, + 4.549662232398987, + 4.744238972663879, + 4.644383430480957, + 4.653741955757141, + 4.563062787055969, + 4.623562812805176, + 4.485032916069031, + 4.639111399650574, + 4.606372356414795, + 4.613298535346985, + 4.652791976928711, + 4.596764922142029, + 4.586933970451355, + 4.571030974388123, + 4.494858384132385, + 4.47888708114624, + 4.586872100830078, + 4.529473423957825, + 4.580072522163391, + 4.598459720611572, + 4.5365554094314575, + 4.51269006729126, + 4.445174932479858, + 4.636111497879028, + 4.407553791999817, + 4.542642951011658, + 4.4074060916900635, + 4.428107142448425, + 4.475101470947266, + 4.515798330307007, + 4.53840959072113, + 4.485602259635925, + 4.53143048286438, + 4.543435573577881, + 4.489747762680054, + 4.320805191993713, + 4.3927918672561646, + 4.303545117378235, + 4.327128767967224, + 4.601690173149109, + 4.34581184387207, + 4.381045818328857, + 4.496167778968811, + 4.489072680473328, + 4.476341724395752, + 4.285025715827942, + 4.4082032442092896, + 4.321226119995117, + 4.386778354644775, + 4.352255702018738, + 4.423845291137695, + 4.281439304351807, + 4.462417125701904, + 4.401914000511169, + 4.429630994796753, + 4.147974491119385, + 4.309469938278198, + 4.266853451728821, + 4.361668944358826, + 4.315401196479797, + 4.320222854614258, + 4.291351795196533, + 4.328492283821106, + 4.223404765129089, + 4.222019672393799, + 4.375847816467285, + 4.261782646179199, + 4.337013602256775, + 4.396339178085327, + 4.366124749183655, + 4.259634852409363, + 4.3874441385269165, + 4.450392246246338, + 4.190065920352936, + 4.3353012800216675, + 4.351216912269592, + 4.232712745666504, + 4.388233423233032, + 4.339042663574219, + 4.31382691860199, + 4.20698094367981, + 4.336641073226929, + 4.271713614463806, + 4.296765565872192, + 4.255584955215454, + 4.238428950309753, + 4.084889769554138, + 4.375288486480713, + 4.249530076980591, + 4.344410300254822, + 4.283156752586365, + 4.3744810819625854, + 4.108480393886566, + 4.216241717338562, + 4.3334572315216064, + 4.301312208175659, + 4.271143913269043, + 4.18949681520462, + 4.000054359436035, + 4.324747920036316, + 4.207120656967163, + 4.283885717391968, + 4.253982186317444, + 4.340124249458313, + 4.182357728481293, + 4.314719796180725, + 4.203280329704285, + 4.159478425979614, + 4.30678927898407, + 4.219112396240234, + 4.211984157562256, + 4.154511332511902, + 4.262783169746399, + 4.178364872932434, + 4.156186461448669, + 4.16645884513855, + 4.188214302062988, + 4.1967490911483765, + 4.229685306549072, + 4.299454510211945, + 4.165329337120056, + 4.183295130729675, + 4.1450148820877075, + 4.161201596260071, + 4.279141426086426, + 4.2462416887283325, + 4.177141726016998, + 4.212609052658081, + 4.141306459903717, + 4.078835129737854, + 4.17713987827301, + 4.192997217178345, + 4.202584862709045, + 4.261310338973999, + 4.127290308475494, + 4.149214148521423, + 4.194107532501221, + 3.9984022974967957, + 4.288090109825134, + 4.169156610965729, + 4.070284605026245, + 4.211891174316406, + 4.145989656448364, + 4.231423258781433, + 4.166235566139221, + 4.133429646492004, + 4.129651606082916, + 4.103983402252197, + 4.255227088928223, + 4.304611802101135, + 4.085617125034332, + 4.118265092372894, + 4.250905513763428, + 4.084896087646484, + 4.09564071893692, + 4.161658525466919, + 3.925981879234314, + 4.1513588428497314, + 4.245167136192322, + 4.314635634422302, + 4.0748560428619385, + 4.030004322528839, + 3.966149687767029, + 3.93072646856308, + 4.087048649787903, + 4.18756103515625, + 4.03075498342514, + 3.986185669898987, + 3.995832145214081, + 4.059855818748474, + 4.0683711767196655, + 4.002596735954285, + 4.02375066280365, + 4.114109218120575, + 4.152167558670044, + 4.1519856452941895, + 4.076746582984924, + 4.11165726184845, + 3.986271023750305, + 4.0327372550964355, + 4.081360340118408, + 3.9855992197990417, + 4.073361814022064, + 4.02244758605957, + 4.037563443183899, + 4.100209593772888, + 4.047679781913757, + 4.091341435909271, + 4.155933380126953, + 4.023740172386169, + 3.9604402780532837, + 3.9588489532470703, + 3.934686064720154, + 4.038203656673431, + 3.9874146580696106, + 4.119954288005829, + 4.074169754981995, + 4.0885443687438965, + 3.9806755781173706, + 3.968889594078064, + 3.963303565979004, + 4.016773521900177, + 3.924777090549469, + 4.060843288898468, + 3.990393817424774, + 3.9138023257255554, + 4.115661263465881, + 4.081140518188477, + 4.029158174991608, + 4.0030242800712585, + 4.129499554634094, + 4.003529191017151, + 4.02143532037735, + 3.9911094307899475, + 4.011569440364838, + 4.032509803771973, + 3.915093243122101, + 3.9904491305351257, + 3.93692809343338, + 4.003243923187256, + 3.865770697593689, + 4.132464587688446, + 3.9781792759895325, + 4.06302547454834, + 4.062969505786896, + 3.9233070611953735, + 4.026380479335785, + 4.075709581375122, + 4.10675048828125, + 4.227143883705139, + 4.015190064907074, + 3.9972680807113647, + 4.065664768218994, + 4.046063542366028, + 4.037449479103088, + 3.9298312664031982, + 3.9745919704437256, + 3.8381391167640686, + 4.02036064863205, + 3.8365719318389893, + 4.178133249282837, + 4.156948506832123, + 3.9157959818840027, + 4.0012500286102295, + 3.9114730954170227, + 3.930005669593811, + 3.994531571865082, + 4.023687720298767, + 4.047671020030975, + 3.91159725189209, + 3.8947490453720093, + 4.0036797523498535, + 4.0043752789497375, + 3.810446858406067, + 3.9969438910484314, + 4.077446341514587, + 3.8559181094169617, + 4.049458980560303, + 4.029154121875763, + 4.1837159395217896, + 3.9550604224205017, + 3.9204387068748474, + 3.999455213546753, + 3.95525461435318, + 3.984743118286133, + 3.985463857650757, + 3.8622447848320007, + 3.914542853832245, + 4.025353372097015, + 3.930656611919403, + 3.9567525386810303, + 3.8592790365219116, + 3.9401023983955383, + 3.9360193014144897, + 3.9712700247764587, + 4.029421746730804, + 4.001932859420776, + 4.097268462181091, + 3.9079862236976624, + 3.989189565181732, + 3.7536062002182007, + 4.003122925758362, + 3.848651170730591, + 3.9653618335723877, + 4.033098459243774, + 3.799568474292755, + 3.9999241828918457, + 3.85162353515625, + 3.726219892501831, + 3.7879725098609924, + 3.863194704055786, + 3.902821958065033, + 3.8173053860664368, + 3.8478777408599854, + 3.7435502409934998, + 4.007396996021271, + 4.014470160007477, + 3.7366389632225037, + 3.841437816619873, + 3.7917783856391907, + 3.8677348494529724, + 4.024018704891205, + 3.749862849712372, + 3.796752691268921, + 3.744233548641205, + 3.9318031072616577, + 3.7684121131896973, + 4.031554877758026, + 3.8224570751190186, + 3.9408825039863586, + 3.848928213119507, + 3.7725019454956055, + 3.918089509010315, + 3.880687713623047, + 3.9467312693595886, + 3.763855457305908, + 3.7859591245651245, + 3.733026444911957, + 3.9120612144470215, + 3.9891350865364075, + 3.7693772315979004, + 3.8261131048202515, + 3.781972825527191, + 3.894092857837677, + 3.9337533712387085, + 3.907396376132965, + 3.7676198482513428, + 3.9027079343795776, + 3.7588332891464233, + 3.8337082266807556, + 3.8212730288505554, + 3.9844982028007507, + 3.7608985900878906, + 3.842369794845581, + 3.8946805000305176, + 3.8710882663726807, + 4.011194586753845, + 3.8052382469177246, + 3.9708272218704224, + 3.82006174325943, + 3.8104166388511658, + 3.9523885846138, + 3.8180383443832397, + 3.7095547914505005, + 4.044153571128845, + 3.7991409301757812, + 3.891858756542206, + 3.7336453795433044, + 3.760367214679718, + 3.953621506690979, + 3.859143555164337, + 3.903642416000366, + 3.793235182762146, + 3.868113160133362, + 3.910280764102936, + 3.829980492591858, + 3.88864928483963, + 3.8661062121391296, + 3.7930394411087036, + 3.7136037349700928, + 3.8162789940834045, + 3.9090630412101746, + 3.8639681935310364, + 3.810487747192383, + 3.872298240661621, + 3.676667809486389, + 3.8539499044418335, + 3.970405638217926, + 3.894469916820526, + 3.914325773715973, + 3.8906561732292175, + 3.805501937866211, + 3.7468491792678833, + 3.9255112409591675, + 3.8717318773269653, + 3.7112056016921997, + 3.797317326068878, + 3.8534393310546875, + 3.7974143624305725, + 3.718916594982147, + 3.895043969154358, + 3.9107216596603394, + 3.7607823610305786, + 3.9441739320755005, + 3.755466878414154, + 3.805622100830078, + 3.7237821221351624, + 3.7709034085273743, + 3.8702679872512817, + 3.8867735266685486, + 3.9152069687843323, + 3.7986875772476196, + 3.9384868144989014, + 3.8266748785972595, + 3.883030652999878, + 3.8760094046592712, + 3.72726172208786, + 3.8568530082702637, + 3.7323354482650757, + 3.7346524596214294, + 3.842785596847534, + 3.810943841934204, + 3.798758029937744, + 3.7618460059165955, + 3.7458801865577698, + 3.702134609222412, + 3.7955442070961, + 3.7248594760894775, + 3.774007022380829, + 3.7917022109031677, + 3.841720461845398, + 3.793367087841034, + 3.8255935311317444, + 3.719066560268402, + 3.6082720160484314, + 3.6625871062278748, + 3.695031225681305, + 3.772721588611603, + 3.5957170724868774, + 3.7109327912330627, + 3.6960257291793823, + 3.7125340700149536, + 3.81498783826828, + 3.6834596395492554, + 3.849091410636902, + 3.605737566947937, + 3.7315263748168945, + 3.724821925163269, + 3.7735179662704468, + 3.821566104888916, + 3.7899006009101868, + 3.6895581483840942, + 3.7020131945610046, + 3.7533432245254517, + 3.687644362449646, + 3.681840479373932, + 3.6172861456871033, + 3.70399671792984, + 3.7811997532844543, + 3.682201325893402, + 3.8524007201194763, + 3.687874913215637, + 3.835459291934967, + 3.6757076382637024, + 3.5769819021224976, + 3.7633330821990967, + 3.7340319752693176, + 3.8297855257987976, + 3.719040095806122, + 3.823285400867462, + 3.7598567605018616, + 3.7136037945747375, + 3.855195462703705, + 3.7260637879371643, + 3.7984964847564697, + 3.769551455974579, + 3.8556384444236755, + 3.698364555835724, + 3.6397518515586853, + 3.6077033281326294, + 3.688969910144806, + 3.8288528323173523, + 3.7726945281028748, + 3.736068606376648, + 3.736111342906952, + 3.7866258025169373, + 3.634745419025421, + 3.6212483644485474, + 3.7750303745269775, + 3.8031569719314575, + 3.6593793630599976, + 3.6505438685417175, + 3.749867796897888, + 3.650844156742096, + 3.6322132349014282, + 3.767669379711151, + 3.6676101088523865, + 3.7373313903808594, + 3.6760867834091187, + 3.7498786449432373, + 3.7987735867500305, + 3.718480944633484, + 3.717052638530731, + 3.726465940475464, + 3.7695884704589844, + 3.7270761132240295, + 3.748378038406372, + 3.630257785320282, + 3.7346365451812744, + 3.6048932671546936, + 3.7406554222106934, + 3.717111825942993, + 3.7641769647598267, + 3.6945388317108154, + 3.6269631385803223, + 3.71897429227829, + 3.6837618350982666, + 3.8273340463638306, + 3.6291356086730957, + 3.7032599449157715, + 3.7958491444587708, + 3.56551992893219, + 3.793949007987976, + 3.8175622820854187, + 3.7667744755744934, + 3.626189887523651, + 3.7191704511642456, + 3.8218994140625, + 3.7498459815979004, + 3.5978437662124634, + 3.8556582927703857, + 3.6578206419944763, + 3.7336288690567017, + 3.7834571599960327, + 3.651327133178711, + 3.6130863428115845, + 3.6278743743896484, + 3.6122589707374573, + 3.871680736541748, + 3.7513794898986816, + 3.8298224806785583, + 3.750628173351288, + 3.7162468433380127, + 3.6770490407943726, + 3.7344258427619934, + 3.6381253004074097, + 3.7295095324516296, + 3.769607663154602, + 3.628950297832489, + 3.7105003595352173, + 3.8196964263916016, + 3.7176249623298645, + 3.6580153703689575, + 3.656064748764038, + 3.6440680027008057, + 3.633617877960205, + 3.7329705953598022, + 3.704680919647217, + 3.640998065471649, + 3.6775216460227966, + 3.5008342266082764, + 3.6573703289031982, + 3.7441978454589844, + 3.727533221244812, + 3.6153826117515564, + 3.5930948853492737, + 3.560195207595825, + 3.510677456855774, + 3.6327015161514282, + 3.6896990537643433, + 3.62625652551651, + 3.6477898359298706, + 3.481342077255249, + 3.601281464099884, + 3.6139367818832397, + 3.548014998435974, + 3.6589696407318115, + 3.6332359313964844, + 3.447618842124939, + 3.5988980531692505, + 3.5515689849853516, + 3.644655227661133, + 3.696317672729492, + 3.659644842147827, + 3.6926082968711853, + 3.671837091445923, + 3.685012459754944, + 3.5043023228645325, + 3.5203269720077515, + 3.589215040206909, + 3.5032686591148376, + 3.574245035648346, + 3.5968304872512817, + 3.6615442037582397, + 3.5277852416038513, + 3.7252758741378784, + 3.604306221008301, + 3.5684589743614197, + 3.621882915496826, + 3.641248106956482, + 3.5954784750938416, + 3.556321918964386, + 3.6368691325187683, + 3.7869670391082764, + 3.5481894612312317, + 3.6295993328094482, + 3.5931308269500732, + 3.656713545322418, + 3.4919254779815674, + 3.6451146006584167, + 3.635913074016571, + 3.7299254536628723, + 3.565729260444641, + 3.671798825263977, + 3.7503402829170227, + 3.519165337085724, + 3.579064428806305, + 3.6583020091056824, + 3.6024409532546997, + 3.605368733406067, + 3.615744650363922, + 3.7427958250045776, + 3.577970325946808, + 3.562410533428192, + 3.6356704235076904, + 3.6335192918777466, + 3.6374810934066772, + 3.6562613248825073, + 3.6939892768859863, + 3.572326958179474, + 3.5460765957832336, + 3.792298376560211, + 3.555198013782501, + 3.6127665042877197, + 3.5719825625419617, + 3.6065601110458374, + 3.639051914215088, + 3.6046961545944214, + 3.5937876105308533, + 3.6118221282958984, + 3.5793187022209167, + 3.676631212234497, + 3.598361074924469, + 3.533904194831848, + 3.6506696939468384, + 3.5711849331855774, + 3.5996041893959045, + 3.6730501651763916, + 3.6930774450302124, + 3.5924772024154663, + 3.6159679293632507, + 3.556951940059662, + 3.619423508644104, + 3.6164085268974304, + 3.5663012266159058, + 3.6496050357818604, + 3.6304473876953125, + 3.599700391292572, + 3.415697455406189, + 3.6691415309906006, + 3.6220678091049194, + 3.641987919807434, + 3.6402485370635986, + 3.5582332611083984, + 3.5943334698677063, + 3.6588903665542603, + 3.529388427734375, + 3.6782408952713013, + 3.598145365715027, + 3.5008957982063293, + 3.614021062850952, + 3.639745771884918, + 3.5106828212738037, + 3.499109208583832, + 3.588572382926941, + 3.779714047908783, + 3.5999860763549805, + 3.5299869775772095, + 3.537274122238159, + 3.5815412402153015, + 3.5208048224449158, + 3.4987118244171143, + 3.5584974884986877, + 3.659028112888336, + 3.617615222930908, + 3.5052716732025146, + 3.6283575296401978, + 3.548335015773773, + 3.403844475746155, + 3.606377601623535, + 3.5610716342926025, + 3.589823842048645, + 3.4909502267837524, + 3.5625345706939697, + 3.526723623275757, + 3.477201819419861, + 3.388136923313141, + 3.548789858818054, + 3.33551162481308, + 3.489378571510315, + 3.502704620361328, + 3.55687814950943, + 3.4564476013183594, + 3.5603857040405273, + 3.3178634643554688, + 3.4521799087524414, + 3.5657589435577393, + 3.4252883791923523, + 3.5938684940338135, + 3.525482416152954, + 3.574934959411621, + 3.6118791103363037, + 3.466817080974579, + 3.5439518690109253, + 3.4223703742027283, + 3.551313638687134, + 3.532452881336212, + 3.580896496772766, + 3.344694972038269, + 3.6123680472373962, + 3.6094332337379456, + 3.5024518966674805, + 3.5091384649276733, + 3.441748321056366, + 3.589293956756592, + 3.3504441380500793, + 3.6140058040618896, + 3.5710604786872864, + 3.6670497059822083, + 3.4541969895362854, + 3.4692532420158386, + 3.460923671722412, + 3.553084671497345, + 3.5805389881134033, + 3.4706841707229614, + 3.612349033355713, + 3.4987571239471436, + 3.4356902837753296, + 3.5341697931289673, + 3.545713722705841, + 3.4882878065109253, + 3.4427850246429443, + 3.4536120891571045, + 3.5351690649986267, + 3.5403395295143127, + 3.5202826261520386, + 3.4781486988067627, + 3.5657953023910522, + 3.4461827874183655, + 3.5157406330108643, + 3.5011228919029236, + 3.657405376434326, + 3.630539894104004, + 3.541626513004303, + 3.467445731163025, + 3.6483777165412903, + 3.460386276245117, + 3.5745416283607483, + 3.451460301876068, + 3.598976492881775, + 3.3685986399650574, + 3.5608588457107544, + 3.511809766292572, + 3.5717262029647827, + 3.499462306499481, + 3.507915198802948, + 3.4479860067367554, + 3.4872241616249084, + 3.63088059425354, + 3.488322079181671, + 3.5204537510871887, + 3.521533191204071, + 3.468255043029785, + 3.533751368522644, + 3.5563730001449585, + 3.497363030910492, + 3.494450867176056, + 3.4968132376670837, + 3.5031895637512207, + 3.4769774675369263, + 3.6069998741149902, + 3.5428149700164795, + 3.5788310766220093, + 3.6101948022842407, + 3.5253883600234985, + 3.5839996933937073, + 3.502967894077301, + 3.519122898578644, + 3.5328763127326965, + 3.4375796914100647, + 3.4148807525634766, + 3.5752636790275574, + 3.579470634460449, + 3.5134008526802063, + 3.5071747303009033, + 3.490296483039856, + 3.3910637497901917, + 3.6495492458343506, + 3.4253739714622498, + 3.612153947353363, + 3.4158900380134583, + 3.4784467816352844, + 3.458151340484619, + 3.472554326057434, + 3.4373050332069397, + 3.7001237273216248, + 3.509928822517395, + 3.5625274181365967, + 3.642047643661499, + 3.5660969614982605, + 3.5652045607566833, + 3.636690855026245, + 3.5676774382591248, + 3.3920421600341797, + 3.512877404689789, + 3.607565760612488, + 3.4685524702072144, + 3.353170335292816, + 3.334070861339569, + 3.5044553875923157, + 3.5398669838905334, + 3.409167766571045, + 3.5384204983711243, + 3.4926498532295227, + 3.371450185775757, + 3.5311628580093384, + 3.495301365852356, + 3.460620880126953, + 3.5397124886512756, + 3.541575849056244, + 3.4834678173065186, + 3.389241099357605, + 3.4989275336265564, + 3.4303776025772095, + 3.5393583178520203, + 3.462011158466339, + 3.5392881631851196, + 3.439518690109253, + 3.528974711894989, + 3.4300856590270996, + 3.442164421081543, + 3.3180195689201355, + 3.5514360666275024, + 3.468357264995575, + 3.4441080689430237, + 3.485886037349701, + 3.5346338748931885, + 3.5511134266853333, + 3.3353278636932373, + 3.5275044441223145, + 3.429461658000946, + 3.6204410791397095, + 3.4088494777679443, + 3.516162157058716, + 3.499482035636902, + 3.579424202442169, + 3.5418867468833923, + 3.5848807096481323, + 3.553049623966217, + 3.424521744251251, + 3.5673290491104126, + 3.422239899635315, + 3.3276207447052, + 3.501420795917511, + 3.4770068526268005, + 3.4837329387664795, + 3.545331060886383, + 3.4316118359565735, + 3.522344648838043, + 3.563161611557007, + 3.363556981086731, + 3.47268009185791, + 3.449557065963745, + 3.489002823829651, + 3.5099461674690247, + 3.430854380130768, + 3.4944202303886414, + 3.442511558532715, + 3.543112099170685, + 3.5439918637275696, + 3.421990156173706, + 3.4365658164024353, + 3.465523362159729, + 3.4029341340065002, + 3.4638975262641907, + 3.5689039826393127, + 3.334779143333435, + 3.4972038865089417, + 3.511310577392578, + 3.462821364402771, + 3.567504346370697, + 3.409484624862671, + 3.516705870628357, + 3.5672789216041565, + 3.500578463077545, + 3.5291337966918945, + 3.3804491758346558, + 3.5895028710365295, + 3.5590386390686035, + 3.522342264652252, + 3.4624316096305847, + 3.628667950630188, + 3.4393250346183777, + 3.4366068840026855, + 3.5463984608650208, + 3.5127782821655273, + 3.4916182160377502, + 3.42312890291214, + 3.5321388244628906, + 3.525566518306732, + 3.56267112493515, + 3.455689787864685, + 3.5133866667747498, + 3.518623650074005, + 3.41375070810318, + 3.4113759994506836, + 3.5496917963027954, + 3.4335299730300903, + 3.601178228855133, + 3.480153441429138, + 3.52137690782547, + 3.4778425097465515, + 3.4035423398017883, + 3.455833673477173, + 3.4372962713241577, + 3.5329184532165527, + 3.434568405151367, + 3.5195468068122864 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 137.5589172962199, + 88.29352124092432, + 69.45074623479121, + 59.59401337961751, + 54.57426763012699, + 50.98237468399825, + 47.779299627872675, + 45.07747487359048, + 43.132484705737156, + 41.43934216303298, + 40.10054434026497, + 38.83385532564136, + 37.632288914868546, + 36.51358553216216, + 35.80382933981595, + 35.42290336726111, + 35.01871157217183, + 34.81596682763043, + 34.7498015604414, + 34.732506623529574 + ], + "best_ppl": 34.732506623529574, + "train_time_s": 53515.56079888344 +} \ No newline at end of file diff --git a/063_sparse_dispatch/sparse_3of5_best.pt b/063_sparse_dispatch/sparse_3of5_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9766e050b6fce28b8307776a29c5d67c38bee70 --- /dev/null +++ b/063_sparse_dispatch/sparse_3of5_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8db74a4be45b6bc7b576d4c4a029ef40464e8424a4a4920353ab0496c1af1ba5 +size 341437138 diff --git a/063_sparse_dispatch/sparse_3of5_results.json b/063_sparse_dispatch/sparse_3of5_results.json new file mode 100644 index 0000000000000000000000000000000000000000..0a8e2baf28945ddefc378564d6006946bef10d2f --- /dev/null +++ b/063_sparse_dispatch/sparse_3of5_results.json @@ -0,0 +1,2057 @@ +{ + "label": "sparse_3of5", + "attention_mode": "sparse", + "top_k_blocks": 3, + "num_thin_blocks": 5, + "total_params": 85301248, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.072399139404297, + 14.951687812805176, + 7.968461036682129, + 7.3158464431762695, + 7.042670726776123, + 6.953172206878662, + 6.9207611083984375, + 6.807319641113281, + 6.646448135375977, + 6.571646690368652, + 6.475419044494629, + 6.39260196685791, + 6.3692626953125, + 6.353853225708008, + 6.229607582092285, + 6.232788562774658, + 6.072934150695801, + 6.053422451019287, + 5.909632205963135, + 5.845078945159912, + 5.879326820373535, + 5.740766525268555, + 5.726682662963867, + 5.704777717590332, + 5.6920881271362305, + 5.588690280914307, + 5.594555854797363, + 5.54947566986084, + 5.405106067657471, + 5.431417465209961, + 5.315252780914307, + 5.320288181304932, + 5.321189880371094, + 5.220855712890625, + 5.1974945068359375, + 5.158652305603027, + 5.193483352661133, + 5.257125377655029, + 5.2157158851623535, + 5.107540607452393, + 5.137856960296631, + 5.060471057891846, + 4.954640865325928, + 5.071658611297607, + 5.004305362701416, + 5.049172401428223, + 4.9669575691223145, + 5.052967548370361, + 5.046236515045166, + 4.956637859344482, + 4.944624423980713, + 4.829961776733398, + 4.875871658325195, + 4.848449230194092, + 4.874717712402344, + 4.775044918060303, + 4.886752128601074, + 4.806126594543457, + 4.797695636749268, + 4.802882194519043, + 4.867430210113525, + 4.842244625091553, + 4.900299549102783, + 4.770301342010498, + 4.784807205200195, + 4.75434684753418, + 4.767013072967529, + 4.816514492034912, + 4.923490047454834, + 4.7767109870910645, + 4.65122127532959, + 4.763698101043701, + 4.593935489654541, + 4.6612229347229, + 4.619241714477539, + 4.608587265014648, + 4.777656555175781, + 4.603956699371338, + 4.688647747039795, + 4.746695041656494, + 4.621192455291748, + 4.499665260314941, + 4.613611221313477, + 4.6122612953186035, + 4.729654312133789, + 4.7245564460754395, + 4.44644021987915, + 4.6094889640808105, + 4.631587505340576, + 4.576406478881836, + 4.700160503387451, + 4.6636223793029785, + 4.505765914916992, + 4.374149322509766, + 4.6461687088012695, + 4.624763488769531, + 4.466754913330078, + 4.514833927154541, + 4.432999134063721, + 4.545431137084961, + 4.483433723449707, + 4.496365070343018, + 4.474416732788086, + 4.417865753173828, + 4.553761005401611, + 4.486048221588135, + 4.416388034820557, + 4.672353744506836, + 4.403458118438721, + 4.53262186050415, + 4.509998798370361, + 4.385572910308838, + 4.489054203033447, + 4.464268207550049, + 4.472114086151123, + 4.483141899108887, + 4.356162071228027, + 4.489251136779785, + 4.422949314117432, + 4.3471174240112305, + 4.445897579193115, + 4.431772708892822, + 4.3024678230285645, + 4.404905796051025, + 4.375491142272949, + 4.427358150482178, + 4.266242980957031, + 4.259321212768555, + 4.240056037902832, + 4.291114330291748, + 4.320362567901611, + 4.265028476715088, + 4.30930757522583, + 4.331760406494141, + 4.209710121154785, + 4.310436248779297, + 4.207474231719971, + 4.307411193847656, + 4.303237438201904, + 4.205215930938721, + 4.074264049530029, + 4.115577697753906, + 4.280413627624512, + 4.19399356842041, + 4.193295478820801, + 4.308166980743408, + 4.2741475105285645, + 4.206488132476807, + 4.14858865737915, + 4.223536014556885, + 4.1181840896606445, + 4.279662609100342, + 4.162535667419434, + 4.248851299285889, + 4.399037837982178, + 4.149134159088135, + 4.247748374938965, + 4.195012092590332, + 4.252321720123291, + 4.22531795501709, + 4.221015453338623, + 4.143991470336914, + 4.214654445648193, + 4.204080104827881, + 4.244068622589111, + 4.290028095245361, + 4.1325154304504395, + 4.200721740722656, + 4.243250370025635, + 4.210155487060547, + 4.186770915985107, + 4.128022193908691, + 4.242994785308838, + 4.080766677856445, + 4.1881256103515625, + 4.081320762634277, + 4.14447021484375, + 4.13832950592041, + 4.068941116333008, + 4.0577616691589355, + 4.308382987976074, + 4.0752482414245605, + 4.251649856567383, + 4.1876220703125, + 3.9705159664154053, + 4.175777912139893, + 4.201783180236816, + 4.101339340209961, + 4.093857765197754, + 4.2106404304504395, + 4.145105838775635, + 4.153482913970947, + 4.004327297210693, + 4.1992292404174805, + 4.1804375648498535, + 4.046049118041992, + 4.088892936706543, + 4.1356658935546875, + 4.152240753173828, + 4.097808837890625, + 4.17466402053833, + 4.087278842926025, + 4.126100540161133, + 4.116613388061523, + 4.136358261108398, + 4.112893581390381, + 4.09843635559082, + 4.237074375152588, + 4.0996503829956055, + 4.031558990478516, + 4.147128582000732, + 4.077393531799316, + 4.014761447906494, + 4.083701133728027, + 4.0049824714660645, + 4.068606853485107, + 4.071672439575195, + 4.11013126373291, + 4.0835957527160645, + 4.001869201660156, + 3.997112512588501, + 4.004251003265381, + 4.181831359863281, + 4.124111652374268, + 4.015565872192383, + 4.077507495880127, + 3.9722986221313477, + 4.04956579208374, + 4.028232574462891, + 4.185471534729004, + 4.122315406799316, + 4.0575971603393555, + 4.109752655029297, + 3.9628472328186035, + 4.096440315246582, + 3.940082311630249, + 4.058355808258057, + 4.065576553344727, + 4.166477680206299, + 4.094417572021484, + 4.066967010498047, + 3.9605295658111572, + 4.041696071624756, + 4.08766508102417, + 3.910512685775757, + 4.115388870239258, + 4.013852119445801, + 3.9330554008483887, + 4.01840877532959, + 4.0492730140686035, + 3.9844532012939453, + 3.9830985069274902, + 4.007513046264648, + 3.8677210807800293, + 3.781808376312256, + 3.9208717346191406, + 3.9223289489746094, + 3.904834032058716, + 3.889390707015991, + 3.94187331199646, + 4.002790927886963, + 3.8850200176239014, + 3.9994518756866455, + 3.8633482456207275, + 3.933314561843872, + 3.9045183658599854, + 3.9635679721832275, + 3.8710198402404785, + 3.958010673522949, + 3.9680607318878174, + 3.85487961769104, + 3.8993494510650635, + 3.94873309135437, + 3.933006525039673, + 3.9051833152770996, + 3.9900293350219727, + 3.999328374862671, + 3.8822014331817627, + 3.8326241970062256, + 3.987492799758911, + 3.7092034816741943, + 4.062590599060059, + 3.9318385124206543, + 3.9572203159332275, + 3.866833448410034, + 3.9728496074676514, + 3.8303442001342773, + 3.87777042388916, + 3.832674741744995, + 3.956998348236084, + 3.9646193981170654, + 3.8418006896972656, + 3.924975872039795, + 3.8767457008361816, + 3.945923089981079, + 3.9425618648529053, + 3.9361789226531982, + 3.798281192779541, + 3.975806951522827, + 3.765434741973877, + 3.973093032836914, + 4.010854721069336, + 3.9306273460388184, + 3.8225252628326416, + 3.8976151943206787, + 3.927813768386841, + 3.93664813041687, + 3.8476619720458984, + 3.80946683883667, + 3.7624993324279785, + 3.9024596214294434, + 3.7514898777008057, + 3.913846492767334, + 3.8463563919067383, + 3.884474992752075, + 3.637791395187378, + 3.768751859664917, + 3.886925458908081, + 3.9533891677856445, + 3.7689781188964844, + 3.8350937366485596, + 3.7298526763916016, + 3.9205033779144287, + 3.8161981105804443, + 4.04212760925293, + 3.8868727684020996, + 3.873356580734253, + 3.8317553997039795, + 3.8568952083587646, + 3.77647066116333, + 3.8818864822387695, + 3.7466232776641846, + 3.9175143241882324, + 3.8239102363586426, + 3.803030490875244, + 3.845167398452759, + 3.891338348388672, + 3.8277945518493652, + 3.828958749771118, + 3.9341869354248047, + 3.900524616241455, + 3.7615184783935547, + 3.7360341548919678, + 3.9845778942108154, + 3.835759401321411, + 3.9257566928863525, + 3.995668888092041, + 3.8833532333374023, + 3.908926248550415, + 3.730506658554077, + 3.8947677612304688, + 3.8408620357513428, + 3.858671188354492, + 3.8609766960144043, + 3.906405210494995, + 3.81259822845459, + 3.9210360050201416, + 3.761014461517334, + 3.8926262855529785, + 3.8563265800476074, + 3.792630434036255, + 3.8702869415283203, + 3.9514453411102295, + 3.8698906898498535, + 3.8001697063446045, + 3.798804521560669, + 3.819242000579834, + 3.9256763458251953, + 3.84892201423645, + 3.866899251937866, + 3.8405632972717285, + 3.891876220703125, + 3.74652099609375, + 3.8579280376434326, + 3.8585193157196045, + 3.7876694202423096, + 3.7802488803863525, + 3.7402617931365967, + 3.8666651248931885, + 3.728992462158203, + 3.701575517654419, + 3.7122373580932617, + 3.6315908432006836, + 3.78544282913208, + 3.7338685989379883, + 3.897888422012329, + 3.864208459854126, + 3.693852186203003, + 3.7818081378936768, + 3.6442646980285645, + 3.9036877155303955, + 3.7362241744995117, + 3.7425360679626465, + 3.8344194889068604, + 3.750169515609741, + 3.7498579025268555, + 3.8234024047851562, + 3.7640233039855957, + 3.797736406326294, + 3.783550977706909, + 3.6851866245269775, + 3.700772762298584, + 3.7815961837768555, + 3.782518148422241, + 3.6194143295288086, + 3.688314199447632, + 3.715989351272583, + 3.643824815750122, + 3.8257038593292236, + 3.685710906982422, + 3.8593409061431885, + 3.8907995223999023, + 3.8180856704711914, + 3.7434194087982178, + 3.7662136554718018, + 3.6864328384399414, + 3.813534736633301, + 3.7925984859466553, + 3.7890822887420654, + 3.8181991577148438, + 3.829453229904175, + 3.808089256286621, + 3.818173885345459, + 3.6687638759613037, + 3.7550671100616455, + 3.8161673545837402, + 3.762561559677124, + 3.7134034633636475, + 3.73213791847229, + 3.8366050720214844, + 3.615265369415283, + 3.6671581268310547, + 3.758538007736206, + 3.7669382095336914, + 3.553828001022339, + 3.8063912391662598, + 3.762777328491211, + 3.761451005935669, + 3.734584093093872, + 3.7566399574279785, + 3.7068638801574707, + 3.858116626739502, + 3.6587822437286377, + 3.790572166442871, + 3.8580198287963867, + 3.6226089000701904, + 3.8466200828552246, + 3.7301931381225586, + 3.722867727279663, + 3.828583240509033, + 3.699800491333008, + 3.661071300506592, + 3.868014335632324, + 3.7039122581481934, + 3.722811460494995, + 3.815716505050659, + 3.792675256729126, + 3.8111679553985596, + 3.75658917427063, + 3.7878897190093994, + 3.813933849334717, + 3.7105050086975098, + 3.8112680912017822, + 3.7382113933563232, + 3.8365213871002197, + 3.696465253829956, + 3.6142942905426025, + 3.7562806606292725, + 3.7291431427001953, + 3.751415491104126, + 3.723539352416992, + 3.774989128112793, + 3.734238624572754, + 3.805177688598633, + 3.816094398498535, + 3.824695110321045, + 3.6996188163757324, + 3.8817138671875, + 3.6887269020080566, + 3.574986219406128, + 3.6386330127716064, + 3.6046273708343506, + 3.632305860519409, + 3.736827850341797, + 3.594163417816162, + 3.7990243434906006, + 3.7414016723632812, + 3.6670689582824707, + 3.7390871047973633, + 3.633012294769287, + 3.6655216217041016, + 3.712462902069092, + 3.7276530265808105, + 3.7604777812957764, + 3.7273600101470947, + 3.6502809524536133, + 3.7383534908294678, + 3.6333539485931396, + 3.7287209033966064, + 3.793323278427124, + 3.7757277488708496, + 3.7905540466308594, + 3.654816150665283, + 3.6760776042938232, + 3.755990743637085, + 3.7993173599243164, + 3.637601137161255, + 3.727459669113159, + 3.7191524505615234, + 3.6059751510620117, + 3.668222427368164, + 3.7387444972991943, + 3.7537295818328857, + 3.6557657718658447, + 3.508143424987793, + 3.4966824054718018, + 3.646127462387085, + 3.472245454788208, + 3.5351076126098633, + 3.5768117904663086, + 3.7070183753967285, + 3.7140796184539795, + 3.582472562789917, + 3.605759859085083, + 3.672464609146118, + 3.662449359893799, + 3.758138418197632, + 3.5810251235961914, + 3.5439014434814453, + 3.6574790477752686, + 3.585580587387085, + 3.5699093341827393, + 3.5548787117004395, + 3.6098852157592773, + 3.5841269493103027, + 3.643261194229126, + 3.583380699157715, + 3.641620635986328, + 3.615023136138916, + 3.6685400009155273, + 3.499929189682007, + 3.643044948577881, + 3.743455410003662, + 3.6939992904663086, + 3.574657678604126, + 3.4800901412963867, + 3.6595728397369385, + 3.643418788909912, + 3.5712409019470215, + 3.638733148574829, + 3.684753894805908, + 3.6548211574554443, + 3.758854866027832, + 3.620964527130127, + 3.5699636936187744, + 3.5785605907440186, + 3.6343955993652344, + 3.797321319580078, + 3.603569746017456, + 3.6021487712860107, + 3.6521801948547363, + 3.439700126647949, + 3.5110294818878174, + 3.801685333251953, + 3.558506965637207, + 3.5205588340759277, + 3.658733606338501, + 3.576467275619507, + 3.6715800762176514, + 3.667586326599121, + 3.6877799034118652, + 3.6476948261260986, + 3.661301612854004, + 3.570411443710327, + 3.797853469848633, + 3.6233551502227783, + 3.7178425788879395, + 3.627011299133301, + 3.6012916564941406, + 3.5887258052825928, + 3.6774985790252686, + 3.617988109588623, + 3.741459608078003, + 3.521084785461426, + 3.578246831893921, + 3.673431873321533, + 3.7553579807281494, + 3.579585313796997, + 3.5923078060150146, + 3.6732027530670166, + 3.6997687816619873, + 3.6182541847229004, + 3.5657501220703125, + 3.5425310134887695, + 3.464693784713745, + 3.601210117340088, + 3.5827808380126953, + 3.654721975326538, + 3.532364845275879, + 3.759856700897217, + 3.577991008758545, + 3.5579001903533936, + 3.6464102268218994, + 3.596334934234619, + 3.53169584274292, + 3.5404341220855713, + 3.4850287437438965, + 3.6442983150482178, + 3.5894055366516113, + 3.51668643951416, + 3.5318543910980225, + 3.5559678077697754, + 3.6853978633880615, + 3.618058681488037, + 3.704547882080078, + 3.5891518592834473, + 3.4420244693756104, + 3.7073171138763428, + 3.6295740604400635, + 3.5303328037261963, + 3.656254529953003, + 3.5313479900360107, + 3.5091378688812256, + 3.6458587646484375, + 3.550842761993408, + 3.6367027759552, + 3.5991733074188232, + 3.5393779277801514, + 3.6993021965026855, + 3.524020195007324, + 3.708144187927246, + 3.6601004600524902, + 3.6069459915161133, + 3.5066778659820557, + 3.6415369510650635, + 3.6810929775238037, + 3.4013113975524902, + 3.4785945415496826, + 3.4885191917419434, + 3.4840290546417236, + 3.639472484588623, + 3.492906093597412, + 3.475762128829956, + 3.51682448387146, + 3.5953426361083984, + 3.4443540573120117, + 3.5107369422912598, + 3.557375431060791, + 3.5346405506134033, + 3.4895594120025635, + 3.4440951347351074, + 3.5559470653533936, + 3.5251870155334473, + 3.4799299240112305, + 3.5631864070892334, + 3.5438992977142334, + 3.6198699474334717, + 3.5835654735565186, + 3.5867109298706055, + 3.497530698776245, + 3.4992141723632812, + 3.4803013801574707, + 3.500370740890503, + 3.5329456329345703, + 3.561899423599243, + 3.564948320388794, + 3.561350107192993, + 3.493612051010132, + 3.5956788063049316, + 3.4895827770233154, + 3.582057476043701, + 3.495250940322876, + 3.598559856414795, + 3.5995821952819824, + 3.4434287548065186, + 3.6204159259796143, + 3.468153953552246, + 3.706766366958618, + 3.4916417598724365, + 3.430102586746216, + 3.6245481967926025, + 3.4904556274414062, + 3.5533947944641113, + 3.5596954822540283, + 3.537945032119751, + 3.400927782058716, + 3.585411787033081, + 3.306241273880005, + 3.493929624557495, + 3.4683938026428223, + 3.5315940380096436, + 3.49133038520813, + 3.4516191482543945, + 3.6112074851989746, + 3.537320137023926, + 3.643012285232544, + 3.477949380874634, + 3.5120739936828613, + 3.4981021881103516, + 3.543278932571411, + 3.5834341049194336, + 3.5899364948272705, + 3.46716046333313, + 3.5901670455932617, + 3.38632869720459, + 3.5344338417053223, + 3.659803867340088, + 3.5069453716278076, + 3.359593391418457, + 3.560777425765991, + 3.4732825756073, + 3.511737823486328, + 3.4954121112823486, + 3.616562604904175, + 3.4300835132598877, + 3.555291175842285, + 3.526355504989624, + 3.4304184913635254, + 3.5839154720306396, + 3.5285451412200928, + 3.5377373695373535, + 3.400437355041504, + 3.570873260498047, + 3.641587972640991, + 3.512140989303589, + 3.4866254329681396, + 3.5587317943573, + 3.4464709758758545, + 3.5169453620910645, + 3.4854471683502197, + 3.544328212738037, + 3.5184504985809326, + 3.702655076980591, + 3.447019338607788, + 3.4387366771698, + 3.477125406265259, + 3.3892860412597656, + 3.473524570465088, + 3.6158173084259033, + 3.624518394470215, + 3.567972421646118, + 3.5685932636260986, + 3.5524232387542725, + 3.613001823425293, + 3.4269580841064453, + 3.5586557388305664, + 3.526240110397339, + 3.27449107170105, + 3.4292407035827637, + 3.4650981426239014, + 3.5698225498199463, + 3.3354454040527344, + 3.6825551986694336, + 3.593169689178467, + 3.469144582748413, + 3.436652421951294, + 3.527209997177124, + 3.525420665740967, + 3.4872422218322754, + 3.46004056930542, + 3.616274356842041, + 3.368767499923706, + 3.3100438117980957, + 3.450939178466797, + 3.5316121578216553, + 3.4611475467681885, + 3.39463472366333, + 3.5242059230804443, + 3.3528597354888916, + 3.441312313079834, + 3.3814589977264404, + 3.6142406463623047, + 3.3977572917938232, + 3.529726982116699, + 3.4886653423309326, + 3.4014949798583984, + 3.408573627471924, + 3.456083059310913, + 3.4848570823669434, + 3.4578146934509277, + 3.4849870204925537, + 3.3908565044403076, + 3.4254393577575684, + 3.4811108112335205, + 3.522127866744995, + 3.4092471599578857, + 3.2912378311157227, + 3.4121084213256836, + 3.55486798286438, + 3.3196728229522705, + 3.3968167304992676, + 3.4150984287261963, + 3.4299144744873047, + 3.433835506439209, + 3.455597400665283, + 3.3200957775115967, + 3.492469549179077, + 3.397317409515381, + 3.3106019496917725, + 3.532729148864746, + 3.329587459564209, + 3.5585899353027344, + 3.483987331390381, + 3.4775843620300293, + 3.4396791458129883, + 3.3817028999328613, + 3.4600319862365723, + 3.5064332485198975, + 3.4953725337982178, + 3.3365955352783203, + 3.4877874851226807, + 3.359065294265747, + 3.5543606281280518, + 3.4744834899902344, + 3.444023370742798, + 3.2742297649383545, + 3.373561143875122, + 3.4088616371154785, + 3.4555885791778564, + 3.461644172668457, + 3.4703257083892822, + 3.474120855331421, + 3.5496461391448975, + 3.429250717163086, + 3.355409622192383, + 3.5330276489257812, + 3.3916749954223633, + 3.5859906673431396, + 3.407924175262451, + 3.435930013656616, + 3.444448232650757, + 3.3591768741607666, + 3.4244883060455322, + 3.5613033771514893, + 3.427474021911621, + 3.4643824100494385, + 3.4253265857696533, + 3.3602161407470703, + 3.4841105937957764, + 3.4398281574249268, + 3.4512338638305664, + 3.528341054916382, + 3.54392409324646, + 3.4139952659606934, + 3.520038604736328, + 3.443354845046997, + 3.4142861366271973, + 3.3563284873962402, + 3.518040657043457, + 3.4013938903808594, + 3.494523048400879, + 3.38337779045105, + 3.4850029945373535, + 3.352771759033203, + 3.458043336868286, + 3.368856430053711, + 3.59045672416687, + 3.4738781452178955, + 3.2260265350341797, + 3.4408457279205322, + 3.475554943084717, + 3.323598623275757, + 3.4529685974121094, + 3.2824349403381348, + 3.4778246879577637, + 3.45884108543396, + 3.554365634918213, + 3.491779088973999, + 3.372840404510498, + 3.262528896331787, + 3.4374237060546875, + 3.4363486766815186, + 3.476006031036377, + 3.5118234157562256, + 3.494236946105957, + 3.4409303665161133, + 3.3454461097717285, + 3.408949375152588, + 3.3541038036346436, + 3.4795725345611572, + 3.3921103477478027, + 3.4882075786590576, + 3.5205774307250977, + 3.40657377243042, + 3.422882556915283, + 3.406764507293701, + 3.4432663917541504, + 3.510897636413574, + 3.399739980697632, + 3.3922293186187744, + 3.324014902114868, + 3.2407381534576416, + 3.2959182262420654, + 3.300712823867798, + 3.48758602142334, + 3.400576591491699, + 3.398529291152954, + 3.4698519706726074, + 3.460012197494507, + 3.4344582557678223, + 3.423583507537842, + 3.357182025909424, + 3.319951295852661, + 3.4145889282226562, + 3.2809205055236816, + 3.5112593173980713, + 3.387594223022461, + 3.4233875274658203, + 3.389755964279175, + 3.3389060497283936, + 3.5021779537200928, + 3.2809693813323975, + 3.276301860809326, + 3.3881657123565674, + 3.4638681411743164, + 3.3783504962921143, + 3.3343005180358887, + 3.399547815322876, + 3.488506555557251, + 3.4571428298950195, + 3.4051434993743896, + 3.429518461227417, + 3.498753309249878, + 3.4046177864074707, + 3.398559331893921, + 3.3064136505126953, + 3.381000518798828, + 3.3358309268951416, + 3.44270396232605, + 3.319154739379883, + 3.4091403484344482, + 3.3947067260742188, + 3.3416285514831543, + 3.345123052597046, + 3.352689027786255, + 3.345968246459961, + 3.5345427989959717, + 3.3092246055603027, + 3.5699636936187744, + 3.416107654571533, + 3.385805368423462, + 3.3980634212493896, + 3.4101455211639404, + 3.31003475189209, + 3.3454837799072266, + 3.3388967514038086, + 3.3549938201904297, + 3.3107993602752686, + 3.474017381668091, + 3.462002754211426, + 3.313995599746704, + 3.4228672981262207, + 3.483330011367798, + 3.2574551105499268, + 3.31276798248291, + 3.286630153656006, + 3.520796060562134, + 3.4415602684020996, + 3.3517682552337646, + 3.477825880050659, + 3.4240293502807617, + 3.3858015537261963, + 3.5228521823883057, + 3.4603559970855713, + 3.352088451385498, + 3.3747997283935547, + 3.355421543121338, + 3.3757643699645996, + 3.4528067111968994, + 3.373262882232666, + 3.3680882453918457, + 3.365508556365967, + 3.385547399520874, + 3.357282876968384, + 3.4425442218780518, + 3.3477659225463867, + 3.5518431663513184, + 3.3550353050231934, + 3.4996001720428467, + 3.4114391803741455, + 3.469775438308716, + 3.3734915256500244, + 3.4286093711853027, + 3.2480573654174805, + 3.384148597717285, + 3.263395071029663, + 3.4476325511932373, + 3.347179412841797, + 3.4459526538848877, + 3.282773494720459, + 3.345118761062622, + 3.5032362937927246, + 3.32737135887146, + 3.3204877376556396, + 3.306814432144165, + 3.3737988471984863, + 3.2704834938049316, + 3.371638536453247, + 3.3094420433044434, + 3.2986934185028076 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 127.79011711854494, + 80.4178543447686, + 63.01517693362947, + 54.55857810504423, + 49.52614175608912, + 46.200244700227685, + 43.46254709692607, + 41.39397696020847, + 39.577604240596834, + 37.85171891381355, + 36.767984865459496, + 35.61596204719958, + 34.76340257855676, + 33.98998231629546, + 33.08657607410046, + 32.79011686056929, + 32.38176421454738, + 32.160107808481506, + 32.05592134632515, + 32.038193485269986 + ], + "best_ppl": 32.038193485269986, + "train_time_s": 21542.466686487198 +} \ No newline at end of file diff --git a/063_sparse_dispatch/sparse_3of5_s137_best.pt b/063_sparse_dispatch/sparse_3of5_s137_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9443424d627247284f1860fb3e98177dd534010 --- /dev/null +++ b/063_sparse_dispatch/sparse_3of5_s137_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be5344bf734a2099ab960a4e1f2c569268deea4d4f4f9da0cdcfeb1800ccea69 +size 341440023 diff --git a/063_sparse_dispatch/sparse_3of5_s137_results.json b/063_sparse_dispatch/sparse_3of5_s137_results.json new file mode 100644 index 0000000000000000000000000000000000000000..fbd0c32374feb3608c4c2b471cf1e0b0696fa756 --- /dev/null +++ b/063_sparse_dispatch/sparse_3of5_s137_results.json @@ -0,0 +1,2055 @@ +{ + "label": "sparse_3of5_s137", + "seed": 137, + "total_params": 85301248, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 19.00624370574951, + 14.764864921569824, + 7.933702230453491, + 7.2879252433776855, + 7.125829696655273, + 6.975056171417236, + 6.874757528305054, + 6.836527347564697, + 6.807802677154541, + 6.587696313858032, + 6.479192018508911, + 6.4619975090026855, + 6.395462274551392, + 6.283653497695923, + 6.169692516326904, + 6.122941255569458, + 6.182207822799683, + 5.95159125328064, + 6.0002968311309814, + 5.959283828735352, + 5.853546857833862, + 5.853346824645996, + 5.5972959995269775, + 5.757469654083252, + 5.670509099960327, + 5.497901201248169, + 5.526661157608032, + 5.312265157699585, + 5.391885280609131, + 5.444512367248535, + 5.382747173309326, + 5.442782878875732, + 5.269933462142944, + 5.391601085662842, + 5.191283702850342, + 5.185875177383423, + 5.256370306015015, + 5.1172990798950195, + 5.17357063293457, + 5.03678035736084, + 5.1457202434539795, + 5.056251287460327, + 5.072100877761841, + 4.982715129852295, + 5.1418373584747314, + 5.030200958251953, + 4.9249656200408936, + 4.862633228302002, + 5.086714029312134, + 5.027366638183594, + 5.003968715667725, + 4.870466709136963, + 4.888205528259277, + 5.0311119556427, + 4.9362030029296875, + 4.8837456703186035, + 4.975872278213501, + 4.871796607971191, + 4.853100538253784, + 4.827196836471558, + 4.851240634918213, + 4.696966171264648, + 4.810621500015259, + 4.81095290184021, + 4.825911283493042, + 4.892751216888428, + 4.706533432006836, + 4.783127069473267, + 4.732372522354126, + 4.746831893920898, + 4.765121698379517, + 4.755466938018799, + 4.696552276611328, + 4.719926118850708, + 4.638725757598877, + 4.748489856719971, + 4.59103536605835, + 4.5878283977508545, + 4.7833967208862305, + 4.770924806594849, + 4.575622797012329, + 4.7394208908081055, + 4.683669328689575, + 4.595155477523804, + 4.6805479526519775, + 4.608510494232178, + 4.527660131454468, + 4.629246711730957, + 4.530029773712158, + 4.5419206619262695, + 4.573042392730713, + 4.621586084365845, + 4.499370574951172, + 4.533714771270752, + 4.583979368209839, + 4.570641994476318, + 4.556313753128052, + 4.415613412857056, + 4.487816572189331, + 4.570837497711182, + 4.499359130859375, + 4.588651180267334, + 4.46393895149231, + 4.588857889175415, + 4.520852327346802, + 4.449071884155273, + 4.387063264846802, + 4.646157741546631, + 4.5457377433776855, + 4.335913419723511, + 4.395728588104248, + 4.315550804138184, + 4.450008392333984, + 4.443650484085083, + 4.373550891876221, + 4.411838531494141, + 4.4019598960876465, + 4.415446043014526, + 4.529035568237305, + 4.410248279571533, + 4.517698526382446, + 4.392451763153076, + 4.366700649261475, + 4.3925940990448, + 4.4733171463012695, + 4.433312892913818, + 4.490865230560303, + 4.200274467468262, + 4.403196096420288, + 4.387848377227783, + 4.37469482421875, + 4.222170352935791, + 4.258625268936157, + 4.352051496505737, + 4.283688545227051, + 4.215829133987427, + 4.368433237075806, + 4.38452672958374, + 4.245287656784058, + 4.340872526168823, + 4.107722997665405, + 4.359358549118042, + 4.308703184127808, + 4.259401321411133, + 4.286249876022339, + 4.201111316680908, + 4.222501277923584, + 4.320613145828247, + 4.3074257373809814, + 4.30997109413147, + 4.224559545516968, + 4.2132322788238525, + 4.199744939804077, + 4.175720453262329, + 4.304201602935791, + 4.192932844161987, + 4.207871675491333, + 4.310330390930176, + 4.222501277923584, + 4.220419406890869, + 4.2684714794158936, + 4.172555446624756, + 4.090263605117798, + 4.160881280899048, + 4.306301832199097, + 4.2747461795806885, + 4.181911945343018, + 4.2338268756866455, + 4.268137454986572, + 4.165443420410156, + 4.188204765319824, + 4.176653861999512, + 4.190473794937134, + 4.259004592895508, + 4.194220304489136, + 4.272334814071655, + 4.109250783920288, + 4.201697587966919, + 4.235952377319336, + 4.213006973266602, + 4.235206604003906, + 4.153294086456299, + 4.043404579162598, + 4.27047872543335, + 4.137582540512085, + 4.1092259883880615, + 4.1706953048706055, + 4.311418056488037, + 4.185211658477783, + 4.137164354324341, + 4.241080284118652, + 4.178656578063965, + 4.201308965682983, + 4.055185794830322, + 4.148422002792358, + 4.077595829963684, + 4.129255533218384, + 4.247796535491943, + 4.246195554733276, + 4.1098792552948, + 3.99810528755188, + 4.058053255081177, + 4.0918262004852295, + 4.14960503578186, + 4.042178630828857, + 4.192123889923096, + 4.110687255859375, + 4.123219728469849, + 4.059433341026306, + 4.252355337142944, + 3.9977502822875977, + 4.060725927352905, + 3.979015827178955, + 4.184646129608154, + 4.143059015274048, + 4.1338653564453125, + 4.124765396118164, + 4.108064889907837, + 4.07691216468811, + 4.044081449508667, + 4.12995171546936, + 4.073822259902954, + 4.031527996063232, + 4.025275468826294, + 3.9148324728012085, + 4.111581444740295, + 4.056290149688721, + 4.0752153396606445, + 4.045640587806702, + 3.9945462942123413, + 4.074704885482788, + 4.154489040374756, + 4.047406911849976, + 4.004008054733276, + 4.150163173675537, + 3.955366849899292, + 4.142703294754028, + 4.100525856018066, + 4.037622928619385, + 3.9858431816101074, + 4.005329728126526, + 3.996452212333679, + 4.071216344833374, + 4.18156623840332, + 4.02788245677948, + 3.9230602979660034, + 4.04823911190033, + 4.003732204437256, + 3.9916317462921143, + 4.100046157836914, + 4.090731620788574, + 3.928518533706665, + 4.075187921524048, + 4.1547980308532715, + 3.913611888885498, + 3.825294613838196, + 3.9599896669387817, + 3.9509769678115845, + 4.026406168937683, + 3.8903571367263794, + 3.8974956274032593, + 3.977272629737854, + 3.7812511920928955, + 3.996170997619629, + 4.038425922393799, + 3.8701130151748657, + 4.010015606880188, + 3.9298043251037598, + 3.973548650741577, + 4.006693959236145, + 3.946632981300354, + 4.022989988327026, + 4.069969892501831, + 4.00744092464447, + 4.021457672119141, + 3.8499525785446167, + 3.9400776624679565, + 3.9344935417175293, + 3.9858295917510986, + 3.945244073867798, + 3.9602867364883423, + 3.8874566555023193, + 3.8970710039138794, + 3.853717803955078, + 3.9625582695007324, + 3.9075849056243896, + 3.945645809173584, + 3.906550168991089, + 4.03108012676239, + 3.8851410150527954, + 3.9820098876953125, + 3.992926597595215, + 4.010279059410095, + 3.8544881343841553, + 3.8428971767425537, + 3.9261090755462646, + 3.8138941526412964, + 3.8890790939331055, + 3.928730845451355, + 4.025186061859131, + 3.898804545402527, + 3.90436851978302, + 3.915238618850708, + 3.8826851844787598, + 4.021950960159302, + 3.9791396856307983, + 3.8739192485809326, + 3.908092498779297, + 3.8888243436813354, + 3.8801605701446533, + 3.95888614654541, + 3.9178465604782104, + 3.9314485788345337, + 3.9492768049240112, + 3.8675988912582397, + 3.9247273206710815, + 3.918030858039856, + 4.0879738330841064, + 3.93335497379303, + 3.9034165143966675, + 3.9411646127700806, + 3.8497331142425537, + 4.012990474700928, + 3.975598096847534, + 3.8926395177841187, + 3.9398744106292725, + 3.8869662284851074, + 3.853181004524231, + 3.8670074939727783, + 3.8133662939071655, + 3.926211953163147, + 3.913663387298584, + 3.982375383377075, + 3.804147243499756, + 3.8009753227233887, + 4.00071918964386, + 3.749038338661194, + 3.913132667541504, + 3.7330241203308105, + 3.8290417194366455, + 3.8405522108078003, + 3.78555691242218, + 3.903803825378418, + 3.9598443508148193, + 3.8836822509765625, + 3.826321840286255, + 3.92606258392334, + 3.8414732217788696, + 3.7616506814956665, + 3.9085947275161743, + 3.895375370979309, + 3.9196680784225464, + 3.813716769218445, + 3.8803430795669556, + 3.967016339302063, + 3.8978819847106934, + 3.873326539993286, + 3.8996083736419678, + 3.8906290531158447, + 3.886269211769104, + 3.8159271478652954, + 3.9637986421585083, + 3.902242064476013, + 3.806087851524353, + 3.9075053930282593, + 3.9086546897888184, + 3.9092906713485718, + 3.924252510070801, + 3.824466109275818, + 3.9854302406311035, + 3.869010329246521, + 3.98656690120697, + 3.938281774520874, + 3.9102782011032104, + 3.8329237699508667, + 3.961005687713623, + 3.848484754562378, + 3.7476208209991455, + 3.783395767211914, + 3.733419895172119, + 3.6240423917770386, + 3.643031358718872, + 3.7845263481140137, + 3.6255669593811035, + 3.747653603553772, + 3.9464138746261597, + 3.7387269735336304, + 3.8592721223831177, + 3.696173906326294, + 3.8488575220108032, + 3.750205397605896, + 3.6836577653884888, + 3.8633264303207397, + 3.694169282913208, + 3.9285460710525513, + 3.715839385986328, + 3.83502733707428, + 3.5552399158477783, + 3.6621071100234985, + 3.867656707763672, + 3.702360510826111, + 3.8248523473739624, + 3.79657244682312, + 3.703390955924988, + 3.847104787826538, + 3.7969815731048584, + 3.682965397834778, + 3.7517818212509155, + 3.7728620767593384, + 3.808842658996582, + 3.5965014696121216, + 3.8315606117248535, + 3.787741541862488, + 3.715541362762451, + 3.6901403665542603, + 3.855749487876892, + 3.6600046157836914, + 3.731532573699951, + 3.8553450107574463, + 3.7466262578964233, + 3.584746837615967, + 3.782003164291382, + 3.7781853675842285, + 3.731833338737488, + 3.8211803436279297, + 3.7875744104385376, + 3.718527913093567, + 3.6803473234176636, + 3.846898317337036, + 3.849267363548279, + 3.7493895292282104, + 3.748850703239441, + 3.5734708309173584, + 3.81915020942688, + 3.6570043563842773, + 3.8252413272857666, + 3.7298567295074463, + 3.825951933860779, + 3.6003445386886597, + 3.7837648391723633, + 3.8390307426452637, + 3.7356821298599243, + 3.7661439180374146, + 3.5376580953598022, + 3.606634736061096, + 3.74052894115448, + 3.7417808771133423, + 3.844759225845337, + 3.7106828689575195, + 3.6825402975082397, + 3.780295968055725, + 3.776592969894409, + 3.582952380180359, + 3.7746609449386597, + 3.6988964080810547, + 3.714148998260498, + 3.8614394664764404, + 3.7645630836486816, + 3.7568209171295166, + 3.7585268020629883, + 3.821237325668335, + 3.7285367250442505, + 3.8211270570755005, + 3.7748029232025146, + 3.7159345149993896, + 3.766482949256897, + 3.4718880653381348, + 3.566105842590332, + 3.6189171075820923, + 3.7674155235290527, + 3.6998953819274902, + 3.8205538988113403, + 3.839950203895569, + 3.7427475452423096, + 3.6931748390197754, + 3.717394471168518, + 3.6694371700286865, + 3.7258881330490112, + 3.70375394821167, + 3.6810871362686157, + 3.7392325401306152, + 3.7302430868148804, + 3.638694167137146, + 3.880134344100952, + 3.7964948415756226, + 3.700586438179016, + 3.698578119277954, + 3.787410855293274, + 3.7415425777435303, + 3.7623883485794067, + 3.7378190755844116, + 3.7622761726379395, + 3.672772765159607, + 3.6790558099746704, + 3.718265414237976, + 3.6467158794403076, + 3.76598060131073, + 3.70906925201416, + 3.7554917335510254, + 3.8009034395217896, + 3.65503990650177, + 3.6481188535690308, + 3.7440284490585327, + 3.7474228143692017, + 3.754581332206726, + 3.7511274814605713, + 3.6938949823379517, + 3.5973498821258545, + 3.6258991956710815, + 3.7489240169525146, + 3.770147681236267, + 3.7026960849761963, + 3.645857334136963, + 3.5669742822647095, + 3.6920758485794067, + 3.608549118041992, + 3.7258493900299072, + 3.6877758502960205, + 3.6421927213668823, + 3.623206853866577, + 3.622333288192749, + 3.735105872154236, + 3.7044215202331543, + 3.7632592916488647, + 3.601902723312378, + 3.6103804111480713, + 3.628949999809265, + 3.657949447631836, + 3.5787853002548218, + 3.619263768196106, + 3.619196057319641, + 3.5591427087783813, + 3.6985793113708496, + 3.5914849042892456, + 3.679292917251587, + 3.6171817779541016, + 3.689308762550354, + 3.7516367435455322, + 3.662622332572937, + 3.8400474786758423, + 3.57422411441803, + 3.6923645734786987, + 3.6485962867736816, + 3.742345094680786, + 3.6321663856506348, + 3.6280155181884766, + 3.63162624835968, + 3.490660309791565, + 3.5902198553085327, + 3.7200803756713867, + 3.7154622077941895, + 3.6096532344818115, + 3.683157205581665, + 3.606266498565674, + 3.6383583545684814, + 3.6010087728500366, + 3.772554874420166, + 3.695768356323242, + 3.5715770721435547, + 3.5506407022476196, + 3.7073649168014526, + 3.5969104766845703, + 3.622011184692383, + 3.548223614692688, + 3.7535594701766968, + 3.6731313467025757, + 3.6069027185440063, + 3.715794086456299, + 3.737573981285095, + 3.659486174583435, + 3.5691195726394653, + 3.5122050046920776, + 3.5970091819763184, + 3.5517642498016357, + 3.5606571435928345, + 3.7144354581832886, + 3.6190345287323, + 3.6469388008117676, + 3.6657259464263916, + 3.5995768308639526, + 3.6363049745559692, + 3.6160295009613037, + 3.707915782928467, + 3.647234082221985, + 3.6883015632629395, + 3.4939602613449097, + 3.595236301422119, + 3.665772557258606, + 3.5552912950515747, + 3.6983195543289185, + 3.6067663431167603, + 3.5698870420455933, + 3.4982383251190186, + 3.7413381338119507, + 3.600411534309387, + 3.7333885431289673, + 3.615244150161743, + 3.587606906890869, + 3.5589678287506104, + 3.650549054145813, + 3.659440279006958, + 3.5505512952804565, + 3.5929064750671387, + 3.6391491889953613, + 3.7528475522994995, + 3.5925893783569336, + 3.6371045112609863, + 3.573063373565674, + 3.4824236631393433, + 3.581717371940613, + 3.6580541133880615, + 3.748503804206848, + 3.6392905712127686, + 3.479975461959839, + 3.6605395078659058, + 3.630835175514221, + 3.582482099533081, + 3.6054930686950684, + 3.526723265647888, + 3.5969265699386597, + 3.608464002609253, + 3.6491113901138306, + 3.7100058794021606, + 3.568949341773987, + 3.699144959449768, + 3.6897313594818115, + 3.7431410551071167, + 3.7678658962249756, + 3.576589584350586, + 3.699389934539795, + 3.650769352912903, + 3.5900213718414307, + 3.456770420074463, + 3.6612515449523926, + 3.5739225149154663, + 3.53406822681427, + 3.4596993923187256, + 3.3667526245117188, + 3.561850070953369, + 3.591255307197571, + 3.5785330533981323, + 3.4658775329589844, + 3.420175552368164, + 3.5603288412094116, + 3.6342830657958984, + 3.6537452936172485, + 3.6189099550247192, + 3.3990739583969116, + 3.5436121225357056, + 3.6559964418411255, + 3.5397316217422485, + 3.6333106756210327, + 3.626753568649292, + 3.6256473064422607, + 3.4860684871673584, + 3.581238269805908, + 3.5821644067764282, + 3.457929849624634, + 3.4576581716537476, + 3.520195722579956, + 3.5145316123962402, + 3.543508291244507, + 3.5183730125427246, + 3.55321729183197, + 3.577145576477051, + 3.625890016555786, + 3.4224793910980225, + 3.587198257446289, + 3.4912800788879395, + 3.509684681892395, + 3.5882095098495483, + 3.5189086198806763, + 3.577429175376892, + 3.5047065019607544, + 3.475231647491455, + 3.4105498790740967, + 3.5606954097747803, + 3.6226420402526855, + 3.4354195594787598, + 3.6158764362335205, + 3.5995508432388306, + 3.5658655166625977, + 3.5295523405075073, + 3.5481185913085938, + 3.3801242113113403, + 3.4383524656295776, + 3.53507137298584, + 3.5401482582092285, + 3.469050168991089, + 3.4002010822296143, + 3.695646047592163, + 3.5295917987823486, + 3.4650168418884277, + 3.6170626878738403, + 3.451113224029541, + 3.602499485015869, + 3.5384289026260376, + 3.439741611480713, + 3.5718133449554443, + 3.4583628177642822, + 3.5482300519943237, + 3.4627450704574585, + 3.5247308015823364, + 3.515368700027466, + 3.440248727798462, + 3.503097891807556, + 3.5555227994918823, + 3.483245015144348, + 3.5626111030578613, + 3.462215542793274, + 3.589805841445923, + 3.5242233276367188, + 3.521987557411194, + 3.5249476432800293, + 3.533484697341919, + 3.52378511428833, + 3.554620146751404, + 3.5298632383346558, + 3.509568452835083, + 3.489190459251404, + 3.529168486595154, + 3.5390704870224, + 3.532436728477478, + 3.6428972482681274, + 3.4951311349868774, + 3.4662009477615356, + 3.63726270198822, + 3.491019606590271, + 3.529609441757202, + 3.6045902967453003, + 3.4632281064987183, + 3.669468641281128, + 3.45047128200531, + 3.5498753786087036, + 3.4888007640838623, + 3.5386061668395996, + 3.5916519165039062, + 3.5166256427764893, + 3.5567857027053833, + 3.532322883605957, + 3.624421715736389, + 3.5047614574432373, + 3.5623902082443237, + 3.591099262237549, + 3.500271797180176, + 3.4562876224517822, + 3.626054286956787, + 3.5194836854934692, + 3.483467936515808, + 3.533430576324463, + 3.547145128250122, + 3.58749783039093, + 3.6364723443984985, + 3.4808562994003296, + 3.5291569232940674, + 3.6306915283203125, + 3.5669487714767456, + 3.4823557138442993, + 3.515567183494568, + 3.431393027305603, + 3.4031453132629395, + 3.420678496360779, + 3.45008647441864, + 3.3197487592697144, + 3.4556018114089966, + 3.516103982925415, + 3.3763599395751953, + 3.5166393518447876, + 3.3770653009414673, + 3.36958646774292, + 3.4958831071853638, + 3.5175058841705322, + 3.5921815633773804, + 3.4676727056503296, + 3.369369864463806, + 3.5528823137283325, + 3.5888683795928955, + 3.449494957923889, + 3.469138264656067, + 3.449410319328308, + 3.4554715156555176, + 3.324034810066223, + 3.4414700269699097, + 3.459394931793213, + 3.4927862882614136, + 3.455292582511902, + 3.4065966606140137, + 3.4859843254089355, + 3.4229795932769775, + 3.519504427909851, + 3.5201969146728516, + 3.389004111289978, + 3.525880455970764, + 3.4344308376312256, + 3.4980862140655518, + 3.4432417154312134, + 3.5198651552200317, + 3.435137629508972, + 3.4136555194854736, + 3.554961085319519, + 3.4781473875045776, + 3.366340160369873, + 3.384047508239746, + 3.544527292251587, + 3.5615121126174927, + 3.401216983795166, + 3.474247455596924, + 3.4928823709487915, + 3.387145519256592, + 3.5415693521499634, + 3.32620370388031, + 3.380547523498535, + 3.4930039644241333, + 3.4492506980895996, + 3.50849187374115, + 3.4749902486801147, + 3.5170425176620483, + 3.5415838956832886, + 3.39301598072052, + 3.4009591341018677, + 3.3820379972457886, + 3.428192138671875, + 3.4221423864364624, + 3.5284204483032227, + 3.3735271692276, + 3.3255600929260254, + 3.3726691007614136, + 3.3341704607009888, + 3.399444580078125, + 3.4092236757278442, + 3.3981146812438965, + 3.598775267601013, + 3.39114773273468, + 3.46566903591156, + 3.273445248603821, + 3.3711997270584106, + 3.589006543159485, + 3.5733567476272583, + 3.4323160648345947, + 3.499995470046997, + 3.3784234523773193, + 3.431877851486206, + 3.511664390563965, + 3.4164836406707764, + 3.559172511100769, + 3.4621243476867676, + 3.453975558280945, + 3.644253134727478, + 3.4418227672576904, + 3.4405444860458374, + 3.628186821937561, + 3.4986188411712646, + 3.415285348892212, + 3.538922429084778, + 3.429821491241455, + 3.531388998031616, + 3.5214842557907104, + 3.5415258407592773, + 3.527928590774536, + 3.3906166553497314, + 3.4698173999786377, + 3.443128228187561, + 3.476876974105835, + 3.3626842498779297, + 3.4439234733581543, + 3.6157784461975098, + 3.3744819164276123, + 3.4137558937072754, + 3.3561023473739624, + 3.4869229793548584, + 3.4100441932678223, + 3.4264039993286133, + 3.5073708295822144, + 3.48713219165802, + 3.4447224140167236, + 3.4868016242980957, + 3.4367541074752808, + 3.517904758453369, + 3.4766459465026855, + 3.456013321876526, + 3.564266085624695, + 3.553146719932556, + 3.3931970596313477, + 3.3859384059906006, + 3.412820816040039, + 3.3655691146850586, + 3.4520225524902344, + 3.2980732917785645, + 3.438832402229309, + 3.351149559020996, + 3.4165380001068115, + 3.38878333568573, + 3.4600692987442017, + 3.527843475341797, + 3.3359874486923218, + 3.48003888130188, + 3.336642861366272, + 3.3811501264572144, + 3.4711817502975464, + 3.4105899333953857, + 3.311923623085022, + 3.392702102661133, + 3.3507156372070312, + 3.3898125886917114, + 3.5262973308563232, + 3.3763598203659058, + 3.359613537788391, + 3.3782798051834106, + 3.4436172246932983, + 3.2634856700897217, + 3.3369256258010864, + 3.4006670713424683, + 3.5494710206985474, + 3.448782205581665, + 3.3294544219970703, + 3.3848321437835693, + 3.4473503828048706, + 3.383387565612793, + 3.4185373783111572, + 3.327703356742859, + 3.4286147356033325, + 3.3342596292495728, + 3.4424190521240234, + 3.477192521095276, + 3.3806605339050293, + 3.4120019674301147, + 3.3259891271591187, + 3.403128743171692, + 3.488956332206726, + 3.359277367591858, + 3.5235272645950317, + 3.5015339851379395, + 3.4845632314682007, + 3.4257967472076416, + 3.26141095161438, + 3.408102035522461, + 3.4456984996795654, + 3.393828868865967, + 3.493906021118164, + 3.474936008453369, + 3.4844125509262085, + 3.5114723443984985, + 3.379406690597534, + 3.412527322769165, + 3.4505211114883423, + 3.4206827878952026, + 3.480284333229065, + 3.328805088996887, + 3.280402183532715, + 3.528708577156067, + 3.372161388397217, + 3.4081509113311768, + 3.2642000913619995, + 3.4033979177474976, + 3.4453628063201904, + 3.390933036804199, + 3.372330665588379, + 3.514410972595215, + 3.326097249984741, + 3.392948627471924, + 3.283918023109436, + 3.4678953886032104, + 3.389270782470703, + 3.4790730476379395, + 3.4638967514038086, + 3.3626328706741333, + 3.423244595527649, + 3.3583770990371704, + 3.396082878112793, + 3.469586491584778, + 3.379392147064209, + 3.3381012678146362, + 3.4549715518951416, + 3.52035129070282, + 3.4890472888946533, + 3.4535155296325684, + 3.392438292503357, + 3.4428740739822388, + 3.3357051610946655, + 3.338693141937256, + 3.439629077911377, + 3.406848907470703, + 3.452675938606262, + 3.4665647745132446, + 3.409509062767029, + 3.497183918952942, + 3.3888643980026245, + 3.491596817970276, + 3.3578566312789917, + 3.317729353904724, + 3.3319952487945557, + 3.3035300970077515, + 3.3618953227996826, + 3.3559364080429077, + 3.395157814025879, + 3.4395357370376587, + 3.364956021308899, + 3.4091702699661255, + 3.5058470964431763, + 3.4085967540740967 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 128.2335926867057, + 79.42060381927247, + 63.70529267513387, + 55.5789015183509, + 50.81495177293574, + 46.91510201904133, + 44.00050455539274, + 42.18729871799023, + 40.14993832822502, + 38.37996234746526, + 37.19369351197014, + 36.01895510865533, + 35.21922606568627, + 34.22140438395747, + 33.43078295403124, + 33.154977956661504, + 32.69678045164968, + 32.45241159570735, + 32.35855937143936, + 32.33161137307153 + ], + "best_ppl": 32.33161137307153, + "train_time_s": 21375.94007253647 +} \ No newline at end of file diff --git a/063_sparse_dispatch/sparse_3of5_s256_best.pt b/063_sparse_dispatch/sparse_3of5_s256_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..02fba7b3ea836d1cfd7b5e8854f9c4e492a6303d --- /dev/null +++ b/063_sparse_dispatch/sparse_3of5_s256_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3343ba18d94636439dda661d50ec3dd9827bc0ae86ec516765c0aa00696e1b5b +size 341440023 diff --git a/063_sparse_dispatch/sparse_3of5_s256_results.json b/063_sparse_dispatch/sparse_3of5_s256_results.json new file mode 100644 index 0000000000000000000000000000000000000000..5cf9bb8e750b933449aacd429b08132c4448b87b --- /dev/null +++ b/063_sparse_dispatch/sparse_3of5_s256_results.json @@ -0,0 +1,2055 @@ +{ + "label": "sparse_3of5_s256", + "seed": 256, + "total_params": 85301248, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_loss": [ + 18.96757221221924, + 14.866062641143799, + 8.023278713226318, + 7.289057493209839, + 7.16545295715332, + 7.018719673156738, + 6.892342805862427, + 6.727680206298828, + 6.661981582641602, + 6.602681398391724, + 6.571227788925171, + 6.510935068130493, + 6.496942520141602, + 6.3190648555755615, + 6.152647972106934, + 6.084249258041382, + 5.986941337585449, + 5.929451942443848, + 5.947634935379028, + 5.780519247055054, + 5.775683164596558, + 5.782166957855225, + 5.789921283721924, + 5.6776018142700195, + 5.615113019943237, + 5.580887079238892, + 5.491368532180786, + 5.439431667327881, + 5.505125522613525, + 5.346136093139648, + 5.296799659729004, + 5.292404413223267, + 5.235638856887817, + 5.3570709228515625, + 5.386775732040405, + 5.2007317543029785, + 5.200268268585205, + 5.0871500968933105, + 5.142003536224365, + 5.170969486236572, + 5.05439305305481, + 5.152141571044922, + 5.0219080448150635, + 5.1390838623046875, + 4.959076166152954, + 5.012432098388672, + 5.048535108566284, + 4.965134859085083, + 4.951212167739868, + 4.958349704742432, + 4.967214822769165, + 5.036473989486694, + 4.867083787918091, + 4.864797353744507, + 4.7333433628082275, + 4.800087928771973, + 4.8621625900268555, + 4.833041667938232, + 4.868873596191406, + 4.8545167446136475, + 4.765375137329102, + 4.793900012969971, + 4.746840715408325, + 4.729151010513306, + 4.786364793777466, + 4.668799877166748, + 4.660365581512451, + 4.904048681259155, + 4.812768459320068, + 4.662990570068359, + 4.789960145950317, + 4.791857481002808, + 4.666048765182495, + 4.74301290512085, + 4.727251291275024, + 4.700099229812622, + 4.764397621154785, + 4.646493673324585, + 4.6758763790130615, + 4.644198179244995, + 4.536417007446289, + 4.71555495262146, + 4.678134918212891, + 4.54308557510376, + 4.713144540786743, + 4.598839521408081, + 4.559847354888916, + 4.6527533531188965, + 4.517800807952881, + 4.508612394332886, + 4.510418176651001, + 4.545610189437866, + 4.506681442260742, + 4.545450925827026, + 4.50616192817688, + 4.423875570297241, + 4.597181797027588, + 4.451627969741821, + 4.595016956329346, + 4.528415679931641, + 4.484739065170288, + 4.534145832061768, + 4.591626167297363, + 4.560283184051514, + 4.575031757354736, + 4.256590127944946, + 4.390413999557495, + 4.358781814575195, + 4.404587030410767, + 4.270052194595337, + 4.440877676010132, + 4.397117853164673, + 4.421557664871216, + 4.4883034229278564, + 4.54542088508606, + 4.312624216079712, + 4.432281494140625, + 4.491764783859253, + 4.437327146530151, + 4.505328416824341, + 4.363782644271851, + 4.4646453857421875, + 4.367968320846558, + 4.304424524307251, + 4.391773462295532, + 4.303222179412842, + 4.201856374740601, + 4.257136344909668, + 4.227877140045166, + 4.334389686584473, + 4.167869806289673, + 4.35490083694458, + 4.234187602996826, + 4.26154899597168, + 4.275919198989868, + 4.199981451034546, + 4.252429723739624, + 4.281959533691406, + 4.359633922576904, + 4.304104566574097, + 4.442125558853149, + 4.204060316085815, + 4.177164077758789, + 4.328666925430298, + 4.129993677139282, + 4.250751733779907, + 4.289444446563721, + 4.252699613571167, + 4.069411039352417, + 4.199191570281982, + 4.114404201507568, + 4.331377029418945, + 4.262859106063843, + 4.203344345092773, + 4.203063488006592, + 4.230396032333374, + 4.2537102699279785, + 4.128608465194702, + 4.1671483516693115, + 4.3149895668029785, + 4.259250164031982, + 4.226618528366089, + 4.216224431991577, + 4.063168525695801, + 4.1391990184783936, + 4.195093631744385, + 4.31343412399292, + 4.2687153816223145, + 4.15732741355896, + 4.194591999053955, + 4.222882032394409, + 4.188368082046509, + 4.180107593536377, + 4.21323823928833, + 4.1058030128479, + 3.987883687019348, + 4.277125835418701, + 4.120357275009155, + 4.184706687927246, + 4.0535818338394165, + 4.088764905929565, + 4.134809494018555, + 4.13041353225708, + 4.162085294723511, + 4.087804794311523, + 4.136915326118469, + 4.1399877071380615, + 4.178204536437988, + 4.0869752168655396, + 4.041394948959351, + 4.073372840881348, + 4.093244910240173, + 4.260671377182007, + 4.2016401290893555, + 4.058653116226196, + 4.271692514419556, + 4.183025598526001, + 4.215696811676025, + 4.080572009086609, + 4.177374362945557, + 4.183256149291992, + 4.0822107791900635, + 4.106240510940552, + 4.1275084018707275, + 4.024310827255249, + 4.0303795337677, + 4.218252182006836, + 4.056309819221497, + 4.192304611206055, + 4.209555387496948, + 4.125459671020508, + 4.16579008102417, + 4.124940633773804, + 4.068594336509705, + 4.000650882720947, + 4.000454545021057, + 4.044957399368286, + 4.1409993171691895, + 3.9981298446655273, + 4.051731586456299, + 4.1507568359375, + 4.066955804824829, + 4.218930006027222, + 4.1443634033203125, + 4.146371841430664, + 4.0705366134643555, + 4.029305458068848, + 4.177032470703125, + 4.076725482940674, + 3.995905876159668, + 4.035491943359375, + 4.137933969497681, + 4.0932700634002686, + 4.116507053375244, + 4.110224962234497, + 4.144790172576904, + 4.068505048751831, + 4.018653512001038, + 4.162392616271973, + 4.045996189117432, + 3.9918333292007446, + 4.012971520423889, + 4.034575700759888, + 3.998724102973938, + 4.08431077003479, + 3.9050865173339844, + 4.080397844314575, + 4.186564683914185, + 4.065509080886841, + 4.059859991073608, + 3.97470486164093, + 4.065500736236572, + 4.008653402328491, + 3.832537889480591, + 3.9953452348709106, + 3.947851300239563, + 4.025566101074219, + 3.9619176387786865, + 3.86188280582428, + 3.858560562133789, + 4.008169054985046, + 3.841923236846924, + 3.8304667472839355, + 3.833884358406067, + 3.899047374725342, + 3.9411121606826782, + 4.01047420501709, + 3.9705511331558228, + 3.8729798793792725, + 3.884615898132324, + 3.95478892326355, + 4.057178258895874, + 3.906615734100342, + 4.0599424839019775, + 3.938186764717102, + 3.9546611309051514, + 3.9483031034469604, + 3.943638324737549, + 3.8839691877365112, + 3.949348211288452, + 3.942635178565979, + 3.8730441331863403, + 3.828065276145935, + 3.957249402999878, + 3.8171485662460327, + 4.028413414955139, + 3.935922622680664, + 3.849112868309021, + 3.9581456184387207, + 3.9934797286987305, + 3.9201819896698, + 4.000360608100891, + 3.975339651107788, + 3.9388539791107178, + 3.9148828983306885, + 3.9042023420333862, + 3.8795770406723022, + 3.7740097045898438, + 3.897286891937256, + 3.8627612590789795, + 3.941628932952881, + 3.9382978677749634, + 3.7954005002975464, + 3.9892596006393433, + 3.959970712661743, + 3.8992998600006104, + 3.8949618339538574, + 3.9654734134674072, + 3.9520918130874634, + 3.9528225660324097, + 3.8690556287765503, + 3.932777166366577, + 3.9197468757629395, + 3.8131717443466187, + 3.815306544303894, + 3.952665090560913, + 3.957404851913452, + 3.9525967836380005, + 3.8974260091781616, + 3.864215850830078, + 3.854920744895935, + 3.917430281639099, + 3.96515691280365, + 3.9417134523391724, + 3.8751174211502075, + 3.869876742362976, + 3.9591245651245117, + 3.865722894668579, + 3.929618716239929, + 3.872953772544861, + 3.943742513656616, + 3.814091920852661, + 3.8001065254211426, + 3.91280734539032, + 3.9066379070281982, + 3.737547516822815, + 4.01117467880249, + 4.07016134262085, + 3.7999273538589478, + 3.91266131401062, + 4.017721891403198, + 3.9375252723693848, + 3.9192328453063965, + 3.776886820793152, + 3.85310959815979, + 3.9109643697738647, + 3.9144372940063477, + 3.8969881534576416, + 3.7484806776046753, + 3.7904484272003174, + 4.018397808074951, + 3.973321318626404, + 3.9044865369796753, + 3.901101231575012, + 3.840906500816345, + 3.8566203117370605, + 3.909640073776245, + 3.920629620552063, + 3.9242974519729614, + 3.8625338077545166, + 3.83441698551178, + 3.8541804552078247, + 3.983644962310791, + 3.8087137937545776, + 3.8699785470962524, + 3.7196033000946045, + 3.9143954515457153, + 3.8852710723876953, + 3.7433929443359375, + 3.936647891998291, + 3.8536105155944824, + 3.884434461593628, + 3.983691096305847, + 3.748361587524414, + 3.884054183959961, + 4.0055171251297, + 3.8943504095077515, + 3.8665194511413574, + 3.8722333908081055, + 3.7968863248825073, + 3.7082464694976807, + 3.7918272018432617, + 3.824888229370117, + 3.834924340248108, + 3.7033324241638184, + 3.7647693157196045, + 3.845695972442627, + 3.804526209831238, + 3.7119550704956055, + 3.7522029876708984, + 3.6884859800338745, + 3.7130091190338135, + 3.685906410217285, + 3.6711270809173584, + 3.7884304523468018, + 3.7829465866088867, + 3.8094745874404907, + 3.810738682746887, + 3.8442084789276123, + 3.8257029056549072, + 3.8190109729766846, + 3.8975465297698975, + 3.9180082082748413, + 3.715232014656067, + 3.875031352043152, + 3.7699116468429565, + 3.921947956085205, + 3.805461049079895, + 3.697029709815979, + 3.8426713943481445, + 3.7302924394607544, + 3.716885566711426, + 3.6828761100769043, + 3.7899208068847656, + 3.8224717378616333, + 3.638226866722107, + 3.8377474546432495, + 3.656056761741638, + 3.7557305097579956, + 3.7976841926574707, + 3.71855890750885, + 3.7407180070877075, + 3.7372392416000366, + 3.7763458490371704, + 3.7430964708328247, + 3.847578763961792, + 3.7415119409561157, + 3.788082242012024, + 3.8822494745254517, + 3.769238233566284, + 3.7925037145614624, + 3.7286112308502197, + 3.576959252357483, + 3.855328679084778, + 3.8367602825164795, + 3.7006794214248657, + 3.6625633239746094, + 3.7820827960968018, + 3.72404146194458, + 3.718287467956543, + 3.755165934562683, + 3.6978728771209717, + 3.7851191759109497, + 3.7381370067596436, + 3.6446664333343506, + 3.7845464944839478, + 3.7374850511550903, + 3.7703124284744263, + 3.8052871227264404, + 3.730509638786316, + 3.7099456787109375, + 3.8878461122512817, + 3.7686687707901, + 3.78555965423584, + 3.8472073078155518, + 3.87832772731781, + 3.831340193748474, + 3.757484555244446, + 3.704502582550049, + 3.679394245147705, + 3.6866912841796875, + 3.7147539854049683, + 3.777335047721863, + 3.700089454650879, + 3.6884491443634033, + 3.7253133058547974, + 3.69391131401062, + 3.731849431991577, + 3.6218347549438477, + 3.7126792669296265, + 3.6978373527526855, + 3.7111154794692993, + 3.671466112136841, + 3.7953308820724487, + 3.826115846633911, + 3.7962746620178223, + 3.6246482133865356, + 3.7611262798309326, + 3.7370975017547607, + 3.7884111404418945, + 3.696696400642395, + 3.788729190826416, + 3.8300724029541016, + 3.7695276737213135, + 3.8292737007141113, + 3.8027989864349365, + 3.760223627090454, + 3.8012828826904297, + 3.782537341117859, + 3.838912606239319, + 3.6482300758361816, + 3.732579827308655, + 3.838380455970764, + 3.657287359237671, + 3.80861234664917, + 3.7687220573425293, + 3.7700787782669067, + 3.698712110519409, + 3.712417721748352, + 3.7416739463806152, + 3.6237672567367554, + 3.5407625436782837, + 3.8341785669326782, + 3.7907410860061646, + 3.7586699724197388, + 3.7045074701309204, + 3.6106306314468384, + 3.599745512008667, + 3.622090458869934, + 3.559893488883972, + 3.565439462661743, + 3.8171719312667847, + 3.760372281074524, + 3.6507679224014282, + 3.6007362604141235, + 3.6169629096984863, + 3.633179545402527, + 3.626110553741455, + 3.664971351623535, + 3.72231924533844, + 3.6344579458236694, + 3.607224702835083, + 3.573524594306946, + 3.5040677785873413, + 3.734881281852722, + 3.684319019317627, + 3.6988390684127808, + 3.584782838821411, + 3.669144868850708, + 3.584994673728943, + 3.7801473140716553, + 3.6567212343215942, + 3.518098473548889, + 3.574553370475769, + 3.7048970460891724, + 3.6988658905029297, + 3.5856337547302246, + 3.745447874069214, + 3.637576699256897, + 3.7164483070373535, + 3.658980131149292, + 3.7083884477615356, + 3.668622612953186, + 3.640216827392578, + 3.5598814487457275, + 3.7241361141204834, + 3.5488394498825073, + 3.5315113067626953, + 3.6532387733459473, + 3.6785924434661865, + 3.6235638856887817, + 3.5802868604660034, + 3.613093852996826, + 3.5223923921585083, + 3.6593403816223145, + 3.789300799369812, + 3.6683613061904907, + 3.644723653793335, + 3.6980713605880737, + 3.710469365119934, + 3.6911808252334595, + 3.663452982902527, + 3.709341883659363, + 3.531451106071472, + 3.611476182937622, + 3.691733479499817, + 3.5605627298355103, + 3.584628939628601, + 3.558501124382019, + 3.515659213066101, + 3.599104642868042, + 3.5951536893844604, + 3.6802510023117065, + 3.7223860025405884, + 3.725156784057617, + 3.8280926942825317, + 3.6063971519470215, + 3.6009594202041626, + 3.6788002252578735, + 3.7308356761932373, + 3.6422958374023438, + 3.708225965499878, + 3.570056438446045, + 3.600681781768799, + 3.5928484201431274, + 3.756663680076599, + 3.7156357765197754, + 3.736914873123169, + 3.610836625099182, + 3.731231451034546, + 3.6221988201141357, + 3.5841691493988037, + 3.515222668647766, + 3.5934410095214844, + 3.6504100561141968, + 3.635503888130188, + 3.60004198551178, + 3.725222110748291, + 3.6645249128341675, + 3.680031418800354, + 3.700846552848816, + 3.5173643827438354, + 3.5735867023468018, + 3.631160616874695, + 3.5574748516082764, + 3.7573342323303223, + 3.707083225250244, + 3.521619439125061, + 3.593973398208618, + 3.5338668823242188, + 3.6918540000915527, + 3.648629665374756, + 3.721274971961975, + 3.6293673515319824, + 3.5644651651382446, + 3.5704997777938843, + 3.4923315048217773, + 3.7078146934509277, + 3.6075724363327026, + 3.4991215467453003, + 3.533569574356079, + 3.525549054145813, + 3.718942880630493, + 3.6319000720977783, + 3.6463783979415894, + 3.6904196739196777, + 3.60663104057312, + 3.618321418762207, + 3.6570613384246826, + 3.540944457054138, + 3.6538716554641724, + 3.478798270225525, + 3.5136798620224, + 3.47834849357605, + 3.5886062383651733, + 3.505182385444641, + 3.527925491333008, + 3.5327125787734985, + 3.4605482816696167, + 3.5344303846359253, + 3.518678307533264, + 3.4839279651641846, + 3.482002854347229, + 3.5369865894317627, + 3.509494185447693, + 3.457978129386902, + 3.444406509399414, + 3.552767753601074, + 3.63609778881073, + 3.6379834413528442, + 3.6444501876831055, + 3.5792442560195923, + 3.5232127904891968, + 3.4874058961868286, + 3.622650146484375, + 3.704205632209778, + 3.543170213699341, + 3.4363831281661987, + 3.628959059715271, + 3.5430644750595093, + 3.5355323553085327, + 3.564101219177246, + 3.5607279539108276, + 3.6354700326919556, + 3.5230079889297485, + 3.5809102058410645, + 3.564030647277832, + 3.586625099182129, + 3.431898593902588, + 3.46332049369812, + 3.5389552116394043, + 3.443215847015381, + 3.526546597480774, + 3.5009021759033203, + 3.6540054082870483, + 3.5583980083465576, + 3.5388994216918945, + 3.477035164833069, + 3.6199817657470703, + 3.532638192176819, + 3.4254482984542847, + 3.530863046646118, + 3.5363235473632812, + 3.5654892921447754, + 3.6051061153411865, + 3.580910325050354, + 3.6704814434051514, + 3.527912974357605, + 3.6234028339385986, + 3.4950222969055176, + 3.5664947032928467, + 3.57356333732605, + 3.546582579612732, + 3.384142279624939, + 3.555962920188904, + 3.50609290599823, + 3.6477901935577393, + 3.3714815378189087, + 3.5281296968460083, + 3.4909862279891968, + 3.535506248474121, + 3.5459076166152954, + 3.517855167388916, + 3.5202300548553467, + 3.4877519607543945, + 3.2962775230407715, + 3.5546082258224487, + 3.5150420665740967, + 3.4699188470840454, + 3.480222702026367, + 3.4860321283340454, + 3.599417209625244, + 3.58327579498291, + 3.5280364751815796, + 3.578550934791565, + 3.5362309217453003, + 3.458209991455078, + 3.5936354398727417, + 3.4296741485595703, + 3.464624524116516, + 3.422774314880371, + 3.630315661430359, + 3.405959129333496, + 3.585939049720764, + 3.4432003498077393, + 3.6662850379943848, + 3.5886462926864624, + 3.5278559923171997, + 3.518200159072876, + 3.442930221557617, + 3.4411401748657227, + 3.5572025775909424, + 3.4782848358154297, + 3.5176596641540527, + 3.5267674922943115, + 3.4528757333755493, + 3.547852873802185, + 3.553868293762207, + 3.51798939704895, + 3.4764050245285034, + 3.527635097503662, + 3.6206018924713135, + 3.655088186264038, + 3.595159888267517, + 3.499835729598999, + 3.5350778102874756, + 3.487264037132263, + 3.536501407623291, + 3.5248756408691406, + 3.5600684881210327, + 3.4253937005996704, + 3.516788363456726, + 3.5633909702301025, + 3.5836448669433594, + 3.6483274698257446, + 3.564439058303833, + 3.519673705101013, + 3.5426385402679443, + 3.556152105331421, + 3.4260836839675903, + 3.4730554819107056, + 3.4827252626419067, + 3.508120059967041, + 3.46681010723114, + 3.5270947217941284, + 3.5053396224975586, + 3.534074544906616, + 3.463060140609741, + 3.4843993186950684, + 3.5514742136001587, + 3.221350312232971, + 3.4136451482772827, + 3.434535503387451, + 3.482162594795227, + 3.463615655899048, + 3.487298369407654, + 3.3610564470291138, + 3.518554925918579, + 3.522691011428833, + 3.384162187576294, + 3.3215585947036743, + 3.580691695213318, + 3.4598793983459473, + 3.493628740310669, + 3.44267201423645, + 3.52397882938385, + 3.468659520149231, + 3.4901134967803955, + 3.5113162994384766, + 3.4657766819000244, + 3.4358896017074585, + 3.5271384716033936, + 3.4355708360671997, + 3.4230867624282837, + 3.359931707382202, + 3.417426824569702, + 3.4726816415786743, + 3.5085254907608032, + 3.5590230226516724, + 3.4018256664276123, + 3.4560022354125977, + 3.4984620809555054, + 3.367214322090149, + 3.464938521385193, + 3.415286898612976, + 3.5415118932724, + 3.409858226776123, + 3.3895208835601807, + 3.3730095624923706, + 3.5056298971176147, + 3.4108976125717163, + 3.4025139808654785, + 3.5034568309783936, + 3.4684518575668335, + 3.4310837984085083, + 3.656331419944763, + 3.5004228353500366, + 3.5844582319259644, + 3.591496706008911, + 3.4370678663253784, + 3.4088621139526367, + 3.4543603658676147, + 3.5614888668060303, + 3.4242212772369385, + 3.3290234804153442, + 3.395100951194763, + 3.4428519010543823, + 3.524747848510742, + 3.4569493532180786, + 3.4051185846328735, + 3.6341484785079956, + 3.3350071907043457, + 3.4012473821640015, + 3.5622050762176514, + 3.2792431116104126, + 3.433184027671814, + 3.465856432914734, + 3.4799692630767822, + 3.510420083999634, + 3.4078774452209473, + 3.507952928543091, + 3.246768593788147, + 3.524749279022217, + 3.4189621210098267, + 3.463918089866638, + 3.5440778732299805, + 3.4549970626831055, + 3.4479068517684937, + 3.465419054031372, + 3.560740113258362, + 3.416944742202759, + 3.449491262435913, + 3.4561089277267456, + 3.536068558692932, + 3.4047372341156006, + 3.4070481061935425, + 3.458114504814148, + 3.4260005950927734, + 3.3527395725250244, + 3.3771088123321533, + 3.38979434967041, + 3.479742169380188, + 3.447990655899048, + 3.4970483779907227, + 3.508475184440613, + 3.4494577646255493, + 3.554807186126709, + 3.5263750553131104, + 3.4257789850234985, + 3.4858975410461426, + 3.4617584943771362, + 3.5720934867858887, + 3.5912153720855713, + 3.4924992322921753, + 3.4762288331985474, + 3.42130446434021, + 3.4782477617263794, + 3.470256209373474, + 3.3607168197631836, + 3.5118342638015747, + 3.6074819564819336, + 3.4189263582229614, + 3.547611117362976, + 3.442160129547119, + 3.4076117277145386, + 3.4049848318099976, + 3.4101881980895996, + 3.418213963508606, + 3.584790825843811, + 3.4963475465774536, + 3.3289874792099, + 3.4086958169937134, + 3.545151948928833, + 3.424457311630249, + 3.4169442653656006, + 3.4563348293304443, + 3.396904706954956, + 3.3458168506622314, + 3.464726209640503, + 3.4412230253219604, + 3.317961096763611, + 3.4955097436904907, + 3.32764995098114, + 3.365397095680237, + 3.308161973953247, + 3.534562587738037, + 3.362737774848938, + 3.408885359764099, + 3.4081913232803345, + 3.3526079654693604, + 3.4101040363311768, + 3.4220200777053833, + 3.4656519889831543, + 3.2630265951156616, + 3.383347511291504, + 3.447035789489746, + 3.3173447847366333, + 3.553940773010254, + 3.4604300260543823, + 3.39722216129303, + 3.379417300224304, + 3.386869192123413, + 3.3560891151428223, + 3.4042574167251587, + 3.5141139030456543, + 3.39306104183197, + 3.412111759185791, + 3.4796252250671387, + 3.505661129951477, + 3.4584097862243652, + 3.4846765995025635, + 3.3945653438568115, + 3.4564119577407837, + 3.458404064178467, + 3.438709020614624, + 3.3847217559814453, + 3.44765043258667, + 3.3958399295806885, + 3.3545440435409546, + 3.393144369125366, + 3.350591540336609, + 3.354534149169922, + 3.270748257637024, + 3.3665117025375366, + 3.3588061332702637, + 3.426237463951111, + 3.4693682193756104, + 3.473602056503296, + 3.458424925804138, + 3.352143406867981, + 3.528066396713257, + 3.522595167160034, + 3.4441555738449097, + 3.442387104034424, + 3.4009580612182617, + 3.4855222702026367, + 3.3990564346313477, + 3.3647927045822144, + 3.348838210105896, + 3.468177080154419, + 3.412871479988098, + 3.4478440284729004, + 3.3657326698303223, + 3.4987446069717407, + 3.376151442527771, + 3.4949240684509277, + 3.454748034477234, + 3.3848360776901245, + 3.3317145109176636, + 3.4542301893234253, + 3.4683843851089478, + 3.3828327655792236, + 3.367355227470398, + 3.4076881408691406, + 3.4281280040740967, + 3.4189329147338867, + 3.4166879653930664, + 3.472869873046875, + 3.333974003791809, + 3.338601231575012, + 3.301185965538025, + 3.3612987995147705, + 3.4736523628234863, + 3.2888697385787964, + 3.3461060523986816, + 3.504823684692383, + 3.2174121141433716, + 3.4556740522384644, + 3.399892210960388, + 3.2830255031585693, + 3.3128055334091187, + 3.442262649536133, + 3.4422030448913574, + 3.491167664527893, + 3.346243381500244, + 3.3406338691711426, + 3.458879590034485, + 3.296101927757263, + 3.5168946981430054, + 3.4488720893859863 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_ppl": [ + 127.45276492023203, + 78.516009661457, + 62.87498504665735, + 55.06345969379717, + 49.912655362818384, + 46.939568865040044, + 43.81203382978657, + 41.693778292981165, + 39.87562167680392, + 38.55126324062145, + 36.979597930599326, + 35.93675831789996, + 35.01756183181574, + 34.195932648762316, + 33.29748651933155, + 33.009698481400164, + 32.50772189729562, + 32.343121576499, + 32.23485827297993, + 32.20878745721878 + ], + "best_ppl": 32.20878745721878, + "train_time_s": 19305.9337952137 +} \ No newline at end of file diff --git a/074_dthin_ablation/fat_k2_d512_best.pt b/074_dthin_ablation/fat_k2_d512_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa953739a111113158d171059e9f4edd82cc84b3 --- /dev/null +++ b/074_dthin_ablation/fat_k2_d512_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:107b10ada81eca0ba05617994a7ca276c3caf45f12967981e55f0aeca6416d6e +size 340236450 diff --git a/074_dthin_ablation/fat_k2_d512_results.json b/074_dthin_ablation/fat_k2_d512_results.json new file mode 100644 index 0000000000000000000000000000000000000000..0964a63a214e873c4f9c1512308f36e490203cb9 --- /dev/null +++ b/074_dthin_ablation/fat_k2_d512_results.json @@ -0,0 +1,2075 @@ +{ + "label": "fat_k2_d512", + "condition": "fat_k2_d512", + "total_params": 85035008, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_lm_loss": [ + 17.593371391296387, + 8.615891456604004, + 7.670632839202881, + 7.378455877304077, + 7.262079954147339, + 7.027265429496765, + 6.905206799507141, + 6.777941942214966, + 6.679468393325806, + 6.432583928108215, + 6.417825698852539, + 6.224363207817078, + 6.166165828704834, + 6.084959983825684, + 6.104968309402466, + 5.935504794120789, + 5.788340926170349, + 5.786000370979309, + 5.744660377502441, + 5.625619292259216, + 5.631098508834839, + 5.435988783836365, + 5.5375816822052, + 5.648394465446472, + 5.316158294677734, + 5.37037456035614, + 5.290357947349548, + 5.400851845741272, + 5.2592445611953735, + 5.210706472396851, + 5.330143570899963, + 5.102874159812927, + 5.256452798843384, + 5.106019377708435, + 5.132410168647766, + 5.091224431991577, + 4.997986435890198, + 4.943429946899414, + 5.078044295310974, + 4.938769221305847, + 5.075108170509338, + 5.074126720428467, + 4.921675443649292, + 4.9570828676223755, + 5.027823805809021, + 4.834596157073975, + 4.950482606887817, + 4.859358429908752, + 4.881774663925171, + 5.004549741744995, + 4.972113013267517, + 4.852477550506592, + 4.847666025161743, + 4.889843702316284, + 4.577494978904724, + 4.634534478187561, + 4.759918570518494, + 4.780282020568848, + 4.739808678627014, + 4.667106628417969, + 4.692970037460327, + 4.600942134857178, + 4.676294803619385, + 4.725580453872681, + 4.738227725028992, + 4.651233434677124, + 4.552926301956177, + 4.640767931938171, + 4.61095654964447, + 4.482519030570984, + 4.65333354473114, + 4.55556321144104, + 4.592855334281921, + 4.57268500328064, + 4.65160596370697, + 4.528197884559631, + 4.5965224504470825, + 4.621302008628845, + 4.456494331359863, + 4.583864331245422, + 4.57192850112915, + 4.513291954994202, + 4.674343585968018, + 4.52767276763916, + 4.588857412338257, + 4.41424024105072, + 4.562580108642578, + 4.527196645736694, + 4.491539716720581, + 4.529844522476196, + 4.428650379180908, + 4.467119574546814, + 4.524372100830078, + 4.4349061250686646, + 4.404075264930725, + 4.434785962104797, + 4.459642171859741, + 4.317628026008606, + 4.4034037590026855, + 4.383883595466614, + 4.415251731872559, + 4.325474143028259, + 4.51414155960083, + 4.390450477600098, + 4.508469343185425, + 4.490736603736877, + 4.302497744560242, + 4.352693200111389, + 4.379652380943298, + 4.534972667694092, + 4.512875914573669, + 4.486465573310852, + 4.518734335899353, + 4.450393438339233, + 4.291369080543518, + 4.463902235031128, + 4.386718034744263, + 4.3045278787612915, + 4.395426273345947, + 4.321804761886597, + 4.351145386695862, + 4.359211802482605, + 4.227985620498657, + 4.385600805282593, + 4.2978129386901855, + 4.202650308609009, + 4.166291773319244, + 4.24677300453186, + 4.218661546707153, + 4.2362858057022095, + 4.2090500593185425, + 4.319211483001709, + 4.311056971549988, + 4.2114856243133545, + 4.198856174945831, + 4.276096224784851, + 4.34166419506073, + 4.277421414852142, + 4.295168042182922, + 4.289906024932861, + 4.17293643951416, + 4.293866515159607, + 4.2216421365737915, + 4.174103736877441, + 4.154576063156128, + 4.088581800460815, + 4.208061993122101, + 4.137741148471832, + 4.284934639930725, + 4.3338276743888855, + 4.1996846199035645, + 4.199255466461182, + 4.134474277496338, + 4.209473252296448, + 4.194827914237976, + 4.251093864440918, + 4.237766861915588, + 4.156221151351929, + 4.230478644371033, + 4.153433382511139, + 4.195586681365967, + 4.137466907501221, + 4.1660661697387695, + 4.215587496757507, + 4.066795706748962, + 4.178393840789795, + 4.176999568939209, + 4.108425676822662, + 4.155689597129822, + 4.093782842159271, + 4.163833379745483, + 4.067497432231903, + 4.1733139753341675, + 4.2714091539382935, + 4.162054896354675, + 4.055314123630524, + 4.104001045227051, + 4.044635534286499, + 4.1910412311553955, + 4.164646029472351, + 4.190171837806702, + 4.145239233970642, + 4.123906254768372, + 4.101635456085205, + 4.075185179710388, + 4.191258668899536, + 4.158064007759094, + 4.119181752204895, + 4.262955665588379, + 3.9791985750198364, + 4.13303142786026, + 4.110267281532288, + 4.140314340591431, + 4.021811068058014, + 4.088198661804199, + 4.167405962944031, + 4.083914160728455, + 4.115623295307159, + 4.14752721786499, + 4.082651913166046, + 4.1134015917778015, + 4.160603165626526, + 4.065694272518158, + 4.119297802448273, + 3.9940837621688843, + 4.113945484161377, + 3.9379436373710632, + 4.1219398975372314, + 4.06059718132019, + 4.056586742401123, + 4.077334702014923, + 4.168805122375488, + 3.968947410583496, + 4.098278760910034, + 4.156968772411346, + 4.155141830444336, + 4.204598546028137, + 4.030603408813477, + 3.9762353897094727, + 4.116004645824432, + 4.049933671951294, + 4.10353547334671, + 4.147756218910217, + 4.284484028816223, + 4.0399728417396545, + 4.111849308013916, + 4.103189647197723, + 4.1084036231040955, + 4.09222674369812, + 4.159319281578064, + 4.000624299049377, + 4.017521142959595, + 4.178442120552063, + 4.063559174537659, + 4.112444281578064, + 4.018282771110535, + 4.090330600738525, + 4.052797079086304, + 4.044416606426239, + 3.9110067486763, + 4.044709920883179, + 3.9488765001296997, + 4.079939246177673, + 4.106925308704376, + 4.205727219581604, + 4.124872088432312, + 4.019649684429169, + 3.9771183133125305, + 4.08052796125412, + 4.116117596626282, + 3.901540160179138, + 4.071512699127197, + 4.004287242889404, + 4.007235407829285, + 3.9099727272987366, + 4.0127023458480835, + 3.968506693840027, + 3.884955108165741, + 3.9934895634651184, + 3.981582283973694, + 4.078922152519226, + 3.9911267161369324, + 4.0229151248931885, + 3.9206641912460327, + 3.807271122932434, + 3.924223482608795, + 3.8494120836257935, + 3.892327308654785, + 3.9805421829223633, + 3.946932554244995, + 3.9181432127952576, + 3.992702901363373, + 4.072226643562317, + 4.17123556137085, + 3.9530303478240967, + 3.915065348148346, + 4.046460151672363, + 4.035461485385895, + 3.869140148162842, + 4.066434025764465, + 3.8664437532424927, + 3.9629687666893005, + 4.08042573928833, + 3.9274056553840637, + 3.9423299431800842, + 3.9215051531791687, + 4.00299596786499, + 3.9502922892570496, + 3.8922160267829895, + 3.9276620149612427, + 4.000955522060394, + 4.002703666687012, + 3.836031675338745, + 3.8647682666778564, + 4.082808077335358, + 3.8503384590148926, + 3.9441674947738647, + 4.07770162820816, + 3.8838642835617065, + 4.086974799633026, + 4.015573680400848, + 3.946977972984314, + 4.0215208530426025, + 3.8848971128463745, + 4.002051770687103, + 3.850814461708069, + 3.9650919437408447, + 3.911784589290619, + 3.9703415632247925, + 3.967207431793213, + 3.887374460697174, + 3.818388879299164, + 3.928144633769989, + 3.8909103870391846, + 3.8451098203659058, + 3.939436376094818, + 3.752828598022461, + 4.072072505950928, + 3.973407745361328, + 3.921144127845764, + 4.030504584312439, + 3.936560034751892, + 3.803256034851074, + 3.875494122505188, + 3.9495678544044495, + 3.968842387199402, + 4.051947593688965, + 4.008354425430298, + 4.013046145439148, + 3.832763969898224, + 3.7631590962409973, + 3.9382394552230835, + 3.9050270915031433, + 4.055655896663666, + 3.8730862736701965, + 3.9682533144950867, + 3.945443034172058, + 3.8956958055496216, + 3.93393474817276, + 3.8636527061462402, + 3.976307690143585, + 3.921903669834137, + 3.9841997623443604, + 3.9254733324050903, + 3.940099358558655, + 4.026372790336609, + 3.87931364774704, + 3.9371424317359924, + 3.855467438697815, + 3.8219879865646362, + 3.906170606613159, + 3.8009113669395447, + 3.8313414454460144, + 3.8962238430976868, + 3.9627997875213623, + 3.72141033411026, + 3.9138585925102234, + 3.8855895400047302, + 3.813308835029602, + 4.015699148178101, + 3.886446714401245, + 3.9445449113845825, + 3.754750967025757, + 3.879067838191986, + 3.9180434346199036, + 3.943216323852539, + 3.8603398203849792, + 3.8012725114822388, + 3.9150198698043823, + 3.885987937450409, + 3.8571590185165405, + 4.0247384905815125, + 3.8146743774414062, + 3.8938441276550293, + 3.955155372619629, + 3.875216484069824, + 3.8899060487747192, + 3.9128770232200623, + 3.9983811378479004, + 3.7793492674827576, + 3.853026270866394, + 3.697811186313629, + 3.9655781984329224, + 3.8560662269592285, + 3.7182199358940125, + 3.7700052857398987, + 3.845334768295288, + 3.7877896428108215, + 3.8885794281959534, + 3.778516709804535, + 3.793917417526245, + 3.6869407296180725, + 3.838708758354187, + 3.875401020050049, + 3.7240450978279114, + 3.8116047978401184, + 3.808272123336792, + 3.6765722036361694, + 3.7110021710395813, + 3.7427860498428345, + 3.859690010547638, + 3.7747711539268494, + 3.6674174666404724, + 3.8311761617660522, + 3.865542709827423, + 3.760772705078125, + 3.782244563102722, + 3.7930864095687866, + 3.816831350326538, + 3.7662359476089478, + 3.832552433013916, + 3.8083502650260925, + 3.815730631351471, + 3.7459311485290527, + 3.813211143016815, + 3.8307151198387146, + 3.8029991388320923, + 3.752304971218109, + 3.912246882915497, + 3.8538248538970947, + 3.7823254466056824, + 3.862412989139557, + 3.7584744095802307, + 3.9138391613960266, + 3.8174992203712463, + 3.8791105151176453, + 3.638350784778595, + 3.654911160469055, + 3.823740541934967, + 3.7457810044288635, + 3.6577929258346558, + 3.8140796422958374, + 3.7925796508789062, + 3.658810079097748, + 3.7495646476745605, + 3.7159247398376465, + 3.7579415440559387, + 3.7233884930610657, + 3.9162302017211914, + 3.764789879322052, + 3.7973087430000305, + 3.816229522228241, + 3.792701482772827, + 3.731752932071686, + 3.9220401644706726, + 3.8793890476226807, + 3.828413665294647, + 3.766660749912262, + 3.934762418270111, + 3.918330729007721, + 3.868453860282898, + 3.7718200087547302, + 3.813161015510559, + 3.8392574787139893, + 3.6984631419181824, + 3.7772459387779236, + 3.887756049633026, + 3.8504589796066284, + 3.8125521540641785, + 3.7228955030441284, + 3.7709259390830994, + 3.6170108914375305, + 3.8545233011245728, + 3.793993890285492, + 3.72869336605072, + 3.814563512802124, + 3.8494983315467834, + 3.7431355714797974, + 3.730987071990967, + 3.88859885931015, + 3.7149869799613953, + 3.7798129320144653, + 3.7536372542381287, + 3.826979160308838, + 3.8851450085639954, + 3.8997384905815125, + 3.984295070171356, + 3.758336901664734, + 3.7077627182006836, + 3.723150908946991, + 3.795274317264557, + 3.8417155742645264, + 3.829464614391327, + 3.7796894907951355, + 3.6434842348098755, + 3.75202614068985, + 3.7150397300720215, + 3.7154557704925537, + 3.7289845943450928, + 3.706716299057007, + 3.832434058189392, + 3.784528374671936, + 3.7818225026130676, + 3.814180374145508, + 3.8255295157432556, + 3.718918561935425, + 3.812308967113495, + 3.775261700153351, + 3.7935603857040405, + 3.8909420371055603, + 3.868490695953369, + 3.746316909790039, + 3.8051822781562805, + 3.753679871559143, + 3.8741760849952698, + 3.8638935685157776, + 3.62926709651947, + 3.7695427536964417, + 3.6430577635765076, + 3.579651355743408, + 3.7490938305854797, + 3.6935430765151978, + 3.5630106329917908, + 3.8544297218322754, + 3.6838480830192566, + 3.631006181240082, + 3.7437158823013306, + 3.644501030445099, + 3.6811503171920776, + 3.710752248764038, + 3.6390013694763184, + 3.6190815567970276, + 3.708713412284851, + 3.670904815196991, + 3.668027937412262, + 3.624551773071289, + 3.7466150522232056, + 3.665662944316864, + 3.6838990449905396, + 3.744770884513855, + 3.6021888852119446, + 3.733695089817047, + 3.598996341228485, + 3.776045024394989, + 3.7950311303138733, + 3.6720232367515564, + 3.6473219990730286, + 3.8040749430656433, + 3.608662962913513, + 3.6349794268608093, + 3.714953303337097, + 3.7722997069358826, + 3.6527950167655945, + 3.7900951504707336, + 3.6395036578178406, + 3.573512375354767, + 3.666409134864807, + 3.618045926094055, + 3.634136438369751, + 3.6026282906532288, + 3.704672634601593, + 3.7816720008850098, + 3.7792410254478455, + 3.6508020162582397, + 3.62242728471756, + 3.678542137145996, + 3.708307385444641, + 3.7012168169021606, + 3.7397642135620117, + 3.6836010813713074, + 3.5903636813163757, + 3.5334434509277344, + 3.7916120290756226, + 3.643624782562256, + 3.6918201446533203, + 3.6912108659744263, + 3.591599404811859, + 3.765861451625824, + 3.707682490348816, + 3.727255880832672, + 3.6743099093437195, + 3.6736221313476562, + 3.7232760190963745, + 3.6237990856170654, + 3.735556423664093, + 3.6960991621017456, + 3.7006987929344177, + 3.6412934064865112, + 3.6752359867095947, + 3.7933253049850464, + 3.7134100794792175, + 3.643900752067566, + 3.554062008857727, + 3.727342426776886, + 3.6475199460983276, + 3.6700103282928467, + 3.5737066864967346, + 3.7469775080680847, + 3.7605100274086, + 3.6170461177825928, + 3.7681961059570312, + 3.6625524759292603, + 3.7146717309951782, + 3.4780275225639343, + 3.7575483322143555, + 3.5638394951820374, + 3.7358844876289368, + 3.630566895008087, + 3.634018361568451, + 3.6832534670829773, + 3.720383107662201, + 3.5596231818199158, + 3.676276743412018, + 3.819012761116028, + 3.6777843236923218, + 3.635394513607025, + 3.4775039553642273, + 3.669287145137787, + 3.6337589025497437, + 3.7145708799362183, + 3.6727872490882874, + 3.645066261291504, + 3.6844720244407654, + 3.5127668976783752, + 3.6700137853622437, + 3.788974404335022, + 3.5920119881629944, + 3.6068843007087708, + 3.7376017570495605, + 3.719326913356781, + 3.79283607006073, + 3.615121841430664, + 3.7604528069496155, + 3.7461882829666138, + 3.611038863658905, + 3.5830376744270325, + 3.592675805091858, + 3.6873619556427, + 3.7400700449943542, + 3.6752508878707886, + 3.6960180401802063, + 3.642270267009735, + 3.583266258239746, + 3.717293620109558, + 3.5941476225852966, + 3.6109562516212463, + 3.5532190799713135, + 3.556881844997406, + 3.3787967562675476, + 3.744163751602173, + 3.5598872303962708, + 3.5306084752082825, + 3.7184404730796814, + 3.5259247422218323, + 3.5742450952529907, + 3.6632816195487976, + 3.5748345255851746, + 3.7504305839538574, + 3.6505106687545776, + 3.5187734365463257, + 3.6143487691879272, + 3.4175184965133667, + 3.54808109998703, + 3.679237484931946, + 3.52364045381546, + 3.5345025062561035, + 3.6708874106407166, + 3.6186588406562805, + 3.5535709261894226, + 3.547949254512787, + 3.6973613500595093, + 3.5815423727035522, + 3.5910269618034363, + 3.6708003282546997, + 3.5927411317825317, + 3.709048092365265, + 3.6537042260169983, + 3.586921215057373, + 3.703429937362671, + 3.7643170952796936, + 3.670318305492401, + 3.6707233786582947, + 3.5833394527435303, + 3.6942543387413025, + 3.544708013534546, + 3.6103376746177673, + 3.5691675543785095, + 3.534975528717041, + 3.6927815675735474, + 3.5810431838035583, + 3.6179187893867493, + 3.699172258377075, + 3.6183783411979675, + 3.4762324690818787, + 3.601687014102936, + 3.5997403264045715, + 3.604677677154541, + 3.505055069923401, + 3.497717559337616, + 3.679884433746338, + 3.6011030673980713, + 3.6083038449287415, + 3.6703783869743347, + 3.5532458424568176, + 3.7018598914146423, + 3.445693075656891, + 3.5294564962387085, + 3.6572806239128113, + 3.7213019728660583, + 3.6919736862182617, + 3.5977277159690857, + 3.5377355813980103, + 3.4814868569374084, + 3.593977451324463, + 3.5963679552078247, + 3.5922685861587524, + 3.566042721271515, + 3.613339066505432, + 3.571771204471588, + 3.5564870834350586, + 3.6520222425460815, + 3.648749887943268, + 3.6295065879821777, + 3.60439270734787, + 3.6399282217025757, + 3.5798118114471436, + 3.5162065625190735, + 3.578220784664154, + 3.8059487342834473, + 3.558323085308075, + 3.551829159259796, + 3.627200424671173, + 3.593502461910248, + 3.5853905081748962, + 3.5943334102630615, + 3.5604461431503296, + 3.700144946575165, + 3.6156598329544067, + 3.5633872151374817, + 3.546835482120514, + 3.6519500613212585, + 3.655335247516632, + 3.5557732582092285, + 3.641532242298126, + 3.6220381259918213, + 3.720620036125183, + 3.5017088651657104, + 3.5745678544044495, + 3.6524983048439026, + 3.6791558861732483, + 3.6895620226860046, + 3.578799784183502, + 3.638112485408783, + 3.6409814953804016, + 3.652482569217682, + 3.6164541244506836, + 3.630449593067169, + 3.528930187225342, + 3.481272280216217, + 3.5971354246139526, + 3.545348644256592, + 3.6369818449020386, + 3.4988136887550354, + 3.549082100391388, + 3.6143213510513306, + 3.619731366634369, + 3.4950673580169678, + 3.605524480342865, + 3.646136701107025, + 3.5257599353790283, + 3.5358309745788574, + 3.4171600341796875, + 3.52278333902359, + 3.5232359766960144, + 3.465148389339447, + 3.513220965862274, + 3.4122793078422546, + 3.46891850233078, + 3.667427182197571, + 3.4660876989364624, + 3.5708658695220947, + 3.524542212486267, + 3.594973623752594, + 3.572457730770111, + 3.5427181124687195, + 3.5700854063034058, + 3.4958359003067017, + 3.6033369302749634, + 3.5978001952171326, + 3.5550637245178223, + 3.4275514483451843, + 3.4137802124023438, + 3.5178836584091187, + 3.5896747708320618, + 3.609309434890747, + 3.5967149138450623, + 3.461303949356079, + 3.515401065349579, + 3.488942325115204, + 3.475443720817566, + 3.6371309757232666, + 3.530739426612854, + 3.5264581441879272, + 3.44410640001297, + 3.5166282653808594, + 3.5358694195747375, + 3.5702564120292664, + 3.5448442697525024, + 3.5635308027267456, + 3.526429057121277, + 3.4832653999328613, + 3.46889865398407, + 3.542026460170746, + 3.3837761878967285, + 3.6215887665748596, + 3.6228492856025696, + 3.4428473114967346, + 3.5661879181861877, + 3.5624367594718933, + 3.5997581481933594, + 3.4738742113113403, + 3.5315909385681152, + 3.6134085655212402, + 3.4998491406440735, + 3.482656717300415, + 3.553369164466858, + 3.5679391622543335, + 3.4923399090766907, + 3.6008391976356506, + 3.596055507659912, + 3.576074779033661, + 3.5227924585342407, + 3.5990980863571167, + 3.345803439617157, + 3.4312846660614014, + 3.5433480739593506, + 3.5503938794136047, + 3.5422422289848328, + 3.5673349499702454, + 3.628373086452484, + 3.503259241580963, + 3.594617545604706, + 3.480763554573059, + 3.462605834007263, + 3.522369384765625, + 3.6075628995895386, + 3.6701005697250366, + 3.528590440750122, + 3.432179629802704, + 3.507729649543762, + 3.375546097755432, + 3.4693488478660583, + 3.5137033462524414, + 3.6160526275634766, + 3.547798991203308, + 3.5965678095817566, + 3.5528615713119507, + 3.440722942352295, + 3.58172208070755, + 3.531825363636017, + 3.6653754711151123, + 3.489143669605255, + 3.5771586894989014, + 3.5920684933662415, + 3.603786885738373, + 3.57408744096756, + 3.4498563408851624, + 3.533388912677765, + 3.569668769836426, + 3.492045521736145, + 3.641917109489441, + 3.5119056701660156, + 3.4957385063171387, + 3.591129422187805, + 3.448542535305023, + 3.5304871201515198, + 3.726737856864929, + 3.673419773578644, + 3.490631580352783, + 3.514852225780487, + 3.550054609775543, + 3.566477656364441, + 3.6336731910705566, + 3.4518293142318726, + 3.6153140664100647, + 3.4325101375579834, + 3.507133722305298, + 3.611715793609619, + 3.6130394339561462, + 3.600353479385376, + 3.420622766017914, + 3.5830631256103516, + 3.5284923911094666, + 3.6625497937202454, + 3.4873071908950806, + 3.44851553440094, + 3.5519707798957825, + 3.3551623821258545, + 3.441202938556671, + 3.557066857814789, + 3.5600330233573914, + 3.6596139669418335, + 3.4779791235923767, + 3.327766537666321, + 3.5552175045013428, + 3.4301247596740723, + 3.6017106771469116, + 3.428759455680847, + 3.579155147075653, + 3.5609182715415955, + 3.473021447658539, + 3.5177143812179565, + 3.458059310913086, + 3.4320635199546814, + 3.44335675239563, + 3.453716218471527, + 3.6047878861427307, + 3.4658583402633667, + 3.6575766801834106, + 3.3947041630744934, + 3.3785316944122314, + 3.3928292989730835, + 3.476392984390259, + 3.557487964630127, + 3.514597535133362, + 3.4433688521385193, + 3.44401615858078, + 3.5041463971138, + 3.5089334845542908, + 3.6018989086151123, + 3.5717285871505737, + 3.64217209815979, + 3.51226270198822, + 3.4631197452545166, + 3.5106133222579956, + 3.5691022872924805, + 3.567513644695282, + 3.646485984325409, + 3.6188907623291016, + 3.4607606530189514, + 3.5456846952438354, + 3.497795522212982, + 3.383936285972595, + 3.4521749019622803, + 3.639424741268158, + 3.4222569465637207, + 3.4635239839553833, + 3.482508361339569, + 3.4333035945892334, + 3.3552820086479187, + 3.433124601840973, + 3.4122740626335144, + 3.4960732460021973, + 3.5929136276245117, + 3.4985368847846985, + 3.4985363483428955, + 3.4355106353759766, + 3.452476680278778, + 3.5027464032173157, + 3.4887693524360657, + 3.5272903442382812, + 3.503446340560913, + 3.5760297775268555, + 3.6000742316246033, + 3.5401470065116882, + 3.519977867603302, + 3.559155523777008, + 3.509692370891571, + 3.5019903779029846, + 3.5460384488105774, + 3.514879286289215, + 3.537106990814209, + 3.48558646440506, + 3.4362871646881104, + 3.4169535040855408, + 3.543427586555481, + 3.4650726914405823, + 3.469789505004883, + 3.560577630996704, + 3.460982024669647, + 3.5944419503211975, + 3.5186089277267456, + 3.3652994632720947, + 3.486658811569214, + 3.492713749408722, + 3.582874059677124, + 3.484554409980774, + 3.5355862975120544, + 3.6116782426834106, + 3.5737169981002808, + 3.623885154724121, + 3.5581822395324707, + 3.539720058441162, + 3.5059598684310913, + 3.41338050365448, + 3.494231164455414, + 3.6090176105499268, + 3.585350275039673, + 3.531272530555725, + 3.484459936618805, + 3.514652729034424, + 3.5751178860664368, + 3.506460726261139, + 3.482465982437134, + 3.5328498482704163, + 3.5960402488708496, + 3.4799234867095947, + 3.5154150128364563, + 3.424070954322815, + 3.6179869174957275 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_lm_loss": [ + 4.730759532363327, + 4.292202605141534, + 4.106572142353764, + 3.9940653023896395, + 3.9249527895892107, + 3.860592842102051, + 3.80927136209276, + 3.7592532104916043, + 3.718546717255204, + 3.683093406535961, + 3.6565136556272155, + 3.6251701249016657, + 3.6002505002198397, + 3.577907262025056, + 3.556569602754381, + 3.54383326459814, + 3.5391001524748624, + 3.535805755191379, + 3.534921813894201, + 3.527988583953292 + ], + "val_ppl": [ + 113.38164668266536, + 73.1273619456279, + 60.738158557308346, + 54.275086114155606, + 50.65068676266321, + 47.4934991703101, + 45.11755252760369, + 42.91636451791131, + 41.204468831410956, + 39.769226374296274, + 38.72609474505665, + 37.53110759249867, + 36.607403457823956, + 35.798545426089646, + 35.04278006776334, + 34.59929356290422, + 34.43591816911258, + 34.322659236236156, + 34.29233342540116, + 34.05539910173824 + ] +} \ No newline at end of file diff --git a/074_dthin_ablation/medium_k5_d256_best.pt b/074_dthin_ablation/medium_k5_d256_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b01a409911129e61a0f7ee792186c10467b48c2 --- /dev/null +++ b/074_dthin_ablation/medium_k5_d256_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92261d5f0f5e75ce583acf708f2185ad4d2cb9235f949ec24d813f07c0df4917 +size 341438933 diff --git a/074_dthin_ablation/medium_k5_d256_results.json b/074_dthin_ablation/medium_k5_d256_results.json new file mode 100644 index 0000000000000000000000000000000000000000..e97fd6f7607c9026a95a5f9e114a371371d49b59 --- /dev/null +++ b/074_dthin_ablation/medium_k5_d256_results.json @@ -0,0 +1,2075 @@ +{ + "label": "medium_k5_d256", + "condition": "medium_k5_d256", + "total_params": 85301248, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_lm_loss": [ + 19.07690191268921, + 15.06089997291565, + 7.9938836097717285, + 7.327017307281494, + 7.060478568077087, + 6.980088472366333, + 6.955796122550964, + 6.827393889427185, + 6.6658079624176025, + 6.575880527496338, + 6.4752538204193115, + 6.379788637161255, + 6.3618340492248535, + 6.325033903121948, + 6.193679332733154, + 6.191814064979553, + 6.043694496154785, + 6.016449809074402, + 5.890402317047119, + 5.799531102180481, + 5.819183826446533, + 5.69842791557312, + 5.688725590705872, + 5.663894772529602, + 5.658753275871277, + 5.54321825504303, + 5.546668291091919, + 5.510203957557678, + 5.350895643234253, + 5.387294054031372, + 5.265266180038452, + 5.281045436859131, + 5.29039990901947, + 5.165512800216675, + 5.154169678688049, + 5.0997806787490845, + 5.151888370513916, + 5.212878704071045, + 5.176437616348267, + 5.063419938087463, + 5.089197754859924, + 5.022033452987671, + 4.908715009689331, + 5.018410086631775, + 4.95183789730072, + 4.988872289657593, + 4.917376518249512, + 4.991174101829529, + 5.015760183334351, + 4.9177738428115845, + 4.902073860168457, + 4.768052935600281, + 4.82747483253479, + 4.796375393867493, + 4.809686183929443, + 4.708108067512512, + 4.842057466506958, + 4.745791792869568, + 4.737133979797363, + 4.766752600669861, + 4.814200401306152, + 4.791795372962952, + 4.832038760185242, + 4.717207431793213, + 4.720402836799622, + 4.695350766181946, + 4.713189244270325, + 4.771606087684631, + 4.859859585762024, + 4.716498494148254, + 4.589824318885803, + 4.695519685745239, + 4.519311308860779, + 4.5891886949539185, + 4.563928008079529, + 4.547561883926392, + 4.708731412887573, + 4.564114332199097, + 4.621268630027771, + 4.673612594604492, + 4.544253349304199, + 4.4396456480026245, + 4.543972849845886, + 4.552196502685547, + 4.677753806114197, + 4.664826512336731, + 4.370947480201721, + 4.5424724817276, + 4.5663710832595825, + 4.499121904373169, + 4.6340491771698, + 4.587262034416199, + 4.452302813529968, + 4.310805261135101, + 4.571579098701477, + 4.541173696517944, + 4.38883912563324, + 4.457138419151306, + 4.3610533475875854, + 4.47153627872467, + 4.415113925933838, + 4.418918490409851, + 4.394588232040405, + 4.3433027267456055, + 4.463629245758057, + 4.431129574775696, + 4.344295382499695, + 4.598978042602539, + 4.339941382408142, + 4.455299735069275, + 4.4293283224105835, + 4.316672086715698, + 4.418018817901611, + 4.379801869392395, + 4.39546263217926, + 4.414585590362549, + 4.298339128494263, + 4.42634904384613, + 4.35063099861145, + 4.2865378856658936, + 4.395834565162659, + 4.366646885871887, + 4.249017953872681, + 4.334885120391846, + 4.319614768028259, + 4.362876653671265, + 4.165022850036621, + 4.174274563789368, + 4.221319675445557, + 4.161334276199341, + 4.1814703941345215, + 4.2011988162994385, + 4.339158415794373, + 4.223176836967468, + 4.244691014289856, + 4.2649736404418945, + 4.108097314834595, + 4.253182053565979, + 4.201514840126038, + 4.268481373786926, + 4.274955153465271, + 4.193342089653015, + 4.080296277999878, + 4.17158305644989, + 4.210350513458252, + 4.19588315486908, + 4.213884115219116, + 4.21293306350708, + 4.217528820037842, + 4.213658809661865, + 4.2370147705078125, + 4.194646596908569, + 4.176769196987152, + 4.198511481285095, + 4.3019373416900635, + 4.092658877372742, + 4.1908575892448425, + 4.214468836784363, + 4.063927352428436, + 4.011255145072937, + 4.140955626964569, + 4.172881722450256, + 4.149149537086487, + 3.990418016910553, + 4.1216471791267395, + 4.194180011749268, + 4.078207850456238, + 4.0327038168907166, + 4.134963512420654, + 4.2162405252456665, + 4.10119503736496, + 4.088599264621735, + 4.056123435497284, + 4.157319664955139, + 4.193384647369385, + 4.054107904434204, + 4.042531788349152, + 4.181956648826599, + 4.1073421239852905, + 4.004284143447876, + 4.10975855588913, + 4.14948582649231, + 4.086794197559357, + 4.085497677326202, + 4.073818325996399, + 4.180313467979431, + 4.041598856449127, + 4.049733102321625, + 4.0540801882743835, + 4.134469509124756, + 4.091803967952728, + 4.051685929298401, + 4.108040690422058, + 4.0563424825668335, + 4.076012492179871, + 4.035238027572632, + 4.11169970035553, + 4.003843367099762, + 4.073352575302124, + 4.15700376033783, + 4.113118290901184, + 4.021407425403595, + 4.103076159954071, + 4.080251395702362, + 4.164442300796509, + 3.95265394449234, + 4.058778166770935, + 3.984954535961151, + 4.039354085922241, + 3.9075732231140137, + 4.124482989311218, + 4.0839720368385315, + 4.002514541149139, + 3.9289793968200684, + 4.081127107143402, + 4.043694853782654, + 4.044926643371582, + 3.9764928817749023, + 4.008680760860443, + 4.073463022708893, + 3.9006245136260986, + 3.991203546524048, + 4.002445936203003, + 3.9762311577796936, + 4.037747323513031, + 4.042621850967407, + 3.982284724712372, + 4.012658894062042, + 4.004940629005432, + 3.9892733097076416, + 4.007197618484497, + 4.10672402381897, + 3.903990924358368, + 4.115413248538971, + 4.049934804439545, + 4.029612839221954, + 3.801010310649872, + 3.9533966779708862, + 3.851983666419983, + 4.029419541358948, + 4.0899717807769775, + 3.900091290473938, + 4.005148112773895, + 3.907813012599945, + 3.9440606832504272, + 4.0508710741996765, + 3.90435528755188, + 3.9075745344161987, + 3.880930185317993, + 3.9551936388015747, + 4.025200009346008, + 3.936802625656128, + 3.8028565049171448, + 3.990792214870453, + 3.88609915971756, + 3.942649483680725, + 3.9000943303108215, + 3.875147223472595, + 4.008297502994537, + 3.874810039997101, + 3.8209261298179626, + 3.83420068025589, + 3.8011351823806763, + 3.9575955867767334, + 3.760631740093231, + 3.913088619709015, + 3.817867875099182, + 3.960912346839905, + 3.8778093457221985, + 3.918474793434143, + 3.891515851020813, + 3.903558909893036, + 3.878151535987854, + 3.9703575372695923, + 3.8672133684158325, + 3.9724056124687195, + 4.014539361000061, + 3.7778181433677673, + 3.8515859842300415, + 3.913736939430237, + 3.9665809273719788, + 3.8546165823936462, + 3.959079623222351, + 3.9169337153434753, + 3.893479347229004, + 3.8996936678886414, + 3.8916974663734436, + 3.819976806640625, + 3.8838146328926086, + 3.8659257888793945, + 3.8804861903190613, + 3.9194403886795044, + 3.855730712413788, + 3.8452444076538086, + 3.917757749557495, + 3.8920605778694153, + 3.8939443826675415, + 3.910735607147217, + 3.8488460183143616, + 3.9202584624290466, + 3.8808364868164062, + 3.7074866890907288, + 3.9353533387184143, + 3.835721433162689, + 3.8406915068626404, + 3.8614258766174316, + 3.8263887763023376, + 3.8910104632377625, + 3.880992889404297, + 3.773061513900757, + 3.806958794593811, + 3.9233238101005554, + 3.8615670204162598, + 3.818481743335724, + 3.957645535469055, + 3.9343684315681458, + 3.81706565618515, + 3.815762758255005, + 3.8323363661766052, + 3.765995740890503, + 3.7732696533203125, + 3.669905126094818, + 3.8592939972877502, + 3.882116973400116, + 3.7804725766181946, + 3.7218785285949707, + 3.902097702026367, + 3.715480327606201, + 3.944928765296936, + 3.8584415912628174, + 3.871944844722748, + 3.809486746788025, + 3.9871302247047424, + 3.9514594674110413, + 3.905209541320801, + 3.787480592727661, + 3.6957173943519592, + 3.7637752294540405, + 3.9466261863708496, + 3.7959726452827454, + 3.8435298800468445, + 3.914907217025757, + 3.848744809627533, + 3.738695502281189, + 3.8367003202438354, + 3.818193018436432, + 3.7983800172805786, + 3.8717010617256165, + 3.8536778688430786, + 3.831671893596649, + 3.786437749862671, + 3.8546182513237, + 3.7929397225379944, + 3.8152758479118347, + 3.831562578678131, + 3.8470864295959473, + 3.6071473360061646, + 3.741546869277954, + 3.7374046444892883, + 3.77769672870636, + 3.823634386062622, + 3.8138246536254883, + 3.7939122915267944, + 3.9130349159240723, + 3.9085293412208557, + 3.7622599005699158, + 3.893942415714264, + 3.7382310032844543, + 3.8805596828460693, + 3.8595321774482727, + 3.8479214310646057, + 3.833198070526123, + 3.9164252281188965, + 3.7931912541389465, + 3.7975417375564575, + 3.826914966106415, + 3.817317306995392, + 3.8946653008461, + 3.871519982814789, + 3.7506288290023804, + 3.7971304059028625, + 3.6427252292633057, + 3.747728168964386, + 3.8397549986839294, + 3.7019823789596558, + 3.6811333894729614, + 3.6541442275047302, + 3.7003358602523804, + 3.8256266117095947, + 3.6481592059135437, + 3.6226560473442078, + 3.6296550035476685, + 3.7258747220039368, + 3.687438666820526, + 3.7174030542373657, + 3.6781469583511353, + 3.6993958353996277, + 3.620010495185852, + 3.691216468811035, + 3.7246735095977783, + 3.77431321144104, + 3.7919746041297913, + 3.788726329803467, + 3.6709134578704834, + 3.6959383487701416, + 3.7439181804656982, + 3.800956964492798, + 3.5992438197135925, + 3.6953033208847046, + 3.7140254378318787, + 3.7661176323890686, + 3.7409615516662598, + 3.8214575052261353, + 3.523919641971588, + 3.7675113677978516, + 3.7582110166549683, + 3.707004427909851, + 3.819295048713684, + 3.7350093126296997, + 3.7129640579223633, + 3.681863784790039, + 3.6613924503326416, + 3.737086296081543, + 3.571808338165283, + 3.738505482673645, + 3.81344735622406, + 3.7416902780532837, + 3.7449766993522644, + 3.7360170483589172, + 3.7539479732513428, + 3.8181700110435486, + 3.6791785955429077, + 3.805078089237213, + 3.7810137271881104, + 3.6547901034355164, + 3.6638604402542114, + 3.7266547083854675, + 3.626627027988434, + 3.6416863203048706, + 3.6159005165100098, + 3.6631045937538147, + 3.8355478048324585, + 3.652977466583252, + 3.6137977242469788, + 3.6249639987945557, + 3.64492928981781, + 3.731159031391144, + 3.742150604724884, + 3.6055034399032593, + 3.6708202362060547, + 3.7260981798171997, + 3.747846007347107, + 3.692859709262848, + 3.6354715824127197, + 3.7537314891815186, + 3.698926627635956, + 3.856418013572693, + 3.7668431997299194, + 3.703269362449646, + 3.7576310634613037, + 3.644724130630493, + 3.6914032101631165, + 3.6969094276428223, + 3.8088292479515076, + 3.6165100932121277, + 3.613904058933258, + 3.7032551765441895, + 3.583567261695862, + 3.7269829511642456, + 3.7281094789505005, + 3.6733750104904175, + 3.7722689509391785, + 3.6132595539093018, + 3.6471412777900696, + 3.7709473967552185, + 3.7469940185546875, + 3.795358717441559, + 3.764907479286194, + 3.657415986061096, + 3.701641619205475, + 3.66881000995636, + 3.7671963572502136, + 3.651298999786377, + 3.6980360746383667, + 3.762771487236023, + 3.669272720813751, + 3.618022084236145, + 3.5981411933898926, + 3.752999722957611, + 3.7683801651000977, + 3.7109276056289673, + 3.5015439987182617, + 3.602179229259491, + 3.7077316641807556, + 3.629833221435547, + 3.697573721408844, + 3.6474465131759644, + 3.5768776535987854, + 3.7208985090255737, + 3.5527933835983276, + 3.677004337310791, + 3.6381970643997192, + 3.7228170037269592, + 3.6285095810890198, + 3.6342010498046875, + 3.5761600732803345, + 3.6109594106674194, + 3.6078954935073853, + 3.51259708404541, + 3.5590043663978577, + 3.5117751359939575, + 3.6353477239608765, + 3.5018155574798584, + 3.50762015581131, + 3.4894593954086304, + 3.6705791354179382, + 3.616646707057953, + 3.698479115962982, + 3.5985516905784607, + 3.6695538759231567, + 3.535810172557831, + 3.6062249541282654, + 3.431481122970581, + 3.586198329925537, + 3.619955837726593, + 3.6663795113563538, + 3.536335289478302, + 3.572522222995758, + 3.6327706575393677, + 3.5118451714515686, + 3.5748175382614136, + 3.57997989654541, + 3.628503978252411, + 3.664009392261505, + 3.626383900642395, + 3.5847516655921936, + 3.6494941115379333, + 3.576300263404846, + 3.6916309595108032, + 3.521195650100708, + 3.633714497089386, + 3.4631524682044983, + 3.6392635703086853, + 3.5984615087509155, + 3.5430567264556885, + 3.567618668079376, + 3.526519000530243, + 3.572174906730652, + 3.5905962586402893, + 3.548408031463623, + 3.5136688947677612, + 3.488716781139374, + 3.531975567340851, + 3.55849951505661, + 3.678005337715149, + 3.614187002182007, + 3.6705889105796814, + 3.5690687894821167, + 3.6094223260879517, + 3.616416275501251, + 3.502914786338806, + 3.602295398712158, + 3.5883609652519226, + 3.5093820095062256, + 3.4747678637504578, + 3.4915407299995422, + 3.557813048362732, + 3.722474455833435, + 3.6042909026145935, + 3.649363934993744, + 3.6964699625968933, + 3.5693934559822083, + 3.553456664085388, + 3.5507628321647644, + 3.6728403568267822, + 3.6459890007972717, + 3.506980299949646, + 3.5192019939422607, + 3.5582236647605896, + 3.610010266304016, + 3.5714427828788757, + 3.5531256198883057, + 3.589378297328949, + 3.534815490245819, + 3.5128785371780396, + 3.659948170185089, + 3.496549367904663, + 3.7080222964286804, + 3.7045637369155884, + 3.495600402355194, + 3.594305396080017, + 3.5793107748031616, + 3.603575825691223, + 3.609942138195038, + 3.4784790873527527, + 3.6247854828834534, + 3.576903522014618, + 3.5519871711730957, + 3.5329078435897827, + 3.59560364484787, + 3.5584115386009216, + 3.5475932359695435, + 3.703045606613159, + 3.53644996881485, + 3.6449939608573914, + 3.574739456176758, + 3.517929196357727, + 3.590487480163574, + 3.5810545086860657, + 3.515048384666443, + 3.5540642142295837, + 3.520013689994812, + 3.5795982480049133, + 3.644042670726776, + 3.5087300539016724, + 3.4115716218948364, + 3.6221604347229004, + 3.60512113571167, + 3.57305109500885, + 3.5440542697906494, + 3.4915067553520203, + 3.566536068916321, + 3.6004151701927185, + 3.4190218448638916, + 3.5424176454544067, + 3.6117507219314575, + 3.63498318195343, + 3.4437795281410217, + 3.7086946964263916, + 3.611219048500061, + 3.540272295475006, + 3.5192102789878845, + 3.3576398491859436, + 3.555576741695404, + 3.3112151622772217, + 3.4875652194023132, + 3.505402147769928, + 3.5837268233299255, + 3.525342583656311, + 3.425486445426941, + 3.5800743103027344, + 3.3808319568634033, + 3.4436550736427307, + 3.466976225376129, + 3.462561547756195, + 3.5331572890281677, + 3.444850206375122, + 3.5231637954711914, + 3.3596644401550293, + 3.467333674430847, + 3.434743344783783, + 3.405024528503418, + 3.5144400000572205, + 3.518627107143402, + 3.44856595993042, + 3.4125701189041138, + 3.530989170074463, + 3.5393117666244507, + 3.3944069743156433, + 3.4002130031585693, + 3.5127179622650146, + 3.405809760093689, + 3.377851128578186, + 3.5314890146255493, + 3.4850090742111206, + 3.5433066487312317, + 3.517196238040924, + 3.3964503407478333, + 3.488090753555298, + 3.6395257711410522, + 3.547140657901764, + 3.3660423159599304, + 3.3948001861572266, + 3.512813091278076, + 3.3713318705558777, + 3.5000727772712708, + 3.4995758533477783, + 3.4611461758613586, + 3.4672040939331055, + 3.4936625361442566, + 3.430328071117401, + 3.4939449429512024, + 3.526476740837097, + 3.4270861744880676, + 3.44223290681839, + 3.550281524658203, + 3.4159215092658997, + 3.4893651604652405, + 3.568204641342163, + 3.494094431400299, + 3.5139575600624084, + 3.508519411087036, + 3.4459537267684937, + 3.4073221683502197, + 3.491217613220215, + 3.61219984292984, + 3.436636984348297, + 3.521315574645996, + 3.4249377846717834, + 3.543221890926361, + 3.5231476426124573, + 3.3958359956741333, + 3.396744191646576, + 3.576049566268921, + 3.608063042163849, + 3.4704520106315613, + 3.3926790356636047, + 3.432269513607025, + 3.40488463640213, + 3.4418318271636963, + 3.4804224371910095, + 3.5035099387168884, + 3.6034152507781982, + 3.4727789759635925, + 3.604865610599518, + 3.474117159843445, + 3.365011751651764, + 3.4697077870368958, + 3.597846269607544, + 3.440912425518036, + 3.650572180747986, + 3.533517062664032, + 3.5529914498329163, + 3.5590859055519104, + 3.5398905873298645, + 3.428609549999237, + 3.5007286071777344, + 3.4261505007743835, + 3.4346742033958435, + 3.442721128463745, + 3.583163022994995, + 3.45884108543396, + 3.559533655643463, + 3.4857500195503235, + 3.5246150493621826, + 3.5172941088676453, + 3.3266632556915283, + 3.4698049426078796, + 3.408397078514099, + 3.5076669454574585, + 3.471220076084137, + 3.52794349193573, + 3.3649222254753113, + 3.525616466999054, + 3.5245441794395447, + 3.46210640668869, + 3.4787108302116394, + 3.569854199886322, + 3.5356485247612, + 3.3306643962860107, + 3.5125752091407776, + 3.52732515335083, + 3.5712857842445374, + 3.508386790752411, + 3.553074359893799, + 3.3874292969703674, + 3.4626858234405518, + 3.4452579617500305, + 3.569409966468811, + 3.4551103115081787, + 3.3741138577461243, + 3.4554208517074585, + 3.3121339678764343, + 3.492552876472473, + 3.382119297981262, + 3.4010539054870605, + 3.4060877561569214, + 3.334234356880188, + 3.4295933842658997, + 3.2196333408355713, + 3.4098663926124573, + 3.343916416168213, + 3.428012788295746, + 3.423416554927826, + 3.407267391681671, + 3.5029749274253845, + 3.3715264201164246, + 3.3258150815963745, + 3.4082708954811096, + 3.3520649671554565, + 3.225723683834076, + 3.4308273792266846, + 3.357459843158722, + 3.347959041595459, + 3.468882918357849, + 3.275366723537445, + 3.33911395072937, + 3.4185219407081604, + 3.461855411529541, + 3.4226766228675842, + 3.4273245334625244, + 3.4320753812789917, + 3.3303974866867065, + 3.4546667337417603, + 3.346523344516754, + 3.3530293703079224, + 3.3357344269752502, + 3.360603988170624, + 3.3128970861434937, + 3.3091185092926025, + 3.5234476923942566, + 3.392733156681061, + 3.4339938163757324, + 3.5912826657295227, + 3.471534848213196, + 3.3876452445983887, + 3.454115927219391, + 3.515151083469391, + 3.4410662055015564, + 3.401175379753113, + 3.3110561966896057, + 3.3150534629821777, + 3.575872004032135, + 3.364234447479248, + 3.460851788520813, + 3.3496164083480835, + 3.369138777256012, + 3.37735378742218, + 3.3534207940101624, + 3.434852719306946, + 3.5136687755584717, + 3.429216742515564, + 3.4774197936058044, + 3.3339158296585083, + 3.379782557487488, + 3.5066667795181274, + 3.346523404121399, + 3.428062915802002, + 3.3072394132614136, + 3.3402352929115295, + 3.4527979493141174, + 3.4994203448295593, + 3.4821797609329224, + 3.3876660466194153, + 3.2868992686271667, + 3.457379460334778, + 3.4093878269195557, + 3.42318856716156, + 3.471855938434601, + 3.2809666991233826, + 3.3200490474700928, + 3.190893828868866, + 3.414003014564514, + 3.310933828353882, + 3.338333547115326, + 3.3687803149223328, + 3.439738154411316, + 3.491325795650482, + 3.3771201372146606, + 3.36210697889328, + 3.4511969089508057, + 3.3439916372299194, + 3.3164947628974915, + 3.490164816379547, + 3.5113629698753357, + 3.4830667972564697, + 3.446448504924774, + 3.371856927871704, + 3.3854429125785828, + 3.5535017251968384, + 3.4655681848526, + 3.512127697467804, + 3.2947436571121216, + 3.4433504343032837, + 3.4168457984924316, + 3.327868640422821, + 3.3983525037765503, + 3.3908745646476746, + 3.2753694653511047, + 3.453220844268799, + 3.398018717765808, + 3.4635006189346313, + 3.347797453403473, + 3.4687788486480713, + 3.452986478805542, + 3.362510025501251, + 3.3445030450820923, + 3.246694326400757, + 3.2573333978652954, + 3.4075173139572144, + 3.3638707399368286, + 3.5250203609466553, + 3.4157589077949524, + 3.43256938457489, + 3.434028148651123, + 3.359765350818634, + 3.515144109725952, + 3.2675710320472717, + 3.43699449300766, + 3.3362948298454285, + 3.2652093172073364, + 3.3961175680160522, + 3.355102241039276, + 3.2976182103157043, + 3.3373173475265503, + 3.539558470249176, + 3.330936074256897, + 3.492289125919342, + 3.3412839770317078, + 3.3572900891304016, + 3.375416338443756, + 3.444121778011322, + 3.281125783920288, + 3.3322773575782776, + 3.2565621733665466, + 3.276177942752838, + 3.4156992435455322, + 3.3179983496665955, + 3.397717773914337, + 3.3287293910980225, + 3.282824218273163, + 3.396279513835907, + 3.3998830318450928, + 3.3866801857948303, + 3.260653078556061, + 3.3726401925086975, + 3.5148839950561523, + 3.2103896141052246, + 3.385660231113434, + 3.4464810490608215, + 3.3939175605773926, + 3.334871530532837, + 3.4709445238113403, + 3.3055017590522766, + 3.3153644800186157, + 3.2158629298210144, + 3.3134058713912964, + 3.414902091026306, + 3.3465041518211365, + 3.310633361339569, + 3.411267578601837, + 3.3782137036323547, + 3.3134886026382446, + 3.468464970588684, + 3.418883740901947, + 3.475451409816742, + 3.3126218914985657, + 3.423898220062256, + 3.474128305912018, + 3.3589322566986084, + 3.392372786998749, + 3.3339115977287292, + 3.440104126930237, + 3.3708523511886597, + 3.3955007195472717, + 3.398788273334503, + 3.3896124958992004, + 3.3099750876426697, + 3.4690650701522827, + 3.2985268235206604, + 3.4121755361557007, + 3.304074764251709, + 3.3394275307655334, + 3.4502868056297302, + 3.3816688656806946, + 3.425717294216156, + 3.4141740798950195, + 3.3257486820220947, + 3.379418909549713, + 3.3677513003349304, + 3.4662922024726868, + 3.4584656357765198, + 3.3150941729545593, + 3.473091781139374, + 3.341224253177643, + 3.3681699633598328, + 3.397334575653076, + 3.2815346717834473, + 3.369007170200348, + 3.442368268966675, + 3.5048148036003113, + 3.4112389087677, + 3.4242634773254395, + 3.4161500334739685, + 3.462536931037903, + 3.4328644275665283, + 3.481093406677246, + 3.352837026119232, + 3.29537570476532, + 3.30436247587204, + 3.4838741421699524, + 3.3804562091827393, + 3.4037250876426697, + 3.3749439120292664, + 3.398979663848877, + 3.425483226776123, + 3.370043158531189, + 3.394019663333893, + 3.316845178604126, + 3.383242189884186, + 3.5020902156829834, + 3.4866048097610474, + 3.379093587398529, + 3.3998862504959106, + 3.450647294521332, + 3.421801805496216, + 3.4819140434265137, + 3.319914758205414, + 3.357910454273224, + 3.4498353600502014 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_lm_loss": [ + 4.800527731577556, + 4.312803339075159, + 4.089153854935257, + 3.957644436094496, + 3.8598262380670616, + 3.7882562743292914, + 3.736310526176735, + 3.684728419339215, + 3.6402066636968544, + 3.597512227517587, + 3.5691902107662625, + 3.532796789098669, + 3.5091094882399947, + 3.482660523167363, + 3.4650541411505804, + 3.4513860278659396, + 3.446083413230048, + 3.4470496001066984, + 3.4434049835911504, + 3.4359429942237005 + ], + "val_ppl": [ + 121.57455932640153, + 74.64946364839228, + 59.6893644693113, + 52.33390478580152, + 47.457104414176214, + 44.17929649818399, + 41.94295690371286, + 39.83430275445393, + 38.099709738701655, + 36.5072995227209, + 35.48784383888827, + 34.219538739225506, + 33.418494966621, + 32.54619690158291, + 31.978191062320715, + 31.544083001540315, + 31.377259576399496, + 31.407590523149274, + 31.293330243944133, + 31.060688809825802 + ] +} \ No newline at end of file diff --git a/074_dthin_ablation/thin_k10_d128_best.pt b/074_dthin_ablation/thin_k10_d128_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..e897189bbc53448f6969b11cb8cc8297a765324f --- /dev/null +++ b/074_dthin_ablation/thin_k10_d128_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6666c76ad8e180b03d9654b3f7062b315413138c2c4d6f57c2210f8ed0ffe6a5 +size 340597959 diff --git a/074_dthin_ablation/thin_k10_d128_results.json b/074_dthin_ablation/thin_k10_d128_results.json new file mode 100644 index 0000000000000000000000000000000000000000..73545d06502ce8998f001267c8ada846e689e47d --- /dev/null +++ b/074_dthin_ablation/thin_k10_d128_results.json @@ -0,0 +1,2075 @@ +{ + "label": "thin_k10_d128", + "condition": "thin_k10_d128", + "total_params": 85035008, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_lm_loss": [ + 19.19176197052002, + 17.749855995178223, + 10.014677286148071, + 7.494815707206726, + 7.173145294189453, + 7.097337007522583, + 6.945642590522766, + 6.827858924865723, + 6.69776451587677, + 6.581432580947876, + 6.571573376655579, + 6.502119779586792, + 6.436146378517151, + 6.374040246009827, + 6.292686462402344, + 6.174090385437012, + 6.2381671667099, + 6.1140899658203125, + 6.053889513015747, + 6.085058331489563, + 6.00771689414978, + 6.0140968561172485, + 5.836256265640259, + 5.757054924964905, + 5.719298601150513, + 5.69845712184906, + 5.557123184204102, + 5.633027791976929, + 5.590327620506287, + 5.514170169830322, + 5.439119219779968, + 5.504186391830444, + 5.500596046447754, + 5.562026500701904, + 5.319220185279846, + 5.307552814483643, + 5.292004585266113, + 5.294865965843201, + 5.368765592575073, + 5.260830044746399, + 5.232339382171631, + 5.148167133331299, + 5.331398963928223, + 5.233230471611023, + 5.238111734390259, + 5.0979920625686646, + 5.009235858917236, + 5.310939431190491, + 5.09771192073822, + 5.141285538673401, + 5.200617432594299, + 5.056146621704102, + 5.010229468345642, + 4.992197275161743, + 5.054776072502136, + 5.125860095024109, + 4.921937942504883, + 4.999404191970825, + 4.905896782875061, + 5.052016258239746, + 5.054597496986389, + 4.94107186794281, + 5.090505480766296, + 5.040469527244568, + 4.986418008804321, + 4.865728974342346, + 4.884744882583618, + 4.8682403564453125, + 4.998300313949585, + 4.868122935295105, + 4.848920106887817, + 4.741814136505127, + 4.779753565788269, + 4.878525972366333, + 4.9051666259765625, + 4.994696736335754, + 4.918200731277466, + 4.883527517318726, + 4.939622759819031, + 4.7473790645599365, + 4.878680229187012, + 4.8146055936813354, + 4.820319175720215, + 4.708106756210327, + 4.7236762046813965, + 4.811763167381287, + 4.819777727127075, + 4.819828748703003, + 4.622236251831055, + 4.7038480043411255, + 4.7462393045425415, + 4.6595379114151, + 4.655217528343201, + 4.641505479812622, + 4.731655836105347, + 4.447583436965942, + 4.608594298362732, + 4.646321177482605, + 4.765825986862183, + 4.674468994140625, + 4.653892517089844, + 4.657962799072266, + 4.646577954292297, + 4.562385559082031, + 4.665558338165283, + 4.625023126602173, + 4.690134286880493, + 4.691829442977905, + 4.637716174125671, + 4.517543911933899, + 4.622917413711548, + 4.598248481750488, + 4.595747351646423, + 4.483474135398865, + 4.666769623756409, + 4.614009976387024, + 4.675269365310669, + 4.520530462265015, + 4.635284781455994, + 4.478766560554504, + 4.518543243408203, + 4.460464954376221, + 4.586846232414246, + 4.488665580749512, + 4.5659695863723755, + 4.455212712287903, + 4.5055800676345825, + 4.430027484893799, + 4.5640528202056885, + 4.3550320863723755, + 4.382484555244446, + 4.528549075126648, + 4.488765358924866, + 4.586813807487488, + 4.454076051712036, + 4.511904239654541, + 4.446958065032959, + 4.457014203071594, + 4.387677907943726, + 4.5435909032821655, + 4.3601202964782715, + 4.515772581100464, + 4.477276802062988, + 4.394829988479614, + 4.327962160110474, + 4.279119491577148, + 4.462505459785461, + 4.233981430530548, + 4.490887880325317, + 4.4542940855026245, + 4.351890206336975, + 4.340730428695679, + 4.38740861415863, + 4.340619206428528, + 4.2981051206588745, + 4.426837086677551, + 4.31691575050354, + 4.274929761886597, + 4.483382821083069, + 4.423704743385315, + 4.352145314216614, + 4.395002841949463, + 4.280207276344299, + 4.372983455657959, + 4.3703049421310425, + 4.530654430389404, + 4.3202691078186035, + 4.235048890113831, + 4.277849435806274, + 4.343338847160339, + 4.460234999656677, + 4.295498609542847, + 4.279441952705383, + 4.345138907432556, + 4.198278307914734, + 4.333808779716492, + 4.206830739974976, + 4.27922511100769, + 4.409626841545105, + 4.174767732620239, + 4.441091775894165, + 4.189976096153259, + 4.395759463310242, + 4.335185408592224, + 4.412021636962891, + 4.358713626861572, + 4.326831579208374, + 4.277551293373108, + 4.303967833518982, + 4.262058913707733, + 4.396944761276245, + 4.320154070854187, + 4.363680958747864, + 4.29715633392334, + 4.354065299034119, + 4.251940965652466, + 4.274547815322876, + 4.325574517250061, + 4.298179030418396, + 4.327387690544128, + 4.260580897331238, + 4.301640152931213, + 4.250829696655273, + 4.140902519226074, + 4.361799478530884, + 4.213551759719849, + 4.338292121887207, + 4.2012165784835815, + 4.234917163848877, + 4.409824728965759, + 4.212668418884277, + 4.123360633850098, + 4.101824402809143, + 4.328238129615784, + 4.091670095920563, + 4.2645286321640015, + 4.268937110900879, + 4.163565754890442, + 4.334329009056091, + 4.211902618408203, + 4.267238020896912, + 4.145011305809021, + 4.158883333206177, + 4.184983968734741, + 4.3186633586883545, + 4.183899402618408, + 4.181943416595459, + 4.155041038990021, + 4.272682189941406, + 4.22030508518219, + 4.227633833885193, + 4.222142457962036, + 4.237360000610352, + 4.186646580696106, + 4.116394758224487, + 4.155487298965454, + 4.068902373313904, + 4.132958292961121, + 4.098584413528442, + 4.211521148681641, + 4.094622373580933, + 4.258368611335754, + 4.167235493659973, + 4.128871560096741, + 4.268102526664734, + 4.225841641426086, + 4.219872236251831, + 4.166925370693207, + 4.184796094894409, + 4.015028655529022, + 4.159048914909363, + 4.241294622421265, + 4.175114870071411, + 3.9993191361427307, + 3.9808963537216187, + 4.061279773712158, + 3.9719104170799255, + 3.976190984249115, + 4.130991697311401, + 3.986549139022827, + 4.08984762430191, + 4.00348562002182, + 4.0401358008384705, + 4.088941752910614, + 4.157692968845367, + 4.05521959066391, + 3.9474469423294067, + 4.055486559867859, + 4.122287750244141, + 4.085309267044067, + 4.143974602222443, + 3.971006214618683, + 4.09621798992157, + 4.0364766120910645, + 4.107633531093597, + 4.068456053733826, + 4.13237339258194, + 4.101041793823242, + 3.9779937267303467, + 4.0938800573349, + 4.0451987981796265, + 4.037050366401672, + 4.0381805300712585, + 4.10213840007782, + 4.034334242343903, + 4.057974755764008, + 4.001451551914215, + 4.132445812225342, + 4.021785914897919, + 3.9969316124916077, + 4.1009151339530945, + 4.052601099014282, + 3.911613404750824, + 4.020875930786133, + 4.030506253242493, + 3.99321848154068, + 4.042596876621246, + 4.114438831806183, + 4.19671368598938, + 4.081141531467438, + 3.96740061044693, + 3.9296679496765137, + 4.069199800491333, + 3.931318998336792, + 4.0305609703063965, + 3.8930493593215942, + 3.990244209766388, + 4.191748857498169, + 4.077753841876984, + 4.082187175750732, + 3.9835193157196045, + 3.9238373041152954, + 4.031708836555481, + 3.9238845109939575, + 4.17694091796875, + 3.9211116433143616, + 4.00191056728363, + 4.038300037384033, + 4.013531565666199, + 4.044775426387787, + 4.071902275085449, + 3.925107002258301, + 3.9590609073638916, + 4.0579787492752075, + 3.9743329882621765, + 4.1148234605789185, + 4.11735475063324, + 3.9734129309654236, + 3.9315057396888733, + 3.9822184443473816, + 3.9964679479599, + 3.981339454650879, + 3.9715059399604797, + 4.01916229724884, + 3.9765742421150208, + 3.998846650123596, + 3.982913553714752, + 4.020352303981781, + 3.9619346261024475, + 3.9104753732681274, + 3.9827964901924133, + 3.901082158088684, + 4.013410925865173, + 3.9780531525611877, + 3.9442522525787354, + 3.936622679233551, + 3.9503027200698853, + 3.9268276691436768, + 4.053134500980377, + 4.027790784835815, + 4.015241980552673, + 3.926534116268158, + 4.02324903011322, + 3.9886816143989563, + 4.007500767707825, + 3.9279401898384094, + 3.9745272397994995, + 3.8971747756004333, + 4.008451342582703, + 3.969311773777008, + 4.014381766319275, + 4.0048105120658875, + 3.8248841762542725, + 3.9304445385932922, + 3.860976219177246, + 4.093779742717743, + 4.009447395801544, + 3.981579840183258, + 4.063955187797546, + 3.960124135017395, + 4.017957866191864, + 3.905344545841217, + 3.9734917283058167, + 3.930870771408081, + 3.9102163910865784, + 3.9459398984909058, + 3.8841881155967712, + 3.901680588722229, + 3.8844926953315735, + 3.85450541973114, + 3.868762254714966, + 3.849038779735565, + 3.9608052372932434, + 3.908285617828369, + 3.735906958580017, + 3.9117976427078247, + 3.8943446278572083, + 3.9814960956573486, + 3.828510105609894, + 3.7754790782928467, + 3.8880529403686523, + 3.77570778131485, + 3.9180965423583984, + 3.936352550983429, + 3.7441731691360474, + 3.859515070915222, + 3.85134357213974, + 3.9105767011642456, + 3.82525497674942, + 3.8298200368881226, + 3.908729612827301, + 3.8409796953201294, + 3.8021721243858337, + 3.967641592025757, + 3.7653477787971497, + 3.8634409308433533, + 3.7875683903694153, + 3.9544323682785034, + 3.8273357152938843, + 3.8873995542526245, + 3.7697811126708984, + 3.8831235766410828, + 3.8238242268562317, + 3.981654405593872, + 3.769556224346161, + 3.844018876552582, + 3.7846707701683044, + 3.840486526489258, + 3.7305290699005127, + 3.8112142086029053, + 3.9890562891960144, + 3.7684147357940674, + 3.743248999118805, + 3.889868140220642, + 3.709853947162628, + 3.8464664220809937, + 3.836733877658844, + 3.852900981903076, + 3.813481569290161, + 3.83096045255661, + 3.8663523197174072, + 3.921950101852417, + 3.8824111819267273, + 3.9816447496414185, + 3.87903094291687, + 3.7679736614227295, + 3.913914740085602, + 3.77968293428421, + 3.9202760457992554, + 3.7730923891067505, + 3.7392603158950806, + 3.8465271592140198, + 3.8879531621932983, + 3.9203349351882935, + 3.8982659578323364, + 3.995663285255432, + 3.827192544937134, + 3.912638306617737, + 3.9481894969940186, + 3.8643043637275696, + 3.878486752510071, + 3.8134310841560364, + 3.917311131954193, + 3.8638696670532227, + 3.8139460682868958, + 3.808407723903656, + 3.821552336215973, + 3.9162797331809998, + 3.8101231455802917, + 3.76029109954834, + 3.7465755343437195, + 3.7395451068878174, + 3.734034776687622, + 3.8053905367851257, + 3.9222505688667297, + 3.631527841091156, + 3.743738055229187, + 3.806977689266205, + 3.7482787370681763, + 3.641030490398407, + 3.6167112588882446, + 3.790681540966034, + 3.8096680641174316, + 3.9102283120155334, + 3.831805944442749, + 3.770898938179016, + 3.8068661093711853, + 3.847506284713745, + 3.7596651911735535, + 3.898711621761322, + 3.717009127140045, + 3.876401364803314, + 3.8014228343963623, + 3.8021788001060486, + 3.8262019753456116, + 3.8808408975601196, + 3.7061957120895386, + 3.642726242542267, + 3.86419016122818, + 3.8664931654930115, + 3.7048638463020325, + 3.822018802165985, + 3.9453471899032593, + 3.817805230617523, + 3.8552945852279663, + 3.6900516152381897, + 3.7599873542785645, + 3.738908350467682, + 3.805435597896576, + 3.7034677863121033, + 3.7725711464881897, + 3.8673095107078552, + 3.649045169353485, + 3.8983787298202515, + 3.794239819049835, + 3.7457557916641235, + 3.6139431595802307, + 3.6919734477996826, + 3.7692940831184387, + 3.704489290714264, + 3.7744147181510925, + 3.710189402103424, + 3.507409453392029, + 3.6841604113578796, + 3.7220422625541687, + 3.5961544513702393, + 3.75495982170105, + 3.660954713821411, + 3.84284508228302, + 3.6808032989501953, + 3.7655239701271057, + 3.6605404019355774, + 3.7357890605926514, + 3.754780411720276, + 3.793479561805725, + 3.8042609691619873, + 3.741646111011505, + 3.7268142104148865, + 3.5669058561325073, + 3.6847500801086426, + 3.6856184601783752, + 3.8564152121543884, + 3.740295886993408, + 3.730651080608368, + 3.7532607316970825, + 3.5919845700263977, + 3.691934883594513, + 3.767329216003418, + 3.762724816799164, + 3.732990026473999, + 3.742927849292755, + 3.716303050518036, + 3.681581735610962, + 3.6820526719093323, + 3.7312793135643005, + 3.722298264503479, + 3.761793553829193, + 3.759587526321411, + 3.796225905418396, + 3.7062246203422546, + 3.582374691963196, + 3.8044768571853638, + 3.670083463191986, + 3.654890716075897, + 3.639878809452057, + 3.6637874841690063, + 3.757590115070343, + 3.6329495310783386, + 3.7597668170928955, + 3.796943783760071, + 3.681839942932129, + 3.6659743785858154, + 3.6895888447761536, + 3.686311960220337, + 3.7957123517990112, + 3.7674129009246826, + 3.728692889213562, + 3.762762486934662, + 3.752412438392639, + 3.7462961673736572, + 3.7190808057785034, + 3.798906147480011, + 3.8056862354278564, + 3.710305392742157, + 3.8287219405174255, + 3.90934956073761, + 3.63561087846756, + 3.6585183143615723, + 3.8162046670913696, + 3.6108638644218445, + 3.8571974635124207, + 3.7285009622573853, + 3.62653911113739, + 3.8645923733711243, + 3.6878238320350647, + 3.7690871357917786, + 3.759834110736847, + 3.5893213152885437, + 3.5242881774902344, + 3.627947449684143, + 3.7712064385414124, + 3.7437827587127686, + 3.741795837879181, + 3.7771612405776978, + 3.7021942138671875, + 3.754234552383423, + 3.6994784474372864, + 3.737087368965149, + 3.6001042127609253, + 3.69114351272583, + 3.6961554884910583, + 3.6451334953308105, + 3.752415955066681, + 3.6447402238845825, + 3.7157459259033203, + 3.776822566986084, + 3.7850656509399414, + 3.7144275307655334, + 3.7662808299064636, + 3.6011103987693787, + 3.7692585587501526, + 3.755720019340515, + 3.7321547269821167, + 3.7682764530181885, + 3.69218373298645, + 3.645968735218048, + 3.65865695476532, + 3.741810977458954, + 3.6941879391670227, + 3.600189447402954, + 3.662043333053589, + 3.7190656065940857, + 3.5978373885154724, + 3.6870716214179993, + 3.6737056970596313, + 3.6627723574638367, + 3.545480728149414, + 3.7313902378082275, + 3.582738220691681, + 3.806025445461273, + 3.6314245462417603, + 3.5855454206466675, + 3.5500744581222534, + 3.584427833557129, + 3.679089844226837, + 3.70182728767395, + 3.6024619340896606, + 3.6764765977859497, + 3.522747278213501, + 3.559336841106415, + 3.63923317193985, + 3.671720504760742, + 3.5141653418540955, + 3.4913381934165955, + 3.71255099773407, + 3.6856289505958557, + 3.674484431743622, + 3.6556333899497986, + 3.5683552622795105, + 3.662504255771637, + 3.7524858117103577, + 3.567614793777466, + 3.531031608581543, + 3.5837584137916565, + 3.656033158302307, + 3.653377413749695, + 3.6099491715431213, + 3.6124138236045837, + 3.5901744961738586, + 3.649082601070404, + 3.6227990984916687, + 3.562268018722534, + 3.6124784350395203, + 3.6126555800437927, + 3.624067962169647, + 3.5443370938301086, + 3.6476243138313293, + 3.612736403942108, + 3.6684531569480896, + 3.6294094920158386, + 3.506846070289612, + 3.5762569308280945, + 3.5933576226234436, + 3.6100088953971863, + 3.6335771083831787, + 3.588502585887909, + 3.669529914855957, + 3.515492081642151, + 3.7027518153190613, + 3.677309989929199, + 3.6751623153686523, + 3.551825165748596, + 3.5721518993377686, + 3.6048982739448547, + 3.6810749173164368, + 3.457333981990814, + 3.8115950226783752, + 3.4852503538131714, + 3.600753664970398, + 3.595928370952606, + 3.622970163822174, + 3.5986160039901733, + 3.705034375190735, + 3.6212212443351746, + 3.658156991004944, + 3.5692983269691467, + 3.745928406715393, + 3.4987694025039673, + 3.622099816799164, + 3.53867769241333, + 3.606979489326477, + 3.7358133792877197, + 3.713529586791992, + 3.621503233909607, + 3.485220491886139, + 3.6914690732955933, + 3.5647152066230774, + 3.5147659182548523, + 3.5625763535499573, + 3.560496211051941, + 3.5898629426956177, + 3.638286530971527, + 3.607525944709778, + 3.6219797134399414, + 3.622349977493286, + 3.540381133556366, + 3.7234543561935425, + 3.596914052963257, + 3.678851902484894, + 3.638051152229309, + 3.627170145511627, + 3.5754899978637695, + 3.603741765022278, + 3.5655710101127625, + 3.4626460671424866, + 3.6013890504837036, + 3.5344597697257996, + 3.5399850606918335, + 3.5248823165893555, + 3.780304729938507, + 3.672513484954834, + 3.531547427177429, + 3.5409721732139587, + 3.687939763069153, + 3.651773691177368, + 3.6981360912323, + 3.564858317375183, + 3.622052252292633, + 3.693567633628845, + 3.6521021127700806, + 3.5408633947372437, + 3.5831121802330017, + 3.6643746495246887, + 3.596319615840912, + 3.429449498653412, + 3.7171595692634583, + 3.6085683703422546, + 3.5477556586265564, + 3.6313828825950623, + 3.5756876468658447, + 3.4314074516296387, + 3.5710368752479553, + 3.5584158301353455, + 3.5795644521713257, + 3.502845048904419, + 3.573672294616699, + 3.649259030818939, + 3.6510260701179504, + 3.580295145511627, + 3.51831716299057, + 3.5726168155670166, + 3.533908486366272, + 3.431344985961914, + 3.5312135815620422, + 3.479523539543152, + 3.6915079951286316, + 3.42625230550766, + 3.5768306851387024, + 3.638262629508972, + 3.5112180709838867, + 3.5891258120536804, + 3.544898509979248, + 3.4029486179351807, + 3.5566001534461975, + 3.587431788444519, + 3.6390103697776794, + 3.58359557390213, + 3.4817785024642944, + 3.5866506099700928, + 3.5742496252059937, + 3.544652581214905, + 3.6005988717079163, + 3.4106934666633606, + 3.615422546863556, + 3.679928183555603, + 3.599842369556427, + 3.5555304288864136, + 3.4652220606803894, + 3.4634650349617004, + 3.4584545493125916, + 3.594955027103424, + 3.6249500513076782, + 3.616303563117981, + 3.4747042059898376, + 3.564768612384796, + 3.641221761703491, + 3.4873066544532776, + 3.6220430731773376, + 3.6099568605422974, + 3.580629825592041, + 3.4909472465515137, + 3.5997923016548157, + 3.587445855140686, + 3.5075010657310486, + 3.5081303119659424, + 3.5323182344436646, + 3.6663320064544678, + 3.575871467590332, + 3.6088608503341675, + 3.4966158866882324, + 3.5934624671936035, + 3.4776411056518555, + 3.587721824645996, + 3.5898842215538025, + 3.4446999430656433, + 3.559806704521179, + 3.4825318455696106, + 3.546854853630066, + 3.6903722286224365, + 3.5998719930648804, + 3.4635772109031677, + 3.522242546081543, + 3.448143482208252, + 3.4691681265830994, + 3.597661793231964, + 3.552363097667694, + 3.5866158604621887, + 3.6768017411231995, + 3.623350143432617, + 3.5804836750030518, + 3.564529001712799, + 3.5073920488357544, + 3.5023159980773926, + 3.5961198806762695, + 3.435886323451996, + 3.619915544986725, + 3.5011886954307556, + 3.4939733743667603, + 3.493607997894287, + 3.4921681880950928, + 3.5897818207740784, + 3.4700319170951843, + 3.6970936059951782, + 3.4979161620140076, + 3.51347953081131, + 3.514089584350586, + 3.5944236516952515, + 3.4338855147361755, + 3.5721628069877625, + 3.5652135610580444, + 3.560681462287903, + 3.578468918800354, + 3.434243083000183, + 3.562955856323242, + 3.4981409311294556, + 3.570349931716919, + 3.5606396198272705, + 3.532760739326477, + 3.6193867325782776, + 3.574935257434845, + 3.429373264312744, + 3.5525636672973633, + 3.596983253955841, + 3.549538314342499, + 3.5723623633384705, + 3.516147792339325, + 3.4458229541778564, + 3.7076441049575806, + 3.607069969177246, + 3.6197968125343323, + 3.5799888968467712, + 3.593482553958893, + 3.5083532333374023, + 3.6531346440315247, + 3.4723575711250305, + 3.6018084287643433, + 3.5737420320510864, + 3.6480382680892944, + 3.6529972553253174, + 3.5666415095329285, + 3.6317354440689087, + 3.478222668170929, + 3.6198890209198, + 3.519287347793579, + 3.5107840299606323, + 3.4638413190841675, + 3.4320979714393616, + 3.5009323358535767, + 3.5206227898597717, + 3.415836215019226, + 3.4117767810821533, + 3.4593698382377625, + 3.5279542803764343, + 3.5435128211975098, + 3.494676649570465, + 3.540445387363434, + 3.468340754508972, + 3.6338070034980774, + 3.524068832397461, + 3.551423728466034, + 3.4649844765663147, + 3.4646684527397156, + 3.4596731662750244, + 3.4699859619140625, + 3.493991434574127, + 3.510548233985901, + 3.5593294501304626, + 3.5162285566329956, + 3.4183799624443054, + 3.6468921303749084, + 3.480936050415039, + 3.535964250564575, + 3.4276192784309387, + 3.6156625151634216, + 3.582808256149292, + 3.440723478794098, + 3.3926047682762146, + 3.4583149552345276, + 3.5006008744239807, + 3.434778392314911, + 3.4018885493278503, + 3.542311668395996, + 3.3753889203071594, + 3.3164941668510437, + 3.5626527070999146, + 3.5052505135536194, + 3.5345972180366516, + 3.46654736995697, + 3.5381049513816833, + 3.676484763622284, + 3.507034182548523, + 3.583761751651764, + 3.584146201610565, + 3.5232813954353333, + 3.6428425908088684, + 3.670961618423462, + 3.610192835330963, + 3.527381658554077, + 3.420125722885132, + 3.6022021174430847, + 3.529938220977783, + 3.578514039516449, + 3.515637457370758, + 3.464073598384857, + 3.575469970703125, + 3.5441725850105286, + 3.353513479232788, + 3.600390315055847, + 3.5933098793029785, + 3.5725865364074707, + 3.6364001035690308, + 3.4574407935142517, + 3.5403501987457275, + 3.616316497325897, + 3.5697805285453796, + 3.4412747621536255, + 3.6130826473236084, + 3.6914462447166443, + 3.541974127292633, + 3.5264499187469482, + 3.433497667312622, + 3.604211986064911, + 3.553445816040039, + 3.4537035822868347, + 3.538024604320526, + 3.613723874092102, + 3.471800744533539, + 3.516095817089081, + 3.5176376700401306, + 3.562239110469818, + 3.532831132411957, + 3.4970932006835938, + 3.678560495376587, + 3.468765914440155, + 3.5826455950737, + 3.4699469804763794, + 3.5991539359092712, + 3.544541895389557, + 3.5529040098190308, + 3.5550767183303833, + 3.591276526451111, + 3.420476496219635, + 3.5027984380722046, + 3.572746157646179, + 3.477246642112732, + 3.595560133457184, + 3.440320909023285, + 3.612122118473053, + 3.4842042326927185, + 3.524068295955658, + 3.606152355670929, + 3.6424405574798584, + 3.5308287739753723, + 3.4965116381645203, + 3.5896754264831543, + 3.5181740522384644, + 3.49838387966156, + 3.5535452365875244, + 3.529670298099518, + 3.455875039100647 + ], + "val_steps": [ + 2500, + 5000, + 7500, + 10000, + 12500, + 15000, + 17500, + 20000, + 22500, + 25000, + 27500, + 30000, + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_lm_loss": [ + 4.998014114521168, + 4.527697368904397, + 4.292870468563503, + 4.145470610371342, + 4.03356029369213, + 3.9537708671004683, + 3.8828854031032987, + 3.8327478920971907, + 3.778701587959572, + 3.7307776433450206, + 3.699858206289786, + 3.662911238493743, + 3.6372606312787092, + 3.6101932525634766, + 3.5875458452436657, + 3.575983895195855, + 3.5703620822341353, + 3.568719643133658, + 3.564243864130091, + 3.5563294092814126 + ], + "val_ppl": [ + 148.11872002295948, + 92.5452180543895, + 73.17621734836877, + 63.147332699782964, + 56.461573611067614, + 52.1315779114255, + 48.564140224943564, + 46.18928718546973, + 43.75918737759952, + 41.71153225067306, + 40.44156959330056, + 38.97464234949844, + 37.98763193217647, + 36.97319728961238, + 36.14526092415712, + 35.72975786434643, + 35.52945540748234, + 35.47114833671631, + 35.31274207590715, + 35.03436403149534 + ] +} \ No newline at end of file diff --git a/075_cosmopedia_mol/dense_softmax_best.pt b/075_cosmopedia_mol/dense_softmax_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..a89f96dbf26f85cf8ffa86b5968bef62fd2fc173 --- /dev/null +++ b/075_cosmopedia_mol/dense_softmax_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6048f193ad8cea66a25cf814c3fe18027ae03b19d600486ae52312650156ac5 +size 413864878 diff --git a/075_cosmopedia_mol/dense_softmax_results.json b/075_cosmopedia_mol/dense_softmax_results.json new file mode 100644 index 0000000000000000000000000000000000000000..919a37cb0a1844b6a15ed80f2a92b972c28e7f39 --- /dev/null +++ b/075_cosmopedia_mol/dense_softmax_results.json @@ -0,0 +1,4644 @@ +{ + "label": "dense_softmax", + "condition": "dense_softmax", + "total_params": 103454464, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000, + 50050, + 50100, + 50150, + 50200, + 50250, + 50300, + 50350, + 50400, + 50450, + 50500, + 50550, + 50600, + 50650, + 50700, + 50750, + 50800, + 50850, + 50900, + 50950, + 51000, + 51050, + 51100, + 51150, + 51200, + 51250, + 51300, + 51350, + 51400, + 51450, + 51500, + 51550, + 51600, + 51650, + 51700, + 51750, + 51800, + 51850, + 51900, + 51950, + 52000, + 52050, + 52100, + 52150, + 52200, + 52250, + 52300, + 52350, + 52400, + 52450, + 52500, + 52550, + 52600, + 52650, + 52700, + 52750, + 52800, + 52850, + 52900, + 52950, + 53000, + 53050, + 53100, + 53150, + 53200, + 53250, + 53300, + 53350, + 53400, + 53450, + 53500, + 53550, + 53600, + 53650, + 53700, + 53750, + 53800, + 53850, + 53900, + 53950, + 54000, + 54050, + 54100, + 54150, + 54200, + 54250, + 54300, + 54350, + 54400, + 54450, + 54500, + 54550, + 54600, + 54650, + 54700, + 54750, + 54800, + 54850, + 54900, + 54950, + 55000, + 55050, + 55100, + 55150, + 55200, + 55250, + 55300, + 55350, + 55400, + 55450, + 55500, + 55550, + 55600, + 55650, + 55700, + 55750, + 55800, + 55850, + 55900, + 55950, + 56000, + 56050, + 56100, + 56150, + 56200, + 56250, + 56300, + 56350, + 56400, + 56450, + 56500, + 56550, + 56600, + 56650, + 56700, + 56750, + 56800, + 56850, + 56900, + 56950, + 57000, + 57050, + 57100, + 57150, + 57200, + 57250, + 57300, + 57350, + 57400, + 57450, + 57500, + 57550, + 57600, + 57650, + 57700, + 57750, + 57800, + 57850, + 57900, + 57950, + 58000, + 58050, + 58100, + 58150, + 58200, + 58250, + 58300, + 58350, + 58400, + 58450, + 58500, + 58550, + 58600, + 58650, + 58700, + 58750, + 58800, + 58850, + 58900, + 58950, + 59000, + 59050, + 59100, + 59150, + 59200, + 59250, + 59300, + 59350, + 59400, + 59450, + 59500, + 59550, + 59600, + 59650, + 59700, + 59750, + 59800, + 59850, + 59900, + 59950, + 60000, + 60050, + 60100, + 60150, + 60200, + 60250, + 60300, + 60350, + 60400, + 60450, + 60500, + 60550, + 60600, + 60650, + 60700, + 60750, + 60800, + 60850, + 60900, + 60950, + 61000, + 61050, + 61100, + 61150, + 61200, + 61250, + 61300, + 61350, + 61400, + 61450, + 61500, + 61550, + 61600, + 61650, + 61700, + 61750, + 61800, + 61850, + 61900, + 61950, + 62000, + 62050, + 62100, + 62150, + 62200, + 62250, + 62300, + 62350, + 62400, + 62450, + 62500, + 62550, + 62600, + 62650, + 62700, + 62750, + 62800, + 62850, + 62900, + 62950, + 63000, + 63050, + 63100, + 63150, + 63200, + 63250, + 63300, + 63350, + 63400, + 63450, + 63500, + 63550, + 63600, + 63650, + 63700, + 63750, + 63800, + 63850, + 63900, + 63950, + 64000, + 64050, + 64100, + 64150, + 64200, + 64250, + 64300, + 64350, + 64400, + 64450, + 64500, + 64550, + 64600, + 64650, + 64700, + 64750, + 64800, + 64850, + 64900, + 64950, + 65000, + 65050, + 65100, + 65150, + 65200, + 65250, + 65300, + 65350, + 65400, + 65450, + 65500, + 65550, + 65600, + 65650, + 65700, + 65750, + 65800, + 65850, + 65900, + 65950, + 66000, + 66050, + 66100, + 66150, + 66200, + 66250, + 66300, + 66350, + 66400, + 66450, + 66500, + 66550, + 66600, + 66650, + 66700, + 66750, + 66800, + 66850, + 66900, + 66950, + 67000, + 67050, + 67100, + 67150, + 67200, + 67250, + 67300, + 67350, + 67400, + 67450, + 67500, + 67550, + 67600, + 67650, + 67700, + 67750, + 67800, + 67850, + 67900, + 67950, + 68000, + 68050, + 68100, + 68150, + 68200, + 68250, + 68300, + 68350, + 68400, + 68450, + 68500, + 68550, + 68600, + 68650, + 68700, + 68750, + 68800, + 68850, + 68900, + 68950, + 69000, + 69050, + 69100, + 69150, + 69200, + 69250, + 69300, + 69350, + 69400, + 69450, + 69500, + 69550, + 69600, + 69650, + 69700, + 69750, + 69800, + 69850, + 69900, + 69950, + 70000, + 70050, + 70100, + 70150, + 70200, + 70250, + 70300, + 70350, + 70400, + 70450, + 70500, + 70550, + 70600, + 70650, + 70700, + 70750, + 70800, + 70850, + 70900, + 70950, + 71000, + 71050, + 71100, + 71150, + 71200, + 71250, + 71300, + 71350, + 71400, + 71450, + 71500, + 71550, + 71600, + 71650, + 71700, + 71750, + 71800, + 71850, + 71900, + 71950, + 72000, + 72050, + 72100, + 72150, + 72200, + 72250, + 72300, + 72350, + 72400, + 72450, + 72500, + 72550, + 72600, + 72650, + 72700, + 72750, + 72800, + 72850, + 72900, + 72950, + 73000, + 73050, + 73100, + 73150, + 73200, + 73250, + 73300, + 73350, + 73400, + 73450, + 73500, + 73550, + 73600, + 73650, + 73700, + 73750, + 73800, + 73850, + 73900, + 73950, + 74000, + 74050, + 74100, + 74150, + 74200, + 74250, + 74300, + 74350, + 74400, + 74450, + 74500, + 74550, + 74600, + 74650, + 74700, + 74750, + 74800, + 74850, + 74900, + 74950, + 75000, + 75050, + 75100, + 75150, + 75200, + 75250, + 75300, + 75350, + 75400, + 75450, + 75500, + 75550, + 75600, + 75650, + 75700, + 75750, + 75800, + 75850, + 75900, + 75950, + 76000, + 76050, + 76100, + 76150, + 76200, + 76250, + 76300, + 76350, + 76400, + 76450, + 76500, + 76550, + 76600, + 76650, + 76700, + 76750, + 76800, + 76850, + 76900, + 76950, + 77000, + 77050, + 77100, + 77150, + 77200, + 77250, + 77300, + 77350, + 77400, + 77450, + 77500, + 77550, + 77600, + 77650, + 77700, + 77750, + 77800, + 77850, + 77900, + 77950, + 78000, + 78050, + 78100, + 78150, + 78200, + 78250, + 78300, + 78350, + 78400, + 78450, + 78500, + 78550, + 78600, + 78650, + 78700, + 78750, + 78800, + 78850, + 78900, + 78950, + 79000, + 79050, + 79100, + 79150, + 79200, + 79250, + 79300, + 79350, + 79400, + 79450, + 79500, + 79550, + 79600, + 79650, + 79700, + 79750, + 79800, + 79850, + 79900, + 79950, + 80000, + 80050, + 80100, + 80150, + 80200, + 80250, + 80300, + 80350, + 80400, + 80450, + 80500, + 80550, + 80600, + 80650, + 80700, + 80750, + 80800, + 80850, + 80900, + 80950, + 81000, + 81050, + 81100, + 81150, + 81200, + 81250, + 81300, + 81350, + 81400, + 81450, + 81500, + 81550, + 81600, + 81650, + 81700, + 81750, + 81800, + 81850, + 81900, + 81950, + 82000, + 82050, + 82100, + 82150, + 82200, + 82250, + 82300, + 82350, + 82400, + 82450, + 82500, + 82550, + 82600, + 82650, + 82700, + 82750, + 82800, + 82850, + 82900, + 82950, + 83000, + 83050, + 83100, + 83150, + 83200, + 83250, + 83300, + 83350, + 83400, + 83450, + 83500, + 83550, + 83600, + 83650, + 83700, + 83750, + 83800, + 83850, + 83900, + 83950, + 84000, + 84050, + 84100, + 84150, + 84200, + 84250, + 84300, + 84350, + 84400, + 84450, + 84500, + 84550, + 84600, + 84650, + 84700, + 84750, + 84800, + 84850, + 84900, + 84950, + 85000, + 85050, + 85100, + 85150, + 85200, + 85250, + 85300, + 85350, + 85400, + 85450, + 85500, + 85550, + 85600, + 85650, + 85700, + 85750, + 85800, + 85850, + 85900, + 85950, + 86000, + 86050, + 86100, + 86150, + 86200, + 86250, + 86300, + 86350, + 86400, + 86450, + 86500, + 86550, + 86600, + 86650, + 86700, + 86750, + 86800, + 86850, + 86900, + 86950, + 87000, + 87050, + 87100, + 87150, + 87200, + 87250, + 87300, + 87350, + 87400, + 87450, + 87500, + 87550, + 87600, + 87650, + 87700, + 87750, + 87800, + 87850, + 87900, + 87950, + 88000, + 88050, + 88100, + 88150, + 88200, + 88250, + 88300, + 88350, + 88400, + 88450, + 88500, + 88550, + 88600, + 88650, + 88700, + 88750, + 88800, + 88850, + 88900, + 88950, + 89000, + 89050, + 89100, + 89150, + 89200, + 89250, + 89300, + 89350, + 89400, + 89450, + 89500, + 89550, + 89600, + 89650, + 89700, + 89750, + 89800, + 89850, + 89900, + 89950, + 90000, + 90050, + 90100, + 90150, + 90200, + 90250, + 90300, + 90350, + 90400, + 90450, + 90500, + 90550, + 90600, + 90650, + 90700, + 90750, + 90800, + 90850, + 90900, + 90950, + 91000, + 91050, + 91100, + 91150, + 91200, + 91250, + 91300, + 91350, + 91400, + 91450, + 91500, + 91550, + 91600, + 91650, + 91700, + 91750, + 91800, + 91850, + 91900, + 91950, + 92000, + 92050, + 92100, + 92150, + 92200, + 92250, + 92300, + 92350, + 92400, + 92450, + 92500, + 92550, + 92600, + 92650, + 92700, + 92750, + 92800, + 92850, + 92900, + 92950, + 93000, + 93050, + 93100, + 93150, + 93200, + 93250, + 93300, + 93350, + 93400, + 93450, + 93500, + 93550, + 93600, + 93650, + 93700, + 93750, + 93800, + 93850, + 93900, + 93950, + 94000, + 94050, + 94100, + 94150, + 94200, + 94250, + 94300, + 94350, + 94400, + 94450, + 94500, + 94550, + 94600, + 94650, + 94700, + 94750, + 94800, + 94850, + 94900, + 94950, + 95000, + 95050, + 95100, + 95150, + 95200, + 95250, + 95300, + 95350, + 95400, + 95450, + 95500, + 95550, + 95600, + 95650, + 95700, + 95750, + 95800, + 95850, + 95900, + 95950, + 96000, + 96050, + 96100, + 96150, + 96200, + 96250, + 96300, + 96350, + 96400, + 96450, + 96500, + 96550, + 96600, + 96650, + 96700, + 96750, + 96800, + 96850, + 96900, + 96950, + 97000, + 97050, + 97100, + 97150, + 97200, + 97250, + 97300, + 97350, + 97400, + 97450, + 97500, + 97550, + 97600, + 97650, + 97700, + 97750, + 97800, + 97850, + 97900, + 97950, + 98000, + 98050, + 98100, + 98150, + 98200, + 98250, + 98300, + 98350, + 98400, + 98450, + 98500, + 98550, + 98600, + 98650, + 98700, + 98750, + 98800, + 98850, + 98900, + 98950, + 99000, + 99050, + 99100, + 99150, + 99200, + 99250, + 99300, + 99350, + 99400, + 99450, + 99500, + 99550, + 99600, + 99650, + 99700, + 99750, + 99800, + 99850, + 99900, + 99950, + 100000, + 100050, + 100100, + 100150, + 100200, + 100250, + 100300, + 100350, + 100400, + 100450, + 100500, + 100550, + 100600, + 100650, + 100700, + 100750, + 100800, + 100850, + 100900, + 100950, + 101000, + 101050, + 101100, + 101150, + 101200, + 101250, + 101300, + 101350, + 101400, + 101450, + 101500, + 101550, + 101600, + 101650, + 101700, + 101750, + 101800, + 101850, + 101900, + 101950, + 102000, + 102050, + 102100, + 102150, + 102200, + 102250, + 102300, + 102350, + 102400, + 102450, + 102500, + 102550, + 102600, + 102650, + 102700, + 102750, + 102800, + 102850, + 102900, + 102950, + 103000, + 103050, + 103100, + 103150, + 103200, + 103250, + 103300, + 103350, + 103400, + 103450, + 103500, + 103550, + 103600, + 103650, + 103700, + 103750, + 103800, + 103850, + 103900, + 103950, + 104000, + 104050, + 104100, + 104150, + 104200, + 104250, + 104300, + 104350, + 104400, + 104450, + 104500, + 104550, + 104600, + 104650, + 104700, + 104750, + 104800, + 104850, + 104900, + 104950, + 105000, + 105050, + 105100, + 105150, + 105200, + 105250, + 105300, + 105350, + 105400, + 105450, + 105500, + 105550, + 105600, + 105650, + 105700, + 105750, + 105800, + 105850, + 105900, + 105950, + 106000, + 106050, + 106100, + 106150, + 106200, + 106250, + 106300, + 106350, + 106400, + 106450, + 106500, + 106550, + 106600, + 106650, + 106700, + 106750, + 106800, + 106850, + 106900, + 106950, + 107000, + 107050, + 107100, + 107150, + 107200, + 107250, + 107300, + 107350, + 107400, + 107450, + 107500, + 107550, + 107600, + 107650, + 107700, + 107750, + 107800, + 107850, + 107900, + 107950, + 108000, + 108050, + 108100, + 108150, + 108200, + 108250, + 108300, + 108350, + 108400, + 108450, + 108500, + 108550, + 108600, + 108650, + 108700, + 108750, + 108800, + 108850, + 108900, + 108950, + 109000, + 109050, + 109100, + 109150, + 109200, + 109250, + 109300, + 109350, + 109400, + 109450, + 109500, + 109550, + 109600, + 109650, + 109700, + 109750, + 109800, + 109850, + 109900, + 109950, + 110000, + 110050, + 110100, + 110150, + 110200, + 110250, + 110300, + 110350, + 110400, + 110450, + 110500, + 110550, + 110600, + 110650, + 110700, + 110750, + 110800, + 110850, + 110900, + 110950, + 111000, + 111050, + 111100, + 111150, + 111200, + 111250, + 111300, + 111350, + 111400, + 111450, + 111500, + 111550, + 111600, + 111650, + 111700, + 111750, + 111800, + 111850, + 111900, + 111950, + 112000, + 112050, + 112100, + 112150, + 112200, + 112250, + 112300, + 112350, + 112400, + 112450, + 112500, + 112550, + 112600, + 112650, + 112700, + 112750, + 112800, + 112850, + 112900, + 112950, + 113000, + 113050, + 113100, + 113150, + 113200, + 113250, + 113300, + 113350, + 113400, + 113450, + 113500, + 113550, + 113600, + 113650, + 113700, + 113750, + 113800, + 113850, + 113900, + 113950, + 114000 + ], + "train_lm_loss": [ + 16.85776388645172, + 9.1920365691185, + 8.236555099487305, + 7.667818605899811, + 7.356138646602631, + 6.91735503077507, + 6.5906194150447845, + 6.357409775257111, + 5.9795006811618805, + 5.809264838695526, + 5.597225368022919, + 5.560283422470093, + 5.3338786363601685, + 5.267808616161346, + 5.115076541900635, + 4.98852214217186, + 4.868507653474808, + 4.767012357711792, + 4.688456177711487, + 4.64098647236824, + 4.524220407009125, + 4.418423891067505, + 4.384797319769859, + 4.303107738494873, + 4.316871553659439, + 4.174407556653023, + 4.164414927363396, + 4.08595110476017, + 3.997199073433876, + 3.9652142077684402, + 3.9132973700761795, + 3.8650206327438354, + 3.8475303798913956, + 3.7618423849344254, + 3.678696885704994, + 3.6759158074855804, + 3.670139864087105, + 3.6325187981128693, + 3.4887911528348923, + 3.516070142388344, + 3.537578210234642, + 3.5044918209314346, + 3.4151192903518677, + 3.370710387825966, + 3.3347098231315613, + 3.2379901856184006, + 3.281196743249893, + 3.2613525092601776, + 3.1993547677993774, + 3.2222185879945755, + 3.163980722427368, + 3.1286679059267044, + 3.1149380207061768, + 3.1066831797361374, + 3.0523914992809296, + 3.0641421526670456, + 3.024475395679474, + 3.062043696641922, + 3.028936266899109, + 2.9895512014627457, + 3.0482160300016403, + 2.9303646236658096, + 2.971750780940056, + 2.9578605741262436, + 2.9189397245645523, + 2.9536808878183365, + 2.8515406101942062, + 2.907557427883148, + 2.932330623269081, + 2.887909010052681, + 2.8449977338314056, + 2.8843569308519363, + 2.8721764087677, + 2.831746518611908, + 2.8095872551202774, + 2.8009496927261353, + 2.8490505516529083, + 2.8373938500881195, + 2.793259710073471, + 2.8101100027561188, + 2.7189568877220154, + 2.816136732697487, + 2.772700607776642, + 2.767932802438736, + 2.7938837856054306, + 2.7987180352211, + 2.7302757054567337, + 2.734313413500786, + 2.7649957835674286, + 2.7606468200683594, + 2.7491340339183807, + 2.735682189464569, + 2.698896110057831, + 2.69393190741539, + 2.6842608600854874, + 2.737307235598564, + 2.6433125138282776, + 2.688019245862961, + 2.6915260702371597, + 2.726158991456032, + 2.721269726753235, + 2.6706016063690186, + 2.640561029314995, + 2.683850422501564, + 2.6450803130865097, + 2.6723240315914154, + 2.655262067914009, + 2.6214464604854584, + 2.6437657177448273, + 2.654960334300995, + 2.6385307759046555, + 2.664942443370819, + 2.5943307876586914, + 2.608480378985405, + 2.5871263444423676, + 2.6296952217817307, + 2.606399968266487, + 2.5702600181102753, + 2.5649630576372147, + 2.6258813589811325, + 2.612287700176239, + 2.5828314274549484, + 2.5828361809253693, + 2.6167477816343307, + 2.5987306386232376, + 2.581043764948845, + 2.574723780155182, + 2.5324336737394333, + 2.588245615363121, + 2.581897422671318, + 2.579303488135338, + 2.5947986245155334, + 2.5624439120292664, + 2.511168286204338, + 2.57496140897274, + 2.51095013320446, + 2.494741827249527, + 2.5473780632019043, + 2.5445783883333206, + 2.5034898072481155, + 2.5067491084337234, + 2.509915590286255, + 2.551539584994316, + 2.5482055246829987, + 2.487907648086548, + 2.5222453474998474, + 2.5030168890953064, + 2.516479045152664, + 2.4786339849233627, + 2.4720835387706757, + 2.448873147368431, + 2.5260716378688812, + 2.494235947728157, + 2.454839378595352, + 2.4436618834733963, + 2.4883143305778503, + 2.4703138768672943, + 2.455879643559456, + 2.4201176837086678, + 2.4733757823705673, + 2.4871658831834793, + 2.4897836446762085, + 2.461534410715103, + 2.4821470826864243, + 2.480172038078308, + 2.43720743060112, + 2.4454943984746933, + 2.465492755174637, + 2.392948232591152, + 2.4018465727567673, + 2.499534845352173, + 2.437292605638504, + 2.4908596575260162, + 2.452893301844597, + 2.3977115750312805, + 2.4772768020629883, + 2.44956037402153, + 2.465890571475029, + 2.452250823378563, + 2.4566021114587784, + 2.4145552217960358, + 2.4989123195409775, + 2.4236185401678085, + 2.3975373655557632, + 2.4029418379068375, + 2.435733437538147, + 2.509815827012062, + 2.441146641969681, + 2.379582941532135, + 2.364986151456833, + 2.473151907324791, + 2.458905413746834, + 2.4218714386224747, + 2.4493881464004517, + 2.4775084406137466, + 2.4164696037769318, + 2.412929058074951, + 2.4309205561876297, + 2.410118341445923, + 2.459929570555687, + 2.385904550552368, + 2.3952574655413628, + 2.3985042423009872, + 2.406740754842758, + 2.337435334920883, + 2.368526741862297, + 2.443936660885811, + 2.346149057149887, + 2.439452052116394, + 2.4238839894533157, + 2.413376420736313, + 2.4044146984815598, + 2.4058752804994583, + 2.404824510216713, + 2.388056233525276, + 2.342642232775688, + 2.434034898877144, + 2.400403618812561, + 2.323973849415779, + 2.3582773208618164, + 2.3917603194713593, + 2.3896029889583588, + 2.419161394238472, + 2.351523384451866, + 2.3721239864826202, + 2.312861815094948, + 2.3666305989027023, + 2.3738170862197876, + 2.385918766260147, + 2.3709674328565598, + 2.3382931649684906, + 2.3703400641679764, + 2.3900093734264374, + 2.3563155084848404, + 2.3841100484132767, + 2.364271178841591, + 2.332897439599037, + 2.3554178327322006, + 2.404633581638336, + 2.4054225087165833, + 2.3336242139339447, + 2.2820213437080383, + 2.4094929695129395, + 2.3530721813440323, + 2.3930183947086334, + 2.365717649459839, + 2.357646808028221, + 2.3590978905558586, + 2.3872420713305473, + 2.3754425793886185, + 2.3687860667705536, + 2.340918242931366, + 2.3677115738391876, + 2.3112191557884216, + 2.392769291996956, + 2.4487893283367157, + 2.341904565691948, + 2.377177894115448, + 2.364643305540085, + 2.324292115867138, + 2.34882303327322, + 2.3602140098810196, + 2.3664713203907013, + 2.342870980501175, + 2.353952944278717, + 2.3571540862321854, + 2.371749296784401, + 2.325178414583206, + 2.3922941237688065, + 2.3534539341926575, + 2.3302333652973175, + 2.303064852952957, + 2.3614823520183563, + 2.314900189638138, + 2.3702807128429413, + 2.368003010749817, + 2.279435880482197, + 2.3157727420330048, + 2.306448608636856, + 2.318261682987213, + 2.308204770088196, + 2.313017800450325, + 2.309318721294403, + 2.3122695684432983, + 2.321071706712246, + 2.305533215403557, + 2.350373685359955, + 2.327108323574066, + 2.3174820095300674, + 2.342223972082138, + 2.35893951356411, + 2.2994923293590546, + 2.318520426750183, + 2.3407992720603943, + 2.364899516105652, + 2.340218812227249, + 2.3087599128484726, + 2.311676397919655, + 2.300605796277523, + 2.334869995713234, + 2.2723247408866882, + 2.295194625854492, + 2.3271916061639786, + 2.2567640393972397, + 2.323041468858719, + 2.279220014810562, + 2.3042054772377014, + 2.2557210624217987, + 2.350845068693161, + 2.2751225009560585, + 2.341839760541916, + 2.2605826407670975, + 2.284370720386505, + 2.3061442375183105, + 2.320046976208687, + 2.26070736348629, + 2.301961049437523, + 2.2589396983385086, + 2.2935024797916412, + 2.3084188401699066, + 2.368765637278557, + 2.3271292597055435, + 2.3010024279356003, + 2.341903477907181, + 2.2869296073913574, + 2.249100014567375, + 2.268570303916931, + 2.259503573179245, + 2.253677263855934, + 2.2985451221466064, + 2.3198659867048264, + 2.3033933490514755, + 2.351640924811363, + 2.290588155388832, + 2.210882283747196, + 2.2539452016353607, + 2.277690038084984, + 2.3271189630031586, + 2.275273323059082, + 2.2493939995765686, + 2.2840996235609055, + 2.282925307750702, + 2.28989577293396, + 2.3321627974510193, + 2.2662953287363052, + 2.247265003621578, + 2.2699142396450043, + 2.3287097811698914, + 2.250497028231621, + 2.2870392501354218, + 2.207040935754776, + 2.29455229640007, + 2.2927375584840775, + 2.251926928758621, + 2.276296019554138, + 2.2299301847815514, + 2.3035916686058044, + 2.294529139995575, + 2.295208767056465, + 2.2836184352636337, + 2.303111031651497, + 2.2596708834171295, + 2.1985993310809135, + 2.2470455169677734, + 2.2288205325603485, + 2.281368777155876, + 2.2728428840637207, + 2.252485290169716, + 2.289734721183777, + 2.2467056810855865, + 2.2696851938962936, + 2.2677999287843704, + 2.2238652259111404, + 2.2699518278241158, + 2.270063728094101, + 2.2998905926942825, + 2.2824141532182693, + 2.226297378540039, + 2.263387307524681, + 2.2282147258520126, + 2.259652242064476, + 2.284238964319229, + 2.256121091544628, + 2.2054974138736725, + 2.20460157841444, + 2.2596182227134705, + 2.2653257995843887, + 2.2642403542995453, + 2.256158769130707, + 2.295551970601082, + 2.292941391468048, + 2.2479084134101868, + 2.21825410425663, + 2.3008078187704086, + 2.287905529141426, + 2.2587230801582336, + 2.2386555522680283, + 2.265548050403595, + 2.1967722699046135, + 2.2117597311735153, + 2.271370865404606, + 2.24901907145977, + 2.2515418976545334, + 2.201303392648697, + 2.2282256484031677, + 2.242774710059166, + 2.30823478102684, + 2.293502375483513, + 2.2595291137695312, + 2.263742983341217, + 2.1737616285681725, + 2.2490208745002747, + 2.27687269449234, + 2.2600390166044235, + 2.2699785381555557, + 2.207936570048332, + 2.2012234926223755, + 2.284150570631027, + 2.2948222756385803, + 2.167818106710911, + 2.2974536269903183, + 2.219902791082859, + 2.135189935564995, + 2.2525680363178253, + 2.206582695245743, + 2.170176714658737, + 2.2423014044761658, + 2.2563272267580032, + 2.207301639020443, + 2.2118865847587585, + 2.2285995855927467, + 2.249342381954193, + 2.2709080576896667, + 2.265319526195526, + 2.211042858660221, + 2.206086330115795, + 2.244234174489975, + 2.1882672160863876, + 2.2363083213567734, + 2.260191097855568, + 2.2293813079595566, + 2.2519228905439377, + 2.2516008019447327, + 2.206386335194111, + 2.210202306509018, + 2.1768613904714584, + 2.216942511498928, + 2.1906247064471245, + 2.240927629172802, + 2.27256079018116, + 2.204475484788418, + 2.1616837456822395, + 2.181714430451393, + 2.2729124277830124, + 2.210827261209488, + 2.20964103192091, + 2.243682563304901, + 2.263919934630394, + 2.1950963735580444, + 2.2109197676181793, + 2.231409475207329, + 2.1842193007469177, + 2.2035921290516853, + 2.2254247218370438, + 2.2613544017076492, + 2.222332686185837, + 2.2079794704914093, + 2.2451836317777634, + 2.1989665180444717, + 2.2130379900336266, + 2.2346679270267487, + 2.1604868322610855, + 2.1857634484767914, + 2.2459794878959656, + 2.247925914824009, + 2.196181535720825, + 2.2550955414772034, + 2.168205387890339, + 2.217329680919647, + 2.227510452270508, + 2.2143067568540573, + 2.239250496029854, + 2.2076960653066635, + 2.182800032198429, + 2.2376834005117416, + 2.221465900540352, + 2.2031675279140472, + 2.2275792732834816, + 2.2495544999837875, + 2.213026314973831, + 2.2207311540842056, + 2.2506443560123444, + 2.2041610032320023, + 2.2087410166859627, + 2.2228233963251114, + 2.2027531266212463, + 2.2385850101709366, + 2.2282775193452835, + 2.2182778492569923, + 2.1611262634396553, + 2.1531791612505913, + 2.213945060968399, + 2.2078852355480194, + 2.2280480712652206, + 2.25068162381649, + 2.2152038291096687, + 2.2025632858276367, + 2.2005130127072334, + 2.1742565482854843, + 2.1923732683062553, + 2.225251153111458, + 2.177602104842663, + 2.236308217048645, + 2.184247486293316, + 2.2467557042837143, + 2.184721067547798, + 2.170271746814251, + 2.2233809530735016, + 2.2393888980150223, + 2.210395433008671, + 2.2440711706876755, + 2.2016809806227684, + 2.1482985615730286, + 2.1959914416074753, + 2.1840501949191093, + 2.2017736732959747, + 2.196066752076149, + 2.1649032905697823, + 2.1769600957632065, + 2.1878055185079575, + 2.195002980530262, + 2.1905601620674133, + 2.197399288415909, + 2.182880572974682, + 2.215973734855652, + 2.1284114122390747, + 2.201974429190159, + 2.2036041766405106, + 2.1640325635671616, + 2.2137678638100624, + 2.1911219507455826, + 2.169425308704376, + 2.128088541328907, + 2.1930107325315475, + 2.1662247627973557, + 2.1626308858394623, + 2.1623848602175713, + 2.2067711502313614, + 2.135978013277054, + 2.202214650809765, + 2.1773889288306236, + 2.2574620842933655, + 2.206412613391876, + 2.229229211807251, + 2.197563126683235, + 2.2229909896850586, + 2.189756840467453, + 2.1775417923927307, + 2.2005001604557037, + 2.224857434630394, + 2.1830659732222557, + 2.1956187933683395, + 2.1639575958251953, + 2.1850698739290237, + 2.1286690160632133, + 2.2109203785657883, + 2.140064626932144, + 2.229867711663246, + 2.209199719130993, + 2.16721860319376, + 2.189882829785347, + 2.1922502592206, + 2.185493089258671, + 2.204291671514511, + 2.189380683004856, + 2.2031512707471848, + 2.182609386742115, + 2.137457199394703, + 2.1563723981380463, + 2.226002424955368, + 2.1575972363352776, + 2.1938856691122055, + 2.2262879610061646, + 2.2217346280813217, + 2.155636318027973, + 2.2105260714888573, + 2.1651603877544403, + 2.2104636281728745, + 2.2164841145277023, + 2.1960049718618393, + 2.1613756269216537, + 2.1695037484169006, + 2.2002034932374954, + 2.1892430782318115, + 2.186606243252754, + 2.1835690438747406, + 2.159831367433071, + 2.153607152402401, + 2.1828832253813744, + 2.1763694286346436, + 2.2097062319517136, + 2.1983217149972916, + 2.1523167118430138, + 2.154193863272667, + 2.1623125076293945, + 2.178277589380741, + 2.168984778225422, + 2.1660810112953186, + 2.2049766406416893, + 2.1611586213111877, + 2.179366186261177, + 2.178453914821148, + 2.151294581592083, + 2.160119116306305, + 2.1711050271987915, + 2.1344854459166527, + 2.158126026391983, + 2.192331448197365, + 2.174198567867279, + 2.1409432142972946, + 2.1883884221315384, + 2.109431177377701, + 2.1541576758027077, + 2.159846216440201, + 2.1815974563360214, + 2.1801084727048874, + 2.197366639971733, + 2.125990107655525, + 2.1545095071196556, + 2.179230570793152, + 2.158467583358288, + 2.1559728384017944, + 2.179513320326805, + 2.1368451565504074, + 2.143274113535881, + 2.11408444494009, + 2.158066228032112, + 2.1454627960920334, + 2.1879200786352158, + 2.1299793422222137, + 2.1318638622760773, + 2.165858618915081, + 2.240453213453293, + 2.1893874406814575, + 2.2083776742219925, + 2.1794038116931915, + 2.1590076982975006, + 2.2216343730688095, + 2.131186217069626, + 2.1580155938863754, + 2.140191912651062, + 2.1457488611340523, + 2.084900349378586, + 2.166950762271881, + 2.184413529932499, + 2.1685364693403244, + 2.1795071959495544, + 2.1575323566794395, + 2.1964693516492844, + 2.1705307364463806, + 2.1872518435120583, + 2.182060644030571, + 2.1625112369656563, + 2.1564896628260612, + 2.1686747670173645, + 2.1683566197752953, + 2.155280388891697, + 2.141076050698757, + 2.164136618375778, + 2.1984047293663025, + 2.187964677810669, + 2.1556742787361145, + 2.1694857478141785, + 2.26653990149498, + 2.1432311087846756, + 2.087953671813011, + 2.1400472223758698, + 2.1649717688560486, + 2.145833730697632, + 2.1545548364520073, + 2.1615284085273743, + 2.1614892780780792, + 2.182577818632126, + 2.1937080696225166, + 2.137799620628357, + 2.217146120965481, + 2.1441346406936646, + 2.184183046221733, + 2.158258728682995, + 2.1548004895448685, + 2.164446122944355, + 2.1481718197464943, + 2.193043038249016, + 2.145757459104061, + 2.1108683943748474, + 2.120889440178871, + 2.1097699627280235, + 2.1610622107982635, + 2.140034057199955, + 2.1424268037080765, + 2.1242280676960945, + 2.141175463795662, + 2.1562083810567856, + 2.144730530679226, + 2.2052100151777267, + 2.1005802750587463, + 2.0938069969415665, + 2.129090651869774, + 2.1451883763074875, + 2.164913885295391, + 2.115574859082699, + 2.1369435787200928, + 2.1491263285279274, + 2.184069201350212, + 2.1896947845816612, + 2.1606741547584534, + 2.194422595202923, + 2.0991248190402985, + 2.1676140129566193, + 2.1359561756253242, + 2.1327217146754265, + 2.1719793304800987, + 2.1031930074095726, + 2.135761983692646, + 2.1668590158224106, + 2.196691043674946, + 2.155332714319229, + 2.0971647426486015, + 2.146366447210312, + 2.1506646275520325, + 2.167107507586479, + 2.0980505123734474, + 2.1557755172252655, + 2.1772454380989075, + 2.092924825847149, + 2.0840039774775505, + 2.228355184197426, + 2.132131151854992, + 2.12177824229002, + 2.154575802385807, + 2.163883849978447, + 2.2237676084041595, + 2.0546372681856155, + 2.133611559867859, + 2.1508577167987823, + 2.101455606520176, + 2.1508906930685043, + 2.12295500934124, + 2.143251411616802, + 2.142909198999405, + 2.116820439696312, + 2.0797159522771835, + 2.0866216272115707, + 2.2015913128852844, + 2.17224258184433, + 2.142866812646389, + 2.130419209599495, + 2.063929110765457, + 2.174641862511635, + 2.1687135100364685, + 2.1647323966026306, + 2.1546528935432434, + 2.1483991518616676, + 2.1301318258047104, + 2.173434942960739, + 2.160346806049347, + 2.1651860028505325, + 2.1453880816698074, + 2.121686279773712, + 2.1837164014577866, + 2.1296181082725525, + 2.1492287814617157, + 2.2091969177126884, + 2.14198200404644, + 2.1838369369506836, + 2.190246857702732, + 2.1204187497496605, + 2.1194376796483994, + 2.174221560359001, + 2.1692560389637947, + 2.116904802620411, + 2.103189505636692, + 2.0723745301365852, + 2.0763985738158226, + 2.118461139500141, + 2.150397129356861, + 2.150474689900875, + 2.1607814580202103, + 2.155224196612835, + 2.1624474823474884, + 2.0811927393078804, + 2.1150838658213615, + 2.0761620849370956, + 2.1776944026350975, + 2.12296012789011, + 2.1304186433553696, + 2.124818228185177, + 2.1265952214598656, + 2.1915649473667145, + 2.0908635407686234, + 2.1069038808345795, + 2.1365443617105484, + 2.1293310299515724, + 2.126703418791294, + 2.1403782814741135, + 2.116341046988964, + 2.1433170586824417, + 2.182603232562542, + 2.145109184086323, + 2.1478084474802017, + 2.092076987028122, + 2.1033250093460083, + 2.140270657837391, + 2.130894757807255, + 2.1388362869620323, + 2.0747509449720383, + 2.1188090294599533, + 2.2317993193864822, + 2.133225306868553, + 2.0391025096178055, + 2.1368575766682625, + 2.129554331302643, + 2.2146957963705063, + 2.0820256024599075, + 2.1196242049336433, + 2.1157122626900673, + 2.1357452049851418, + 2.121550850570202, + 2.1386459693312645, + 2.1702114641666412, + 2.107653461396694, + 2.0805357918143272, + 2.1311780139803886, + 2.085243947803974, + 2.0745886638760567, + 2.166841544210911, + 2.0822411626577377, + 2.139682576060295, + 2.085096597671509, + 2.16248120367527, + 2.1336643397808075, + 2.0613886415958405, + 2.1879687756299973, + 2.1150797829031944, + 2.119518034160137, + 2.1282371431589127, + 2.1682773381471634, + 2.101556196808815, + 2.147344261407852, + 2.1192181929945946, + 2.1449124217033386, + 2.1146250516176224, + 2.100040629506111, + 2.0773393511772156, + 2.102326810359955, + 2.0561509802937508, + 2.104710564017296, + 2.1211454272270203, + 2.127342715859413, + 2.1339519321918488, + 2.1547621488571167, + 2.134501576423645, + 2.1328621804714203, + 2.184201031923294, + 2.0860368609428406, + 2.120128743350506, + 2.1744554862380028, + 2.0713039115071297, + 2.0943262577056885, + 2.098053142428398, + 2.134252190589905, + 2.1250083968043327, + 2.084468886256218, + 2.17497281730175, + 2.1288093104958534, + 2.1395000517368317, + 2.1124981865286827, + 2.153906151652336, + 2.066654346883297, + 2.1544894129037857, + 2.127647891640663, + 2.1177515387535095, + 2.07787524163723, + 2.1294006928801537, + 2.128833532333374, + 2.1331701204180717, + 2.049657814204693, + 2.0967564806342125, + 2.0897674784064293, + 2.1401848569512367, + 2.0930611044168472, + 2.113971270620823, + 2.110783316195011, + 2.1361953243613243, + 2.090254418551922, + 2.1032763570547104, + 2.158109173178673, + 2.1163772642612457, + 2.1360705345869064, + 2.1255224719643593, + 2.1434421241283417, + 2.1142918318510056, + 2.0853144749999046, + 2.1344446018338203, + 2.096809007227421, + 2.1376030817627907, + 2.1024411618709564, + 2.101695716381073, + 2.121945798397064, + 2.1086570993065834, + 2.054915964603424, + 2.1117512583732605, + 2.126697763800621, + 2.123760759830475, + 2.1378553435206413, + 2.141614779829979, + 2.1552577391266823, + 2.1266572922468185, + 2.107217915356159, + 2.089830093085766, + 2.1158564761281013, + 2.128531150519848, + 2.0791483595967293, + 2.047775946557522, + 2.169277884066105, + 2.1232942566275597, + 2.1230533346533775, + 2.076633833348751, + 2.1073708906769753, + 2.1052179113030434, + 2.0849156752228737, + 2.0707033053040504, + 2.0594907701015472, + 2.1482200026512146, + 2.1294965744018555, + 2.1285038739442825, + 2.1981706395745277, + 2.0912225991487503, + 2.0867964923381805, + 2.127059184014797, + 2.107911095023155, + 2.1190354079008102, + 2.095177225768566, + 2.1163056045770645, + 2.1124512180685997, + 2.085825636982918, + 2.1925466656684875, + 2.101822964847088, + 2.174862340092659, + 2.1196212247014046, + 2.150236427783966, + 2.091476246714592, + 2.043076530098915, + 2.11587568372488, + 2.0896103754639626, + 2.148995488882065, + 2.0899141505360603, + 2.0565314516425133, + 2.1163345277309418, + 2.071822188794613, + 2.0606581792235374, + 2.143792577087879, + 2.1202134266495705, + 2.1338164135813713, + 2.1171406507492065, + 2.0856682658195496, + 2.045985236763954, + 2.1203959360718727, + 2.06377811729908, + 2.086655780673027, + 2.1371421813964844, + 2.1074946969747543, + 2.04456052929163, + 2.087636321783066, + 2.046764336526394, + 2.1393957138061523, + 2.060795083642006, + 2.0664133951067924, + 2.1127325370907784, + 2.088113896548748, + 2.0516619458794594, + 2.067213572561741, + 2.0947535187005997, + 2.067729912698269, + 2.1177423894405365, + 2.1033549159765244, + 2.064347706735134, + 2.0886844992637634, + 2.0968950614333153, + 2.1136304810643196, + 2.1175004318356514, + 2.129902921617031, + 2.124113418161869, + 2.07846812158823, + 2.1072779670357704, + 2.118053786456585, + 2.1073956564068794, + 2.1178341060876846, + 2.0376839712262154, + 2.1195977106690407, + 2.0801999494433403, + 2.086342476308346, + 2.1069431006908417, + 2.131090834736824, + 2.003604017198086, + 2.078594982624054, + 2.064763732254505, + 2.056250810623169, + 2.0798662677407265, + 2.1190195605158806, + 2.0970626175403595, + 2.0835975483059883, + 2.0705263391137123, + 2.092038720846176, + 2.113932952284813, + 2.1045149862766266, + 2.0855739638209343, + 2.109140656888485, + 2.1330899819731712, + 2.0982739105820656, + 2.0556586533784866, + 2.1456620916724205, + 2.1010462939739227, + 2.066707618534565, + 2.1027609407901764, + 2.1376034021377563, + 2.15145543217659, + 2.076833575963974, + 2.133496932685375, + 2.08060759305954, + 2.064087614417076, + 2.1057182028889656, + 2.0989591851830482, + 2.0523927584290504, + 2.052446350455284, + 2.061856970191002, + 2.1001479774713516, + 2.0673284605145454, + 2.0771031752228737, + 2.1331471353769302, + 2.059754855930805, + 2.0580356493592262, + 2.077908307313919, + 2.1502393782138824, + 2.122028611600399, + 2.096871070563793, + 2.0514773949980736, + 2.0312906056642532, + 2.053397722542286, + 2.0589247420430183, + 2.103885419666767, + 2.0766085237264633, + 2.061582997441292, + 2.04033350199461, + 2.0817313492298126, + 2.0717427879571915, + 2.0676568523049355, + 2.1026962026953697, + 2.075967289507389, + 2.118939459323883, + 2.098557449877262, + 2.080316014587879, + 2.1389375254511833, + 2.1193755120038986, + 2.0674487948417664, + 2.073130264878273, + 2.121606722474098, + 2.0773168429732323, + 2.048414282500744, + 2.088956482708454, + 2.0548322200775146, + 2.0983495712280273, + 2.1067300215363503, + 2.0788237676024437, + 2.1082055270671844, + 2.1532573252916336, + 2.1114791855216026, + 2.097907207906246, + 2.1139452159404755, + 2.0704490691423416, + 2.1057537719607353, + 2.1028573140501976, + 2.059957802295685, + 2.114586465060711, + 2.04319416731596, + 2.104722738265991, + 2.0735750421881676, + 2.090588852763176, + 2.1125890910625458, + 2.0925528705120087, + 2.1331923827528954, + 2.066437415778637, + 2.076450876891613, + 2.1041653528809547, + 2.1197346448898315, + 2.0614292472600937, + 2.1566169559955597, + 2.082108221948147, + 2.076836995780468, + 2.1157222986221313, + 2.058776803314686, + 2.1156536042690277, + 2.078630216419697, + 2.0842374488711357, + 2.0728675350546837, + 2.128107413649559, + 2.0521582886576653, + 2.0710431784391403, + 2.075315900146961, + 2.068170167505741, + 2.08226827532053, + 2.073641888797283, + 2.0832217186689377, + 2.068214178085327, + 2.109774425625801, + 2.045437276363373, + 2.0598939955234528, + 2.077075056731701, + 2.056674510240555, + 2.07688170671463, + 2.101207673549652, + 2.1084860488772392, + 2.1308348178863525, + 2.0629074573516846, + 2.0740512385964394, + 2.0292365923523903, + 2.08596920222044, + 2.09300035238266, + 2.1157858967781067, + 2.080493703484535, + 2.0922201052308083, + 2.037645496428013, + 2.0744452700018883, + 2.1128039360046387, + 2.1196825727820396, + 2.0467619150877, + 2.10732788592577, + 2.057101532816887, + 2.124672159552574, + 2.0332978814840317, + 2.0976179614663124, + 2.068111687898636, + 2.076121963560581, + 2.0682152211666107, + 2.0596002712845802, + 2.0821236297488213, + 2.088251866400242, + 2.1112310141324997, + 2.033425912261009, + 2.085014633834362, + 2.0549292489886284, + 2.0522144734859467, + 2.099467448890209, + 2.0510906353592873, + 2.057126462459564, + 2.1039365977048874, + 2.064421132206917, + 2.0823196545243263, + 2.0316421017050743, + 2.0718842521309853, + 2.0907010659575462, + 2.0398018807172775, + 2.011564679443836, + 2.0866399481892586, + 2.1224156990647316, + 2.080210879445076, + 2.032454065978527, + 2.078737385571003, + 2.075644128024578, + 2.0446126833558083, + 2.1186984851956367, + 2.0806473940610886, + 2.0866662710905075, + 2.104400284588337, + 2.0846058130264282, + 2.0354090854525566, + 2.0357372388243675, + 2.079992964863777, + 2.0707548335194588, + 2.0807266384363174, + 2.0822779461741447, + 2.117115095257759, + 2.055205076932907, + 2.104802504181862, + 2.0238883644342422, + 2.116937406361103, + 2.0601454079151154, + 2.070825830101967, + 2.0914126560091972, + 2.0838413760066032, + 2.111621953547001, + 2.0483533293008804, + 2.067333698272705, + 2.0818786323070526, + 2.083373084664345, + 2.126552164554596, + 2.045739971101284, + 2.0384513586759567, + 2.090736001729965, + 2.054653577506542, + 2.084614537656307, + 2.1163672357797623, + 2.0287013053894043, + 2.076931782066822, + 2.09414578974247, + 2.0601787492632866, + 2.0259914472699165, + 2.064942367374897, + 2.088176131248474, + 2.057134859263897, + 2.0494123324751854, + 2.075972341001034, + 2.058980368077755, + 2.080832064151764, + 2.126746214926243, + 2.0967922061681747, + 2.067984528839588, + 2.078324928879738, + 2.083553299307823, + 2.0578985065221786, + 2.072483539581299, + 2.044805496931076, + 2.0852983742952347, + 2.043850503861904, + 2.0372587963938713, + 2.0842889696359634, + 1.9920414984226227, + 2.075049340724945, + 2.055201105773449, + 2.018791325390339, + 2.05525042116642, + 2.0577879771590233, + 2.0639662966132164, + 2.080051749944687, + 2.1018550246953964, + 2.069247253239155, + 2.0822381153702736, + 2.066514641046524, + 2.0661729127168655, + 2.02827188372612, + 2.0263261795043945, + 2.102808766067028, + 2.06929961591959, + 2.0931807085871696, + 2.083178848028183, + 2.03596992790699, + 2.083343245089054, + 2.0619414150714874, + 2.0622674003243446, + 2.08769328892231, + 2.051968030631542, + 2.0675388276576996, + 2.0659406930208206, + 2.047129712998867, + 2.0285030230879784, + 2.0553610995411873, + 2.0101452097296715, + 2.0670553520321846, + 2.0554268285632133, + 2.0480434373021126, + 2.075541414320469, + 2.087974674999714, + 2.0689631178975105, + 2.0987142100930214, + 2.0369644314050674, + 2.0185140818357468, + 1.9826836660504341, + 2.0912550911307335, + 2.118071086704731, + 2.0410716012120247, + 2.0337745398283005, + 2.083700567483902, + 2.03350630402565, + 2.07258553057909, + 2.0181403160095215, + 2.030014507472515, + 2.0185409113764763, + 2.0494685396552086, + 2.093616083264351, + 2.0718869417905807, + 2.069687105715275, + 2.0288492292165756, + 2.098213531076908, + 2.0168078616261482, + 2.017090529203415, + 2.0453367605805397, + 2.097784049808979, + 2.0060554519295692, + 2.0848663598299026, + 2.0542651265859604, + 2.1182328313589096, + 2.029650829732418, + 2.029473416507244, + 2.0524057745933533, + 2.066158801317215, + 2.056597039103508, + 2.087187558412552, + 2.04720252007246, + 2.063532665371895, + 2.0641530379652977, + 2.073301278054714, + 2.0974854975938797, + 2.0092081651091576, + 2.023013412952423, + 2.0640034452080727, + 2.0544734746217728, + 2.036905586719513, + 2.068603925406933, + 2.144832856953144, + 2.0576113909482956, + 2.077566012740135, + 1.9822317063808441, + 2.0526579469442368, + 2.1084584072232246, + 2.0632589980959892, + 2.021683782339096, + 2.0989444255828857, + 2.039842374622822, + 2.05978524684906, + 2.077068515121937, + 2.0002360567450523, + 2.0340574085712433, + 2.0386503487825394, + 2.074206866323948, + 2.063395470380783, + 2.047794759273529, + 2.0238468423485756, + 2.022355481982231, + 2.0868667662143707, + 2.026085302233696, + 2.0367030575871468, + 2.0589287728071213, + 2.0324468165636063, + 2.0922194495797157, + 2.121002785861492, + 2.006329670548439, + 2.0526232719421387, + 2.0887759774923325, + 2.029031626880169, + 2.041236825287342, + 2.1070928499102592, + 2.0375802740454674, + 2.0745116993784904, + 2.0264559015631676, + 2.093472383916378, + 2.110783375799656, + 2.0557553619146347, + 2.032823197543621, + 2.054245062172413, + 2.058667443692684, + 2.0230545848608017, + 2.085764594376087, + 2.046250246465206, + 2.1057860031723976, + 2.033843234181404, + 2.0536908581852913, + 2.0431628227233887, + 2.060614585876465, + 2.1106194481253624, + 2.121265508234501, + 2.029979668557644, + 2.081477001309395, + 2.0234320610761642, + 2.0588094666600227, + 2.011410892009735, + 2.0782287269830704, + 2.0901995822787285, + 2.0549020022153854, + 2.093630224466324, + 2.1010064482688904, + 2.0614402294158936, + 2.055327296257019, + 2.032518044114113, + 2.071324221789837, + 2.015162892639637, + 2.0478126406669617, + 2.0950972214341164, + 2.0453572049736977, + 2.0726634189486504, + 2.0363599359989166, + 2.0398057848215103, + 2.073083184659481, + 2.1169033274054527, + 2.0282982140779495, + 2.029928542673588, + 2.0086976662278175, + 2.070713274180889, + 2.0160971581935883, + 2.061804488301277, + 2.01869910210371, + 2.0549182072281837, + 2.0632846653461456, + 2.0085569471120834, + 2.0084597542881966, + 2.087781600654125, + 2.0453952476382256, + 2.009856164455414, + 2.0178299322724342, + 2.0514163970947266, + 2.0555821135640144, + 2.0508276745676994, + 2.039798878133297, + 2.104607068002224, + 2.0175361782312393, + 1.9955153688788414, + 2.0499988421797752, + 2.0227396935224533, + 2.0326089560985565, + 2.076279826462269, + 1.9718514308333397, + 2.058347500860691, + 2.031898632645607, + 2.051825501024723, + 2.0818720757961273, + 2.062887415289879, + 2.0417758747935295, + 2.053724706172943, + 1.999840460717678, + 2.0297943875193596, + 2.0607473105192184, + 1.9970363900065422, + 2.0438899621367455, + 1.9868263229727745, + 2.076843537390232, + 2.0807366892695427, + 2.0745331570506096, + 2.0920959264039993, + 1.9695633798837662, + 2.042664349079132, + 2.058349959552288, + 1.9999414086341858, + 2.0284797847270966, + 2.0602335184812546, + 2.0056714713573456, + 1.9921028167009354, + 2.1593693420290947, + 2.044172026216984, + 2.0021911412477493, + 2.0614453181624413, + 2.026482582092285, + 2.0023489370942116, + 2.066790319979191, + 2.057253137230873, + 1.9949829652905464, + 2.0671448707580566, + 2.0784787982702255, + 2.0610187724232674, + 1.9885017722845078, + 2.017619550228119, + 2.04525326192379, + 2.070211812853813, + 2.0225620344281197, + 2.051934577524662, + 2.0487052351236343, + 2.0572201311588287, + 2.0467306077480316, + 2.063438504934311, + 2.066483959555626, + 2.0624671280384064, + 1.994805671274662, + 1.958943746984005, + 2.0815182849764824, + 2.013330839574337, + 2.073805496096611, + 2.0056604370474815, + 2.104478642344475, + 2.0570818707346916, + 2.0079680383205414, + 2.0193314403295517, + 2.084199033677578, + 2.0312812998890877, + 2.044229172170162, + 1.975568749010563, + 2.0826079100370407, + 2.067619614303112, + 2.039142280817032, + 2.0739936754107475, + 2.0214963555336, + 2.055845022201538, + 2.022305518388748, + 2.0638862773776054, + 2.0678136870265007, + 1.9931604787707329, + 2.0481941625475883, + 2.025836259126663, + 2.012361690402031, + 2.0683428794145584, + 2.027590349316597, + 2.0216799080371857, + 1.9618375301361084, + 2.045212544500828, + 2.050030365586281, + 1.960022822022438, + 2.026993528008461, + 2.0613368451595306, + 2.0154469683766365, + 2.0330786928534508, + 1.989981859922409, + 2.0983831733465195, + 2.015031598508358, + 1.9950316697359085, + 2.0651411041617393, + 2.02780432254076, + 2.0577921867370605, + 2.0536393523216248, + 2.0224162489175797, + 1.9919483065605164, + 2.0348687395453453, + 2.0335993096232414, + 2.0406234487891197, + 2.0218030214309692, + 2.040086306631565, + 2.0524434745311737, + 2.0658682882785797, + 2.080804154276848, + 2.0457578003406525, + 2.0782298520207405, + 1.9904474392533302, + 1.9774739295244217, + 2.0017569810152054, + 2.036637954413891, + 2.0258431658148766, + 2.086254708468914, + 1.9715771898627281, + 2.042479805648327, + 2.0448756515979767, + 2.053536757826805, + 2.0711889639496803, + 2.0541395768523216, + 1.9685304909944534, + 2.0069988891482353, + 2.0558991208672523, + 2.038026876747608, + 2.0435478389263153, + 2.0009077936410904, + 1.9866508394479752, + 2.0461611449718475, + 2.063485972583294, + 2.0282352194190025, + 2.0273979380726814, + 2.004512242972851, + 2.087530590593815, + 2.026432752609253, + 2.0064602419734, + 2.0508701726794243, + 2.044722869992256, + 2.0563568472862244, + 2.0036481767892838, + 2.024157330393791, + 1.9885678738355637, + 2.056172326207161, + 2.0569653138518333, + 2.040420688688755, + 2.023186393082142, + 1.9928800240159035, + 2.0476977974176407, + 2.0309769436717033, + 2.0606814697384834, + 2.0714243575930595, + 2.042622745037079, + 2.022701919078827, + 2.017118066549301, + 2.03256393969059, + 2.0463165417313576, + 1.9934095069766045, + 2.0738991647958755, + 1.994753658771515, + 1.991907574236393, + 1.9940093085169792, + 1.9978253096342087, + 2.015462137758732, + 2.007128916680813, + 2.0346174240112305, + 2.014202758669853, + 2.0050712153315544, + 2.0700870975852013, + 2.009524144232273, + 2.047173462808132, + 2.0346701741218567, + 2.0449811443686485, + 2.0229199901223183, + 2.0363209173083305, + 2.038638561964035, + 1.9816213846206665, + 2.0461468398571014, + 2.0289588049054146, + 2.0237071961164474, + 2.032471738755703, + 2.106120251119137, + 2.003002554178238, + 1.972887560725212, + 2.0429940223693848, + 2.0269825309515, + 2.046481505036354, + 2.0203612744808197, + 1.9956261813640594, + 2.0257449001073837, + 2.0783135816454887, + 2.0632865875959396, + 1.9859484359622002, + 2.0526953861117363, + 1.9820646718144417, + 2.0333433523774147, + 2.004299335181713, + 2.0479058995842934, + 2.075914040207863, + 2.000350259244442, + 2.0143738240003586, + 2.0270480439066887, + 2.0027455016970634, + 2.007543809711933, + 2.0101728588342667, + 1.966880775988102, + 2.035865865647793, + 2.017409086227417, + 2.0276251807808876, + 2.015930064022541, + 2.020614892244339, + 2.0093763023614883, + 2.027725301682949, + 2.0325542762875557, + 2.003512516617775, + 2.071848914027214, + 2.0136645436286926, + 2.0264983102679253, + 2.060220345854759, + 2.0099631547927856, + 2.0311823338270187, + 1.9808486551046371, + 2.0413676500320435, + 1.9700573161244392, + 2.0348811000585556, + 2.0381607115268707, + 2.012981101870537, + 2.0416990518569946, + 1.9974445775151253, + 1.9945291057229042, + 2.0560403019189835, + 2.036359779536724, + 2.0528222173452377, + 2.0124075785279274, + 1.9894162490963936, + 1.9630667641758919, + 1.9987764582037926, + 1.9410181045532227, + 2.031112313270569, + 2.0471931248903275, + 1.9797803312540054, + 2.0029540434479713, + 2.0433869883418083, + 2.0410597026348114, + 2.065158851444721, + 2.0206254199147224, + 2.0114581659436226, + 2.0456648617982864, + 1.9799518957734108, + 2.017271488904953, + 2.0023151710629463, + 2.0668729320168495, + 1.9886837303638458, + 2.057582698762417, + 2.0003577917814255, + 1.992318071424961, + 2.031634010374546, + 2.0148660242557526, + 2.0072066858410835, + 2.0167081356048584, + 2.013246349990368, + 2.0172711610794067, + 1.990559384226799, + 2.0047906190156937, + 2.030572734773159, + 1.9941681697964668, + 2.0326825603842735, + 2.043383911252022, + 1.9990619719028473, + 2.010693833231926, + 2.0308566093444824, + 2.034300744533539, + 1.9911048114299774, + 2.022041730582714, + 2.02203556150198, + 1.9916928187012672, + 1.990910790860653, + 2.0777962878346443, + 1.9750299900770187, + 2.0433941707015038, + 2.059152752161026, + 2.0610751137137413, + 2.0589925050735474, + 1.9296481311321259, + 2.057456724345684, + 2.0292029678821564, + 2.0544185116887093, + 2.040156900882721, + 2.0448417738080025, + 1.9992524981498718, + 2.029926173388958, + 2.1046773940324783, + 2.0263595655560493, + 1.9716458916664124, + 2.041224256157875, + 2.0075801983475685, + 2.0205018669366837, + 1.9596107751131058, + 2.0846390649676323, + 2.0033578351140022, + 2.0168448612093925, + 2.066888853907585, + 2.0142345651984215, + 1.9947561249136925, + 2.0512561425566673, + 1.9953018501400948, + 2.005210943520069, + 2.031134970486164, + 1.99045130610466, + 2.072590984404087, + 2.0758411064743996, + 1.9937042891979218, + 2.009852260351181, + 2.0004496201872826, + 2.0051789060235023, + 2.0322783067822456, + 1.9972547069191933, + 2.026237800717354, + 1.9712481200695038, + 1.9821988865733147, + 1.9888686761260033, + 2.0764504820108414, + 2.0488183200359344, + 2.0689162835478783, + 1.9954717680811882, + 2.0361037254333496, + 2.0621556118130684, + 2.0238041430711746, + 1.9911487475037575, + 2.01306165009737, + 1.9846514463424683, + 2.015174485743046, + 1.9894301891326904, + 2.053854778409004, + 2.0386984944343567, + 1.979624718427658, + 1.9551764577627182, + 2.0090260952711105, + 2.0078627839684486, + 2.013547867536545, + 2.0141994282603264, + 2.047642044723034, + 1.9671053066849709, + 1.9769804403185844, + 1.9792799577116966, + 1.9991566613316536, + 2.051173686981201, + 2.01029534637928, + 2.0041516050696373, + 2.031491734087467, + 2.03504778444767, + 1.927995465695858, + 2.029096268117428, + 2.0124726444482803, + 2.03600662201643, + 2.037153534591198, + 2.014124795794487, + 1.99077308177948, + 1.983078919351101, + 2.0575252324342728, + 1.9865586459636688, + 1.9982961267232895, + 2.031304709613323, + 2.019311673939228, + 1.9892891570925713, + 2.0130565837025642, + 1.97503362596035, + 1.9897939711809158, + 1.9290409311652184, + 1.9744504615664482, + 2.0139475911855698, + 2.026855945587158, + 1.985922746360302, + 1.9905302450060844, + 1.9880499690771103, + 2.0436705872416496, + 2.0377974808216095, + 2.054058387875557, + 1.9796514809131622, + 1.9854420349001884, + 2.017201267182827, + 1.9645578563213348, + 2.043427936732769, + 2.061902053654194, + 2.028269685804844, + 2.0513908937573433, + 1.9847638756036758, + 1.9675751999020576, + 2.0590119883418083, + 1.9933500587940216, + 2.0278498828411102, + 2.056687369942665, + 1.9991180673241615, + 2.016737014055252, + 2.0064490735530853, + 1.9934021309018135, + 1.9742969200015068, + 2.036618374288082, + 1.9888476952910423, + 1.9983019679784775, + 2.0347175672650337, + 2.0472301468253136, + 2.006411924958229, + 2.007497139275074, + 1.993650995194912, + 2.0415332466363907, + 2.01625943928957, + 2.0142936259508133, + 1.9919017627835274, + 2.033961273729801, + 2.041066437959671, + 1.9774931967258453, + 1.9719897657632828, + 2.0475282222032547, + 2.0262749418616295, + 2.0471938252449036, + 2.0023239329457283, + 2.008810192346573, + 1.995789960026741, + 2.035289518535137, + 1.9795952886343002, + 1.9619763791561127, + 2.0455771014094353, + 2.023497946560383, + 2.0199597254395485, + 2.018619030714035, + 1.9838962256908417, + 1.9986294284462929, + 2.0189720019698143, + 1.9658852592110634, + 2.0057236179709435, + 1.9838148579001427, + 2.016438476741314, + 2.047451540827751, + 2.0054284185171127, + 1.9911517649888992, + 2.0006784051656723, + 2.002092733979225, + 2.013062469661236, + 1.9906757846474648, + 1.9903575405478477, + 1.9856652095913887, + 2.072898641228676, + 2.008611097931862, + 2.0325816944241524, + 1.9989771395921707, + 1.983209677040577, + 2.0169480443000793, + 2.0373481065034866, + 1.9937742799520493, + 2.0305481255054474, + 2.022393062710762, + 1.9923921823501587, + 2.0438044667243958, + 1.9870337471365929, + 1.9978091642260551, + 2.0043741688132286, + 2.005836457014084, + 2.024130068719387, + 2.034245029091835, + 2.0160465836524963, + 2.030436173081398, + 2.0236209332942963, + 2.039498656988144, + 2.0302647203207016, + 1.988513968884945, + 1.9951966777443886, + 2.0522387102246284, + 1.9879936948418617, + 2.0086864605545998, + 1.9969478845596313, + 2.031145930290222, + 2.0352999716997147, + 2.0665007531642914, + 1.976129725575447, + 1.9708246141672134, + 2.0373434275388718, + 2.0277241840958595, + 2.0635389909148216, + 1.956991508603096, + 1.9855423644185066, + 2.0393706262111664, + 1.9730364382266998, + 2.029839888215065, + 2.071782238781452, + 1.9815437495708466, + 2.0081322342157364, + 2.0220490247011185, + 1.9511517509818077, + 1.9960035011172295, + 2.019466631114483, + 2.036836802959442, + 2.0275320634245872, + 2.0239167287945747, + 2.0640180334448814, + 2.0219478756189346, + 2.0388383120298386, + 1.9863403663039207, + 2.014947973191738, + 2.029127225279808, + 1.9865209087729454, + 2.0248272120952606, + 1.9699107557535172, + 2.0222839936614037, + 1.9993940517306328, + 2.006730727851391, + 1.985050119459629, + 1.9684884622693062, + 2.0682719349861145, + 1.994780145585537, + 2.0208920538425446, + 1.9858718141913414, + 1.964306689798832, + 2.002820134162903, + 2.0008208975195885, + 2.0100094750523567, + 1.994920901954174, + 2.034286566078663, + 1.9859086573123932, + 2.017009325325489, + 2.050813816487789, + 2.035660296678543, + 2.00095147639513, + 2.072943150997162, + 1.935829021036625, + 2.0453990548849106, + 1.9569045454263687, + 1.9811904057860374, + 1.9589426517486572, + 2.0479939952492714, + 1.982429914176464, + 2.0081235840916634, + 2.0321198105812073, + 2.0204659178853035, + 1.9720355495810509, + 1.972931057214737, + 2.0252153500914574, + 1.992736041545868, + 2.0073248967528343, + 1.9887183606624603, + 2.051975265145302, + 2.053317241370678, + 1.9906044825911522, + 1.9840009063482285, + 2.0747819542884827, + 2.03570593893528, + 2.0051850974559784, + 2.046360269188881, + 1.9546746090054512, + 2.004560723900795, + 2.0061267241835594, + 2.0078068301081657, + 2.047250621020794, + 2.036908909678459, + 1.8985976055264473, + 2.0168354213237762, + 1.9784022122621536, + 2.0081952959299088, + 2.0015434697270393, + 2.0263729989528656, + 2.0032437667250633, + 1.9936833381652832, + 1.9926146194338799, + 2.0650923028588295, + 1.993078850209713, + 2.0102521404623985, + 2.039547525346279, + 2.0172265991568565, + 2.0336327850818634, + 2.0429108887910843, + 1.994009368121624, + 2.0042456462979317, + 2.0011273846030235, + 2.0087851211428642, + 2.0348391234874725, + 2.0001969635486603, + 2.023073971271515, + 2.048934228718281, + 1.996514469385147, + 2.0594981610774994, + 1.9761760383844376, + 2.0081776082515717, + 2.0475327894091606, + 1.987195909023285, + 2.062518909573555, + 2.0151293501257896, + 1.9887761622667313, + 2.0190157145261765, + 1.9758436679840088, + 2.0503030344843864, + 2.013340823352337, + 2.0525011494755745, + 1.9824495613574982, + 2.0522356629371643, + 2.0132360458374023, + 2.0294456258416176, + 2.059056870639324, + 2.034105896949768, + 1.9698210060596466, + 1.9919957295060158, + 2.028317727148533, + 2.039343371987343, + 1.9717760756611824, + 2.0363448783755302, + 1.9941799491643906, + 1.9802324548363686, + 1.992448277771473, + 2.072346158325672, + 2.0459095910191536, + 2.0401524379849434, + 2.0261582881212234, + 2.037331707775593, + 1.952724114060402, + 2.0327851697802544, + 2.011854760348797, + 2.0011244267225266, + 1.9710886776447296, + 2.0306184589862823, + 1.980744644999504, + 2.015463672578335, + 1.9463814124464989, + 1.9874341934919357, + 1.96126888692379, + 1.9869138672947884, + 2.0325425043702126, + 2.045391134917736, + 1.898050531744957, + 2.059508055448532, + 2.022269554436207, + 1.9935850873589516, + 1.9667118787765503, + 1.9506275206804276, + 2.0154733061790466, + 2.0027915611863136, + 1.9563188999891281, + 2.018349438905716, + 2.0011549666523933, + 2.011819139122963, + 2.0129420459270477, + 2.0394232347607613, + 2.0425062403082848, + 1.989676095545292, + 2.049164444208145, + 2.0087298154830933, + 2.002370424568653, + 2.0055968910455704, + 2.0109176859259605, + 1.9486911371350288, + 2.0022728964686394, + 2.10099396109581, + 2.0243823379278183, + 2.0096457079052925, + 2.024029403924942, + 1.9766075238585472, + 2.014873504638672, + 1.992447353899479, + 2.0179089158773422, + 2.00634478777647, + 2.0346834138035774, + 1.9679845198988914, + 1.99566088616848, + 2.0278744474053383, + 1.9934719130396843, + 2.002937264740467, + 2.0324525386095047, + 2.0112536177039146, + 2.0068087205290794, + 2.011663965880871, + 2.053352214396, + 2.0674925446510315, + 2.0385855361819267, + 1.9636224433779716, + 1.9800540208816528, + 2.0298780873417854, + 1.9783809781074524, + 2.009504809975624, + 2.050186537206173, + 2.004037484526634, + 1.9919017776846886, + 2.0202462896704674, + 2.013695649802685, + 2.0088053941726685, + 2.015388049185276, + 2.0048750936985016, + 1.984981656074524, + 1.9766478911042213, + 1.9656663984060287, + 1.954810120165348, + 2.1082138270139694, + 2.0361176505684853, + 2.0235401019454002, + 1.9376945793628693, + 1.98851428180933, + 1.9733050838112831, + 1.9832649305462837, + 2.041266515851021, + 2.089221216738224, + 2.0145920887589455, + 1.993557870388031, + 2.0535949617624283, + 1.9940952062606812, + 1.986963413655758, + 1.9858919829130173, + 2.0403983294963837, + 2.023009367287159, + 1.9929057210683823, + 2.059767633676529, + 1.9872056543827057, + 2.0207050666213036, + 1.946231447160244, + 1.9919718354940414, + 2.0148510932922363, + 1.9834365993738174, + 2.043611764907837, + 1.9916072711348534, + 2.0247369036078453, + 2.0751812532544136, + 2.0518119111657143, + 1.987902507185936, + 2.015834502875805, + 2.0299999564886093, + 2.0114872977137566, + 2.00143476575613, + 2.0098708793520927, + 2.041247747838497, + 1.9988224059343338, + 1.9506307914853096, + 2.019528903067112, + 1.9915261790156364, + 2.0000013038516045, + 2.0582819506525993, + 2.0237764418125153, + 2.033445492386818, + 1.991778589785099, + 1.916336566209793, + 2.024529218673706, + 1.9642040580511093, + 1.9697676077485085, + 2.0131810531020164, + 2.011799320578575, + 2.004597544670105, + 1.9951446428894997, + 2.0021052956581116, + 2.0166311115026474, + 1.9419043213129044, + 1.986458919942379, + 1.9948315098881721, + 1.9934724420309067, + 2.008989594876766, + 2.0529000014066696, + 2.0233055129647255, + 1.9887127876281738, + 2.0661687403917313, + 1.9629738479852676, + 1.9921198412775993, + 2.0163868814706802, + 2.045803777873516, + 1.9683387130498886, + 1.9731265380978584, + 2.04167091101408, + 2.027327224612236, + 1.968782089650631, + 2.0465513169765472, + 1.9786500334739685, + 1.9983206167817116, + 2.023652955889702, + 2.025234892964363, + 2.0554421842098236, + 2.022333540022373, + 1.916933685541153, + 2.030999667942524, + 2.0391074419021606, + 2.017983987927437, + 2.030944801867008, + 2.0134077668190002, + 1.9515063613653183, + 1.982171081006527, + 2.0237578451633453, + 1.987555243074894, + 2.019579105079174, + 2.005756102502346, + 1.9832525476813316, + 2.0005519539117813, + 2.004529044032097, + 1.9886600822210312, + 1.9639106914401054, + 2.0045959874987602, + 2.015386678278446, + 2.0129428282380104, + 1.9857064187526703, + 1.987038254737854, + 1.9994425848126411, + 2.006143756210804, + 2.041514925658703, + 2.018173709511757, + 2.0382527112960815, + 2.013184629380703, + 1.9237553104758263, + 2.0075599253177643, + 1.9910276010632515, + 1.9707064256072044, + 2.0266252756118774, + 2.032939836382866, + 1.9928220435976982, + 2.014900177717209, + 2.0156758427619934, + 1.990025445818901, + 2.038992464542389, + 2.0452344715595245, + 1.994781956076622, + 2.007961943745613, + 2.029971182346344, + 2.0122359320521355, + 1.9665468409657478, + 2.0119063332676888, + 1.9913077056407928, + 1.9978028759360313, + 2.04204224050045, + 2.0188374742865562, + 2.042632393538952, + 1.9534260630607605, + 1.9824335351586342, + 1.9986923038959503, + 2.0220199301838875, + 1.9932574182748795, + 1.9181597232818604, + 2.0238416120409966, + 1.9878152832388878, + 1.9782094284892082, + 2.0193252116441727, + 1.9845579043030739, + 2.046739801764488, + 2.020950235426426, + 1.9819394797086716, + 2.014990523457527, + 2.0171911492943764, + 1.9729845449328423, + 1.9985864087939262, + 1.9808142483234406, + 2.044060692191124, + 2.0136253610253334, + 2.007102943956852, + 2.0055581256747246, + 2.0118783488869667, + 1.9803542867302895, + 1.9742417559027672, + 2.002647653222084, + 2.035079687833786, + 1.9990946501493454, + 2.0111722126603127, + 2.023270756006241, + 2.023766651749611, + 2.011401891708374, + 2.0225120559334755, + 1.992932379245758, + 1.9212822243571281, + 2.0185446217656136, + 1.980887547135353, + 2.0115440785884857, + 1.9734217822551727, + 2.0147181376814842, + 2.0025433972477913, + 1.9461408481001854, + 1.9672214016318321, + 2.0352651178836823, + 2.008639059960842, + 2.0139544904232025, + 1.996946319937706, + 1.9962545037269592, + 2.02795672416687, + 2.0335527881979942, + 2.0064235627651215, + 1.9810653254389763 + ], + "val_steps": [ + 5000, + 10000, + 15000, + 20000, + 25000, + 30000, + 35000, + 40000, + 45000, + 50000, + 55000, + 60000, + 65000, + 70000, + 75000, + 80000, + 85000, + 90000, + 95000, + 100000, + 105000, + 110000, + 114000 + ], + "val_lm_loss": [ + 2.5849878358840943, + 2.320927424430847, + 2.2150809240341185, + 2.1528186082839964, + 2.1099301528930665, + 2.0791278100013733, + 2.0539102220535277, + 2.0340168333053588, + 2.0123712754249574, + 1.9963451862335204, + 1.9808279085159302, + 1.9664747095108033, + 1.9524222064018248, + 1.9410901188850402, + 1.929000527858734, + 1.9196160244941711, + 1.9107489800453186, + 1.9033442211151124, + 1.8995578336715697, + 1.8982738971710205, + 1.8966897821426392, + 1.8952697968482972, + 1.894669349193573 + ], + "val_ppl": [ + 13.263127754763513, + 10.18511585250881, + 9.162150519790792, + 8.60908988366503, + 8.247665188955184, + 7.997490540241393, + 7.798334786969767, + 7.64473238323566, + 7.481035927848285, + 7.36209976517497, + 7.24874179664803, + 7.1454422758193905, + 7.045733148250999, + 6.966340972737332, + 6.882627808001111, + 6.818339891079386, + 6.758148621902485, + 6.70829098026094, + 6.682938818368296, + 6.674363855320459, + 6.663799265204597, + 6.65434348336799, + 6.6503490977594195 + ] +} \ No newline at end of file diff --git a/075_cosmopedia_mol/mol_1plus2of5_best.pt b/075_cosmopedia_mol/mol_1plus2of5_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..15c5834505dd7eb62a964a403bcbafe603787951 --- /dev/null +++ b/075_cosmopedia_mol/mol_1plus2of5_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:524283bfce2ffb1dac2b969b5334b07e6e61bf40b83f889d468ba2cdf73bcc07 +size 458256644 diff --git a/075_cosmopedia_mol/mol_1plus2of5_results.json b/075_cosmopedia_mol/mol_1plus2of5_results.json new file mode 100644 index 0000000000000000000000000000000000000000..a82686ec296026ddf5e9985f3cff2fa5821320d6 --- /dev/null +++ b/075_cosmopedia_mol/mol_1plus2of5_results.json @@ -0,0 +1,584 @@ +{ + "label": "mol_1plus2of5", + "condition": "mol_1plus2of5", + "total_params": 114494464, + "steps": [ + 100050, + 100100, + 100150, + 100200, + 100250, + 100300, + 100350, + 100400, + 100450, + 100500, + 100550, + 100600, + 100650, + 100700, + 100750, + 100800, + 100850, + 100900, + 100950, + 101000, + 101050, + 101100, + 101150, + 101200, + 101250, + 101300, + 101350, + 101400, + 101450, + 101500, + 101550, + 101600, + 101650, + 101700, + 101750, + 101800, + 101850, + 101900, + 101950, + 102000, + 102050, + 102100, + 102150, + 102200, + 102250, + 102300, + 102350, + 102400, + 102450, + 102500, + 102550, + 102600, + 102650, + 102700, + 102750, + 102800, + 102850, + 102900, + 102950, + 103000, + 103050, + 103100, + 103150, + 103200, + 103250, + 103300, + 103350, + 103400, + 103450, + 103500, + 103550, + 103600, + 103650, + 103700, + 103750, + 103800, + 103850, + 103900, + 103950, + 104000, + 104050, + 104100, + 104150, + 104200, + 104250, + 104300, + 104350, + 104400, + 104450, + 104500, + 104550, + 104600, + 104650, + 104700, + 104750, + 104800, + 104850, + 104900, + 104950, + 105000, + 105050, + 105100, + 105150, + 105200, + 105250, + 105300, + 105350, + 105400, + 105450, + 105500, + 105550, + 105600, + 105650, + 105700, + 105750, + 105800, + 105850, + 105900, + 105950, + 106000, + 106050, + 106100, + 106150, + 106200, + 106250, + 106300, + 106350, + 106400, + 106450, + 106500, + 106550, + 106600, + 106650, + 106700, + 106750, + 106800, + 106850, + 106900, + 106950, + 107000, + 107050, + 107100, + 107150, + 107200, + 107250, + 107300, + 107350, + 107400, + 107450, + 107500, + 107550, + 107600, + 107650, + 107700, + 107750, + 107800, + 107850, + 107900, + 107950, + 108000, + 108050, + 108100, + 108150, + 108200, + 108250, + 108300, + 108350, + 108400, + 108450, + 108500, + 108550, + 108600, + 108650, + 108700, + 108750, + 108800, + 108850, + 108900, + 108950, + 109000, + 109050, + 109100, + 109150, + 109200, + 109250, + 109300, + 109350, + 109400, + 109450, + 109500, + 109550, + 109600, + 109650, + 109700, + 109750, + 109800, + 109850, + 109900, + 109950, + 110000, + 110050, + 110100, + 110150, + 110200, + 110250, + 110300, + 110350, + 110400, + 110450, + 110500, + 110550, + 110600, + 110650, + 110700, + 110750, + 110800, + 110850, + 110900, + 110950, + 111000, + 111050, + 111100, + 111150, + 111200, + 111250, + 111300, + 111350, + 111400, + 111450, + 111500, + 111550, + 111600, + 111650, + 111700, + 111750, + 111800, + 111850, + 111900, + 111950, + 112000, + 112050, + 112100, + 112150, + 112200, + 112250, + 112300, + 112350, + 112400, + 112450, + 112500, + 112550, + 112600, + 112650, + 112700, + 112750, + 112800, + 112850, + 112900, + 112950, + 113000, + 113050, + 113100, + 113150, + 113200, + 113250, + 113300, + 113350, + 113400, + 113450, + 113500, + 113550, + 113600, + 113650, + 113700, + 113750, + 113800, + 113850, + 113900, + 113950, + 114000 + ], + "train_lm_loss": [ + 2.001289501786232, + 1.9551296681165695, + 2.041274458169937, + 2.0209092274308205, + 2.0154442861676216, + 1.9923203736543655, + 1.9646783024072647, + 2.042202278971672, + 2.030858024954796, + 2.0716991275548935, + 2.078910730779171, + 1.9829012230038643, + 2.0394605100154877, + 2.020716480910778, + 2.0341348201036453, + 2.0213283002376556, + 2.055521972477436, + 2.0073730871081352, + 2.0151448398828506, + 2.002665027976036, + 2.0325949117541313, + 2.0228537917137146, + 2.008303590118885, + 2.035650633275509, + 2.030245862901211, + 2.017840199172497, + 2.021420508623123, + 2.046848066151142, + 1.9852649569511414, + 2.0331853851675987, + 2.030922368168831, + 2.0433784648776054, + 2.0342104211449623, + 2.0316098108887672, + 1.9445945918560028, + 1.9997464790940285, + 2.0768306627869606, + 2.007960759103298, + 1.9766897335648537, + 2.0065804794430733, + 2.058190993964672, + 1.952880136668682, + 2.0640340596437454, + 2.0173595771193504, + 2.029501959681511, + 2.014172025024891, + 2.047446183860302, + 2.016378380358219, + 1.991636961698532, + 2.0500078722834587, + 2.0504421293735504, + 2.033398002386093, + 2.029442608356476, + 2.0273576006293297, + 2.0209117084741592, + 2.0142086297273636, + 2.018518641591072, + 2.0031053498387337, + 2.0388581454753876, + 2.079189859330654, + 1.9669712334871292, + 1.9348777085542679, + 2.0345763713121414, + 2.0655450969934464, + 2.0574153065681458, + 2.025576137006283, + 2.0668927282094955, + 2.0143177658319473, + 2.022562049329281, + 2.011000894010067, + 2.0100079774856567, + 2.0039597675204277, + 2.0492690950632095, + 1.998364895582199, + 2.0485087111592293, + 1.9845037013292313, + 2.058056801557541, + 1.92637949436903, + 2.026476889848709, + 2.0170892626047134, + 1.9927861914038658, + 2.0430187955498695, + 1.9847640544176102, + 2.012586109340191, + 1.9935407638549805, + 2.03887839615345, + 1.960207998752594, + 2.024299681186676, + 2.081016629934311, + 2.005470871925354, + 1.999426119029522, + 2.0584355890750885, + 1.9979061037302017, + 1.992972306907177, + 2.0322128608822823, + 2.012782834470272, + 1.9925430119037628, + 2.0488206371665, + 2.0784814432263374, + 1.9993099570274353, + 2.005149967968464, + 1.990713968873024, + 2.026819072663784, + 2.0588905662298203, + 2.0180658996105194, + 2.06297717243433, + 2.0159102380275726, + 2.0013976469635963, + 2.015216186642647, + 1.9832714721560478, + 2.0320935621857643, + 2.021993190050125, + 1.9819439500570297, + 2.0237843468785286, + 2.0035738572478294, + 1.969828687608242, + 2.045904867351055, + 2.0052198097109795, + 2.0496697425842285, + 2.0762228220701218, + 2.0097621083259583, + 2.015100918710232, + 1.9911921545863152, + 2.035028539597988, + 1.980125479400158, + 2.0044068098068237, + 2.004147857427597, + 2.0491931810975075, + 1.9799936041235924, + 1.9580971747636795, + 1.9996263310313225, + 2.0339426919817924, + 2.011688381433487, + 1.9978149682283401, + 2.010793149471283, + 1.9922733083367348, + 1.9866824820637703, + 2.008448511362076, + 2.0051350072026253, + 2.020344413816929, + 2.0719862952828407, + 2.0170309245586395, + 2.0842183604836464, + 2.016372174024582, + 1.9605874791741371, + 2.0864978209137917, + 2.024491623044014, + 2.0358652994036674, + 1.9797537848353386, + 2.038851112127304, + 1.98940771818161, + 2.0087709948420525, + 2.032870002090931, + 2.0209733992815018, + 2.031089924275875, + 2.052151121199131, + 2.018813133239746, + 2.0522635877132416, + 2.015391543507576, + 1.9672488048672676, + 1.9891734719276428, + 1.953943982720375, + 2.0051299929618835, + 2.0355597138404846, + 1.9637102782726288, + 2.0100039169192314, + 1.9660901352763176, + 2.002008743584156, + 2.0053940042853355, + 2.031912200152874, + 2.007842503488064, + 2.0202999487519264, + 1.9940487816929817, + 1.922476477921009, + 2.0949045419692993, + 2.0341909304261208, + 2.0582487285137177, + 2.0316267758607864, + 2.009194239974022, + 1.9916953146457672, + 2.036462441086769, + 2.005436100065708, + 1.9858692213892937, + 2.033693052828312, + 2.0256827399134636, + 2.0464863032102585, + 2.0220676362514496, + 1.9574101641774178, + 2.010128788650036, + 1.9932949915528297, + 2.037899672985077, + 2.0013824328780174, + 2.0240444391965866, + 2.0150470808148384, + 2.004674270749092, + 2.0050778165459633, + 1.9178737476468086, + 2.029038652777672, + 1.9876767545938492, + 2.113120898604393, + 2.018742524087429, + 2.0390004962682724, + 2.0193790420889854, + 2.03308168053627, + 2.0047967731952667, + 2.0266451463103294, + 1.9766735136508942, + 1.984155811369419, + 2.0263472571969032, + 2.03941098600626, + 2.068647585809231, + 1.9963684305548668, + 2.0733064115047455, + 2.0019498094916344, + 2.0128383859992027, + 2.079353131353855, + 2.0508283898234367, + 1.9761463403701782, + 1.935135431587696, + 2.0658843889832497, + 2.0081754699349403, + 1.9817817658185959, + 1.9804302528500557, + 2.016237683594227, + 2.024924263358116, + 2.0671009868383408, + 2.0319792926311493, + 2.0597032085061073, + 2.000779412686825, + 2.0325248688459396, + 1.989590123295784, + 2.018339663743973, + 2.035477563738823, + 2.0223865434527397, + 2.048140287399292, + 1.9760183915495872, + 2.0070212185382843, + 2.0188827365636826, + 2.040121555328369, + 2.049492172896862, + 2.008780688047409, + 2.0317240580916405, + 2.0310164019465446, + 2.0429823771119118, + 2.045803263783455, + 2.0519163385033607, + 1.9996186271309853, + 2.047646664083004, + 2.074254520237446, + 2.0257286429405212, + 2.0395289063453674, + 1.981814980506897, + 2.0362001806497574, + 2.037872776389122, + 2.043371468782425, + 2.0086323469877243, + 1.9568907618522644, + 2.013168141245842, + 2.030580073595047, + 2.0319169014692307, + 2.0644395127892494, + 1.9886726960539818, + 2.0538244396448135, + 2.0172510147094727, + 1.9884796813130379, + 2.0327902734279633, + 2.0182114019989967, + 1.9967751279473305, + 2.0479191839694977, + 1.9911805093288422, + 1.9680505841970444, + 2.012942358851433, + 1.9893337786197662, + 2.0182977691292763, + 1.9982693269848824, + 2.065761335194111, + 2.0564338490366936, + 2.0361185148358345, + 1.9943371564149857, + 2.0239871963858604 + ], + "val_steps": [ + 105000, + 110000, + 114000 + ], + "val_lm_loss": [ + 1.9138352632522584, + 1.9116535091400146, + 1.9109137439727784 + ], + "val_ppl": [ + 6.77903840181122, + 6.76426432945801, + 6.759262212749069 + ] +} \ No newline at end of file diff --git a/075_cosmopedia_mol/mol_1plus2of5_sparse_best.pt b/075_cosmopedia_mol/mol_1plus2of5_sparse_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5491e0581695ca26b657c061d30a2f88b279b9c --- /dev/null +++ b/075_cosmopedia_mol/mol_1plus2of5_sparse_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fafda231b88458b8c98c24ec9ab087908180a920c01b619aa27d2348be6d66d +size 425185115 diff --git a/075_cosmopedia_mol/mol_1plus2of5_sparse_results.json b/075_cosmopedia_mol/mol_1plus2of5_sparse_results.json new file mode 100644 index 0000000000000000000000000000000000000000..84052045de195ef4559b65e1399cc8b2f16e52f1 --- /dev/null +++ b/075_cosmopedia_mol/mol_1plus2of5_sparse_results.json @@ -0,0 +1,2208 @@ +{ + "label": "mol_1plus2of5_sparse", + "condition": "mol_1plus2of5_sparse", + "total_params": 106218752, + "steps": [ + 60050, + 60100, + 60150, + 60200, + 60250, + 60300, + 60350, + 60400, + 60450, + 60500, + 60550, + 60600, + 60650, + 60700, + 60750, + 60800, + 60850, + 60900, + 60950, + 61000, + 61050, + 61100, + 61150, + 61200, + 61250, + 61300, + 61350, + 61400, + 61450, + 61500, + 61550, + 61600, + 61650, + 61700, + 61750, + 61800, + 61850, + 61900, + 61950, + 62000, + 62050, + 62100, + 62150, + 62200, + 62250, + 62300, + 62350, + 62400, + 62450, + 62500, + 62550, + 62600, + 62650, + 62700, + 62750, + 62800, + 62850, + 62900, + 62950, + 63000, + 63050, + 63100, + 63150, + 63200, + 63250, + 63300, + 63350, + 63400, + 63450, + 63500, + 63550, + 63600, + 63650, + 63700, + 63750, + 63800, + 63850, + 63900, + 63950, + 64000, + 64050, + 64100, + 64150, + 64200, + 64250, + 64300, + 64350, + 64400, + 64450, + 64500, + 64550, + 64600, + 64650, + 64700, + 64750, + 64800, + 64850, + 64900, + 64950, + 65000, + 65050, + 65100, + 65150, + 65200, + 65250, + 65300, + 65350, + 65400, + 65450, + 65500, + 65550, + 65600, + 65650, + 65700, + 65750, + 65800, + 65850, + 65900, + 65950, + 66000, + 66050, + 66100, + 66150, + 66200, + 66250, + 66300, + 66350, + 66400, + 66450, + 66500, + 66550, + 66600, + 66650, + 66700, + 66750, + 66800, + 66850, + 66900, + 66950, + 67000, + 67050, + 67100, + 67150, + 67200, + 67250, + 67300, + 67350, + 67400, + 67450, + 67500, + 67550, + 67600, + 67650, + 67700, + 67750, + 67800, + 67850, + 67900, + 67950, + 68000, + 68050, + 68100, + 68150, + 68200, + 68250, + 68300, + 68350, + 68400, + 68450, + 68500, + 68550, + 68600, + 68650, + 68700, + 68750, + 68800, + 68850, + 68900, + 68950, + 69000, + 69050, + 69100, + 69150, + 69200, + 69250, + 69300, + 69350, + 69400, + 69450, + 69500, + 69550, + 69600, + 69650, + 69700, + 69750, + 69800, + 69850, + 69900, + 69950, + 70000, + 70050, + 70100, + 70150, + 70200, + 70250, + 70300, + 70350, + 70400, + 70450, + 70500, + 70550, + 70600, + 70650, + 70700, + 70750, + 70800, + 70850, + 70900, + 70950, + 71000, + 71050, + 71100, + 71150, + 71200, + 71250, + 71300, + 71350, + 71400, + 71450, + 71500, + 71550, + 71600, + 71650, + 71700, + 71750, + 71800, + 71850, + 71900, + 71950, + 72000, + 72050, + 72100, + 72150, + 72200, + 72250, + 72300, + 72350, + 72400, + 72450, + 72500, + 72550, + 72600, + 72650, + 72700, + 72750, + 72800, + 72850, + 72900, + 72950, + 73000, + 73050, + 73100, + 73150, + 73200, + 73250, + 73300, + 73350, + 73400, + 73450, + 73500, + 73550, + 73600, + 73650, + 73700, + 73750, + 73800, + 73850, + 73900, + 73950, + 74000, + 74050, + 74100, + 74150, + 74200, + 74250, + 74300, + 74350, + 74400, + 74450, + 74500, + 74550, + 74600, + 74650, + 74700, + 74750, + 74800, + 74850, + 74900, + 74950, + 75000, + 75050, + 75100, + 75150, + 75200, + 75250, + 75300, + 75350, + 75400, + 75450, + 75500, + 75550, + 75600, + 75650, + 75700, + 75750, + 75800, + 75850, + 75900, + 75950, + 76000, + 76050, + 76100, + 76150, + 76200, + 76250, + 76300, + 76350, + 76400, + 76450, + 76500, + 76550, + 76600, + 76650, + 76700, + 76750, + 76800, + 76850, + 76900, + 76950, + 77000, + 77050, + 77100, + 77150, + 77200, + 77250, + 77300, + 77350, + 77400, + 77450, + 77500, + 77550, + 77600, + 77650, + 77700, + 77750, + 77800, + 77850, + 77900, + 77950, + 78000, + 78050, + 78100, + 78150, + 78200, + 78250, + 78300, + 78350, + 78400, + 78450, + 78500, + 78550, + 78600, + 78650, + 78700, + 78750, + 78800, + 78850, + 78900, + 78950, + 79000, + 79050, + 79100, + 79150, + 79200, + 79250, + 79300, + 79350, + 79400, + 79450, + 79500, + 79550, + 79600, + 79650, + 79700, + 79750, + 79800, + 79850, + 79900, + 79950, + 80000, + 80050, + 80100, + 80150, + 80200, + 80250, + 80300, + 80350, + 80400, + 80450, + 80500, + 80550, + 80600, + 80650, + 80700, + 80750, + 80800, + 80850, + 80900, + 80950, + 81000, + 81050, + 81100, + 81150, + 81200, + 81250, + 81300, + 81350, + 81400, + 81450, + 81500, + 81550, + 81600, + 81650, + 81700, + 81750, + 81800, + 81850, + 81900, + 81950, + 82000, + 82050, + 82100, + 82150, + 82200, + 82250, + 82300, + 82350, + 82400, + 82450, + 82500, + 82550, + 82600, + 82650, + 82700, + 82750, + 82800, + 82850, + 82900, + 82950, + 83000, + 83050, + 83100, + 83150, + 83200, + 83250, + 83300, + 83350, + 83400, + 83450, + 83500, + 83550, + 83600, + 83650, + 83700, + 83750, + 83800, + 83850, + 83900, + 83950, + 84000, + 84050, + 84100, + 84150, + 84200, + 84250, + 84300, + 84350, + 84400, + 84450, + 84500, + 84550, + 84600, + 84650, + 84700, + 84750, + 84800, + 84850, + 84900, + 84950, + 85000, + 85050, + 85100, + 85150, + 85200, + 85250, + 85300, + 85350, + 85400, + 85450, + 85500, + 85550, + 85600, + 85650, + 85700, + 85750, + 85800, + 85850, + 85900, + 85950, + 86000, + 86050, + 86100, + 86150, + 86200, + 86250, + 86300, + 86350, + 86400, + 86450, + 86500, + 86550, + 86600, + 86650, + 86700, + 86750, + 86800, + 86850, + 86900, + 86950, + 87000, + 87050, + 87100, + 87150, + 87200, + 87250, + 87300, + 87350, + 87400, + 87450, + 87500, + 87550, + 87600, + 87650, + 87700, + 87750, + 87800, + 87850, + 87900, + 87950, + 88000, + 88050, + 88100, + 88150, + 88200, + 88250, + 88300, + 88350, + 88400, + 88450, + 88500, + 88550, + 88600, + 88650, + 88700, + 88750, + 88800, + 88850, + 88900, + 88950, + 89000, + 89050, + 89100, + 89150, + 89200, + 89250, + 89300, + 89350, + 89400, + 89450, + 89500, + 89550, + 89600, + 89650, + 89700, + 89750, + 89800, + 89850, + 89900, + 89950, + 90000, + 90050, + 90100, + 90150, + 90200, + 90250, + 90300, + 90350, + 90400, + 90450, + 90500, + 90550, + 90600, + 90650, + 90700, + 90750, + 90800, + 90850, + 90900, + 90950, + 91000, + 91050, + 91100, + 91150, + 91200, + 91250, + 91300, + 91350, + 91400, + 91450, + 91500, + 91550, + 91600, + 91650, + 91700, + 91750, + 91800, + 91850, + 91900, + 91950, + 92000, + 92050, + 92100, + 92150, + 92200, + 92250, + 92300, + 92350, + 92400, + 92450, + 92500, + 92550, + 92600, + 92650, + 92700, + 92750, + 92800, + 92850, + 92900, + 92950, + 93000, + 93050, + 93100, + 93150, + 93200, + 93250, + 93300, + 93350, + 93400, + 93450, + 93500, + 93550, + 93600, + 93650, + 93700, + 93750, + 93800, + 93850, + 93900, + 93950, + 94000, + 94050, + 94100, + 94150, + 94200, + 94250, + 94300, + 94350, + 94400, + 94450, + 94500, + 94550, + 94600, + 94650, + 94700, + 94750, + 94800, + 94850, + 94900, + 94950, + 95000, + 95050, + 95100, + 95150, + 95200, + 95250, + 95300, + 95350, + 95400, + 95450, + 95500, + 95550, + 95600, + 95650, + 95700, + 95750, + 95800, + 95850, + 95900, + 95950, + 96000, + 96050, + 96100, + 96150, + 96200, + 96250, + 96300, + 96350, + 96400, + 96450, + 96500, + 96550, + 96600, + 96650, + 96700, + 96750, + 96800, + 96850, + 96900, + 96950, + 97000, + 97050, + 97100, + 97150, + 97200, + 97250, + 97300, + 97350, + 97400, + 97450, + 97500, + 97550, + 97600, + 97650, + 97700, + 97750, + 97800, + 97850, + 97900, + 97950, + 98000, + 98050, + 98100, + 98150, + 98200, + 98250, + 98300, + 98350, + 98400, + 98450, + 98500, + 98550, + 98600, + 98650, + 98700, + 98750, + 98800, + 98850, + 98900, + 98950, + 99000, + 99050, + 99100, + 99150, + 99200, + 99250, + 99300, + 99350, + 99400, + 99450, + 99500, + 99550, + 99600, + 99650, + 99700, + 99750, + 99800, + 99850, + 99900, + 99950, + 100000, + 100050, + 100100, + 100150, + 100200, + 100250, + 100300, + 100350, + 100400, + 100450, + 100500, + 100550, + 100600, + 100650, + 100700, + 100750, + 100800, + 100850, + 100900, + 100950, + 101000, + 101050, + 101100, + 101150, + 101200, + 101250, + 101300, + 101350, + 101400, + 101450, + 101500, + 101550, + 101600, + 101650, + 101700, + 101750, + 101800, + 101850, + 101900, + 101950, + 102000, + 102050, + 102100, + 102150, + 102200, + 102250, + 102300, + 102350, + 102400, + 102450, + 102500, + 102550, + 102600, + 102650, + 102700, + 102750, + 102800, + 102850, + 102900, + 102950, + 103000, + 103050, + 103100, + 103150, + 103200, + 103250, + 103300, + 103350, + 103400, + 103450, + 103500, + 103550, + 103600, + 103650, + 103700, + 103750, + 103800, + 103850, + 103900, + 103950, + 104000, + 104050, + 104100, + 104150, + 104200, + 104250, + 104300, + 104350, + 104400, + 104450, + 104500, + 104550, + 104600, + 104650, + 104700, + 104750, + 104800, + 104850, + 104900, + 104950, + 105000, + 105050, + 105100, + 105150, + 105200, + 105250, + 105300, + 105350, + 105400, + 105450, + 105500, + 105550, + 105600, + 105650, + 105700, + 105750, + 105800, + 105850, + 105900, + 105950, + 106000, + 106050, + 106100, + 106150, + 106200, + 106250, + 106300, + 106350, + 106400, + 106450, + 106500, + 106550, + 106600, + 106650, + 106700, + 106750, + 106800, + 106850, + 106900, + 106950, + 107000, + 107050, + 107100, + 107150, + 107200, + 107250, + 107300, + 107350, + 107400, + 107450, + 107500, + 107550, + 107600, + 107650, + 107700, + 107750, + 107800, + 107850, + 107900, + 107950, + 108000, + 108050, + 108100, + 108150, + 108200, + 108250, + 108300, + 108350, + 108400, + 108450, + 108500, + 108550, + 108600, + 108650, + 108700, + 108750, + 108800, + 108850, + 108900, + 108950, + 109000, + 109050, + 109100, + 109150, + 109200, + 109250, + 109300, + 109350, + 109400, + 109450, + 109500, + 109550, + 109600, + 109650, + 109700, + 109750, + 109800, + 109850, + 109900, + 109950, + 110000, + 110050, + 110100, + 110150, + 110200, + 110250, + 110300, + 110350, + 110400, + 110450, + 110500, + 110550, + 110600, + 110650, + 110700, + 110750, + 110800, + 110850, + 110900, + 110950, + 111000, + 111050, + 111100, + 111150, + 111200, + 111250, + 111300, + 111350, + 111400, + 111450, + 111500, + 111550, + 111600, + 111650, + 111700, + 111750, + 111800, + 111850, + 111900, + 111950, + 112000, + 112050, + 112100, + 112150, + 112200, + 112250, + 112300, + 112350, + 112400, + 112450, + 112500, + 112550, + 112600, + 112650, + 112700, + 112750, + 112800, + 112850, + 112900, + 112950, + 113000, + 113050, + 113100, + 113150, + 113200, + 113250, + 113300, + 113350, + 113400, + 113450, + 113500, + 113550, + 113600, + 113650, + 113700, + 113750, + 113800, + 113850, + 113900, + 113950, + 114000 + ], + "train_lm_loss": [ + 2.0250542759895325, + 2.057331882417202, + 2.1021080389618874, + 2.008370690047741, + 2.029400125145912, + 2.0877985060214996, + 1.995040588080883, + 2.0120574682950974, + 2.0191750898957253, + 2.0670781806111336, + 2.034296914935112, + 2.0482332035899162, + 2.0352775380015373, + 1.9688724279403687, + 2.0359607189893723, + 1.9948134124279022, + 2.0493184700608253, + 2.0434341579675674, + 2.020620636641979, + 2.0272665843367577, + 2.0568934828042984, + 2.047292411327362, + 2.0316168665885925, + 1.9822102040052414, + 2.039115823805332, + 2.0555785968899727, + 2.0891698971390724, + 2.0050444304943085, + 2.027771033346653, + 1.9571530893445015, + 2.0144145116209984, + 2.024355284869671, + 2.0130489096045494, + 2.072992704808712, + 2.082683317363262, + 1.999529890716076, + 2.0411072373390198, + 2.00109501183033, + 2.0709204003214836, + 2.004510946571827, + 2.069338396191597, + 2.0721755027770996, + 2.048461966216564, + 2.06471349298954, + 2.037347711622715, + 2.0071116983890533, + 2.0021990686655045, + 2.016736939549446, + 2.001908525824547, + 2.021449774503708, + 2.031206928193569, + 2.010918915271759, + 2.0163877829909325, + 1.9930402114987373, + 2.012791559100151, + 2.075031541287899, + 1.9873631671071053, + 2.0952394530177116, + 2.0284029841423035, + 2.028021566569805, + 2.0060755908489227, + 2.0138602927327156, + 1.995758056640625, + 2.0740970075130463, + 2.0445369854569435, + 2.035561092197895, + 2.022085890173912, + 2.0610901042819023, + 2.056417778134346, + 2.0270422995090485, + 2.08294128626585, + 2.0640812441706657, + 2.0111047476530075, + 2.0116816461086273, + 1.9510306119918823, + 2.036032997071743, + 2.0003975927829742, + 2.0299641638994217, + 2.034666381776333, + 2.0171654522418976, + 2.0490723848342896, + 2.029898524284363, + 2.0482155829668045, + 2.019521065056324, + 1.9920669272542, + 2.0310504734516144, + 2.002720460295677, + 2.0670372396707535, + 2.0000401437282562, + 2.032965213060379, + 2.019978068768978, + 2.038719691336155, + 1.9943210557103157, + 1.986189879477024, + 2.020727574825287, + 2.02372258156538, + 2.026793524622917, + 1.9615137800574303, + 2.035525731742382, + 2.0742262378335, + 1.9920580759644508, + 1.9771457836031914, + 1.9666787385940552, + 2.01411559432745, + 2.029321752488613, + 1.9762040302157402, + 2.056596390902996, + 2.045867197215557, + 2.0053212121129036, + 2.0479394048452377, + 2.0246034115552902, + 2.0505540668964386, + 2.018379382789135, + 2.056056298315525, + 2.0490079149603844, + 2.059238977730274, + 2.0074499547481537, + 2.0270326659083366, + 2.0107194930315018, + 2.0299482345581055, + 2.0345055386424065, + 1.9994812980294228, + 2.0305938497185707, + 2.008163519203663, + 2.0178981125354767, + 2.044559009373188, + 1.9793435782194138, + 2.0487285256385803, + 1.9650212153792381, + 2.035706415772438, + 2.0507244765758514, + 1.98606588691473, + 2.0119302943348885, + 1.9971163645386696, + 2.0637854263186455, + 2.0214947313070297, + 2.0211668089032173, + 2.033094972372055, + 2.0172595530748367, + 1.9918307811021805, + 1.9990672022104263, + 2.025709740817547, + 2.013679303228855, + 1.9565951600670815, + 2.0742776915431023, + 1.9777011349797249, + 1.979258194565773, + 2.009093716740608, + 2.010389134287834, + 2.0058331564068794, + 2.072892352938652, + 1.963305503129959, + 2.0380348563194275, + 2.0211242586374283, + 2.059187188744545, + 2.0046968907117844, + 2.016166426241398, + 1.9803659617900848, + 1.9844737648963928, + 1.9689635932445526, + 2.0118320882320404, + 2.019295610487461, + 2.0416675209999084, + 2.058293156325817, + 2.0338614508509636, + 2.0233554989099503, + 2.0202361196279526, + 2.020300768315792, + 1.9974136352539062, + 2.0254733934998512, + 2.0233226642012596, + 2.030871294438839, + 1.9794270396232605, + 2.003244198858738, + 1.9725883156061172, + 2.0017478242516518, + 2.013525404036045, + 2.015048027038574, + 2.0752932652831078, + 2.0724506825208664, + 2.0284092873334885, + 2.048761546611786, + 2.0387242436408997, + 2.037851169705391, + 2.0451302528381348, + 1.9938908591866493, + 2.021722286939621, + 2.019249700009823, + 2.033338852226734, + 1.9686501771211624, + 1.9704557284712791, + 2.015447363257408, + 1.9894745349884033, + 2.049222379922867, + 2.013758637011051, + 2.0334222465753555, + 1.9714605957269669, + 2.0460540279746056, + 2.0015268474817276, + 2.024698421359062, + 2.036707393825054, + 1.957229882478714, + 2.0089554712176323, + 2.001823380589485, + 1.9542280435562134, + 2.0586759001016617, + 2.0275282338261604, + 2.005808465182781, + 2.0031338706612587, + 1.9624977186322212, + 2.005478821694851, + 2.015153616666794, + 2.0037220492959023, + 2.018815368413925, + 2.041570045053959, + 2.05567529797554, + 2.0188386663794518, + 2.011980913579464, + 1.9957773685455322, + 1.981917418539524, + 1.9995735734701157, + 2.065388463437557, + 2.0122538954019547, + 2.0824172347784042, + 1.9996418431401253, + 2.0122305899858475, + 2.062305472791195, + 2.014561377465725, + 1.956069953739643, + 2.010718747973442, + 1.9619438871741295, + 2.0273731723427773, + 1.9980881586670876, + 2.046758733689785, + 1.964179813861847, + 1.9947218894958496, + 2.0344211906194687, + 2.0473815351724625, + 2.0003839135169983, + 2.0133262425661087, + 1.9776896238327026, + 2.0218959152698517, + 2.048094391822815, + 1.9203830659389496, + 2.0325102731585503, + 2.0270719826221466, + 2.014791816473007, + 2.0410363525152206, + 2.044798254966736, + 2.050816386938095, + 2.013152502477169, + 2.0016823783516884, + 2.0143284052610397, + 2.032889999449253, + 1.9730764776468277, + 2.0200945511460304, + 2.017214812338352, + 2.0226190835237503, + 2.0471665114164352, + 2.0350357592105865, + 2.02042019367218, + 1.9656536728143692, + 1.9759925231337547, + 1.9444510713219643, + 1.9634347930550575, + 2.0442440435290337, + 2.006098695099354, + 2.007126122713089, + 2.00041700899601, + 2.022779867053032, + 1.962333358824253, + 2.053527556359768, + 1.9606906920671463, + 1.993899755179882, + 1.9878468438982964, + 1.9777364432811737, + 1.975278116762638, + 1.9977806434035301, + 1.959518276154995, + 2.0048135966062546, + 2.0153064876794815, + 2.002032697200775, + 2.0110738798975945, + 2.030898578464985, + 1.9813065454363823, + 1.9492442235350609, + 2.0400026217103004, + 2.0476904436945915, + 1.992023415863514, + 1.9916251748800278, + 2.0238887071609497, + 1.981847956776619, + 2.01261442899704, + 2.0002564042806625, + 1.9913906678557396, + 2.0548205971717834, + 1.9864331260323524, + 2.017364002764225, + 1.977040782570839, + 1.9433584362268448, + 2.012100987136364, + 1.9905450344085693, + 2.017070434987545, + 2.0125022009015083, + 2.0232071056962013, + 1.9404983818531036, + 2.02436426281929, + 1.9438798874616623, + 1.9540192112326622, + 1.999972902238369, + 2.0199707075953484, + 2.0393287613987923, + 1.9841329455375671, + 1.9825616255402565, + 1.983955331146717, + 1.9861484467983246, + 2.0404275357723236, + 1.9728491306304932, + 2.0372123271226883, + 1.999896228313446, + 1.9998879805207253, + 2.0138180628418922, + 1.9249490201473236, + 1.9762063696980476, + 1.9862096086144447, + 2.0059803873300552, + 1.9982033595442772, + 1.9728611037135124, + 2.0085658356547356, + 1.9786823242902756, + 1.991175316274166, + 2.030266225337982, + 2.0166886523365974, + 1.996365062892437, + 1.9840614348649979, + 1.9421532079577446, + 2.009944774210453, + 2.0076978281140327, + 2.0053780674934387, + 2.054280750453472, + 2.00362940877676, + 2.0251545533537865, + 1.9411423802375793, + 2.011134497821331, + 1.9914167895913124, + 1.9673414453864098, + 1.951619140803814, + 1.9881834238767624, + 2.0358935594558716, + 1.951860524713993, + 1.9917467534542084, + 1.9945037886500359, + 2.0347493439912796, + 2.0088369846343994, + 1.9983920231461525, + 2.034728057682514, + 1.9964691251516342, + 2.0294446125626564, + 1.9399973824620247, + 2.05042814463377, + 2.0080306604504585, + 1.927702121436596, + 1.9906114786863327, + 1.9854803085327148, + 2.022420294582844, + 2.0236504897475243, + 1.9986706376075745, + 1.980454370379448, + 2.000146046280861, + 1.9737649410963058, + 2.0059226751327515, + 1.9811391457915306, + 2.0057298988103867, + 2.004379227757454, + 1.9604864194989204, + 1.9966696351766586, + 2.004213899374008, + 1.9903723150491714, + 1.961173728108406, + 1.9531460627913475, + 1.9962880313396454, + 2.0056291818618774, + 2.0036266520619392, + 1.9487337619066238, + 1.9799667969346046, + 1.993325062096119, + 1.9837997108697891, + 1.955023393034935, + 1.9754050597548485, + 1.9930489659309387, + 1.954698458313942, + 1.9787892922759056, + 1.9843337088823318, + 1.993442177772522, + 1.962953619658947, + 1.9816113784909248, + 2.01363193243742, + 1.9633957669138908, + 1.9694047421216965, + 1.9805756956338882, + 2.0262081250548363, + 1.953658677637577, + 1.9774283170700073, + 1.976919747889042, + 1.969951055943966, + 1.965876154601574, + 1.983060136437416, + 1.9562422409653664, + 2.01482305675745, + 1.981579914689064, + 1.995715171098709, + 1.9473037645220757, + 2.001805752515793, + 1.9499823674559593, + 1.9620179161429405, + 2.007576070725918, + 1.9981181845068932, + 1.9761736541986465, + 1.9659725055098534, + 2.006350316107273, + 2.0897654816508293, + 1.9819058403372765, + 1.9415889829397202, + 2.051852725446224, + 1.9846997782588005, + 1.9986457452178001, + 1.9711838364601135, + 1.9515183344483376, + 2.008591838181019, + 1.9392461478710175, + 2.0023389980196953, + 1.9618056192994118, + 1.9464947879314423, + 2.0027160197496414, + 1.9691316187381744, + 1.9860540926456451, + 1.9665094017982483, + 1.951232187449932, + 1.9346229881048203, + 1.9572809785604477, + 1.9946486204862595, + 1.9989130795001984, + 1.9919464066624641, + 1.9411803856492043, + 1.9834791868925095, + 1.9743818491697311, + 1.974260687828064, + 1.9731061309576035, + 1.9631740525364876, + 1.9268346279859543, + 1.9550068601965904, + 1.9735533595085144, + 2.020361542701721, + 1.970843717455864, + 2.034765236079693, + 1.9569761231541634, + 1.9949899539351463, + 1.9655024856328964, + 1.9689453840255737, + 2.002186708152294, + 1.9538202062249184, + 2.003508523106575, + 1.940126694738865, + 1.9587059766054153, + 1.9323561787605286, + 1.9882853627204895, + 1.9820037633180618, + 2.007380671799183, + 1.9410480335354805, + 2.023584596812725, + 1.9808835089206696, + 2.0247578471899033, + 2.0643976107239723, + 2.0238001868128777, + 1.9859128519892693, + 1.9568789452314377, + 1.9943653866648674, + 1.9907899051904678, + 1.936934731900692, + 2.0010597929358482, + 1.9592686221003532, + 1.9333073794841766, + 1.9686818942427635, + 1.9730372428894043, + 1.9541517719626427, + 1.9538477808237076, + 1.9465704560279846, + 2.012022480368614, + 1.9894006848335266, + 1.90452391654253, + 1.9927686154842377, + 1.9736082553863525, + 2.009472504258156, + 2.008554272353649, + 2.003619320690632, + 1.997200109064579, + 1.9958697259426117, + 2.0225279480218887, + 1.9320406839251518, + 1.9606122821569443, + 1.9789583384990692, + 1.9089524149894714, + 1.9197222143411636, + 1.9483937248587608, + 1.911456547677517, + 1.9671903774142265, + 1.9706987589597702, + 2.003311313688755, + 1.981112003326416, + 1.9607456475496292, + 1.9722808450460434, + 1.968250721693039, + 1.9753756299614906, + 1.929234467446804, + 1.962664656341076, + 1.9623637348413467, + 2.0529779493808746, + 1.9691200107336044, + 1.9979122281074524, + 1.9453459903597832, + 1.928570568561554, + 1.9977025240659714, + 1.979574330151081, + 1.9393123537302017, + 1.9499324634671211, + 2.005984880030155, + 1.9871726632118225, + 1.9496242702007294, + 1.9526836350560188, + 1.9717027097940445, + 1.9772717207670212, + 1.9702864363789558, + 1.956548422574997, + 1.9911331981420517, + 1.9935265928506851, + 1.9639040678739548, + 1.9686717689037323, + 1.9534214213490486, + 1.984277419745922, + 1.9721368998289108, + 1.964972898364067, + 2.0017483979463577, + 1.9843996539711952, + 1.9520883038640022, + 1.948622703552246, + 1.9577801749110222, + 1.954174779355526, + 2.0052175149321556, + 1.9419321119785309, + 1.9913478344678879, + 1.9980424121022224, + 1.9597927331924438, + 1.965664952993393, + 1.9848425909876823, + 1.9758091941475868, + 1.9757762253284454, + 1.959643431007862, + 1.9860613122582436, + 2.0070865526795387, + 1.9926846101880074, + 1.9339968264102936, + 2.0583666265010834, + 1.95473463088274, + 2.0238833352923393, + 2.0075209513306618, + 1.923511415719986, + 1.9665187299251556, + 1.9420418217778206, + 1.9419189617037773, + 1.9881150349974632, + 1.9575437307357788, + 1.9417406022548676, + 1.9408239722251892, + 1.9926910176873207, + 1.9927591159939766, + 1.932097814977169, + 1.9486452117562294, + 1.9782610908150673, + 1.9209966287016869, + 1.9904899597167969, + 1.9732653573155403, + 1.9349883496761322, + 1.986519232392311, + 1.9811458438634872, + 1.9426657855510712, + 1.9491769149899483, + 1.937469370663166, + 1.9167005941271782, + 1.9595447927713394, + 1.9909798502922058, + 1.9920724406838417, + 1.9805895760655403, + 2.0356534644961357, + 1.9637053236365318, + 1.968021422624588, + 1.9557213559746742, + 1.9121548607945442, + 1.954408422112465, + 2.0089591816067696, + 1.9416941180825233, + 1.9418287873268127, + 1.9672783315181732, + 1.994765892624855, + 1.9989753291010857, + 1.9897397831082344, + 1.9895069226622581, + 1.9318257346749306, + 1.930019311606884, + 2.0100742280483246, + 1.9902346730232239, + 1.9529986679553986, + 1.9543410316109657, + 1.9548768028616905, + 1.9421386122703552, + 1.9703772887587547, + 2.043737731873989, + 2.0120805725455284, + 1.96657744795084, + 2.00029443949461, + 2.033767431974411, + 1.9900243133306503, + 2.01279079914093, + 1.948428064584732, + 1.9108914136886597, + 2.008934609591961, + 1.9317841231822968, + 1.9978442937135696, + 1.9607024043798447, + 2.000874362885952, + 1.937518060207367, + 1.9396809935569763, + 1.939461275935173, + 1.9768397212028503, + 1.8846075311303139, + 1.9311695620417595, + 1.9592086151242256, + 1.9465832188725471, + 1.9753689393401146, + 1.969374381005764, + 1.9787705391645432, + 1.9410793408751488, + 1.979812890291214, + 2.01040118932724, + 1.949989303946495, + 2.0019362047314644, + 1.9223362281918526, + 1.9524292647838593, + 1.9334975257515907, + 2.009930767118931, + 1.979419782757759, + 1.9809469431638718, + 1.9625727087259293, + 1.985398106276989, + 1.9135854318737984, + 1.9715737998485565, + 1.9631994143128395, + 1.9175495132803917, + 1.9753544852137566, + 1.9695724546909332, + 1.965698704123497, + 1.943047247827053, + 1.9343064799904823, + 1.9648913517594337, + 1.9528540521860123, + 1.916188970208168, + 1.9789985492825508, + 2.0059542134404182, + 1.9455155357718468, + 1.9402378872036934, + 1.9459763318300247, + 1.9907563924789429, + 1.9861151650547981, + 1.9091672003269196, + 1.9686033502221107, + 1.9130907952785492, + 1.9684004113078117, + 1.948289342224598, + 1.969929315149784, + 1.9502933248877525, + 1.9647818133234978, + 1.9688132032752037, + 1.9469343796372414, + 1.9954149723052979, + 1.953538030385971, + 1.9333030059933662, + 1.930277168750763, + 1.965681791305542, + 1.956645779311657, + 1.928692027926445, + 1.9473898708820343, + 1.9436273276805878, + 1.9747768640518188, + 1.9672619253396988, + 1.931155450642109, + 1.9339511543512344, + 1.9290276244282722, + 1.983862355351448, + 1.9478144496679306, + 1.9308874234557152, + 1.9385715648531914, + 1.9149080142378807, + 1.9526645317673683, + 1.9937643185257912, + 1.978908695280552, + 1.9775174707174301, + 1.9078204780817032, + 1.9751239717006683, + 1.9723815247416496, + 1.974222056567669, + 1.9622621983289719, + 1.9527732953429222, + 1.9364110380411148, + 1.9810910299420357, + 1.91212560236454, + 1.9945035204291344, + 2.0037218928337097, + 1.9823364540934563, + 1.926272600889206, + 1.9279989302158356, + 1.9269895628094673, + 1.936452865600586, + 1.957732118666172, + 1.9088626354932785, + 2.036380358040333, + 1.9294232577085495, + 1.971923179924488, + 1.9120749905705452, + 1.9402766451239586, + 1.92438642680645, + 1.979606844484806, + 2.0190165042877197, + 1.992316611111164, + 1.9867475777864456, + 1.9533641338348389, + 1.9276315569877625, + 1.9176368042826653, + 1.9596388190984726, + 1.9358405321836472, + 1.9337618425488472, + 1.948826976120472, + 1.9724335968494415, + 1.9742985144257545, + 1.928933598101139, + 1.9621814340353012, + 1.952053502202034, + 1.9411645010113716, + 2.013727717101574, + 2.0138801485300064, + 1.8990284353494644, + 1.9612069204449654, + 1.9756510183215141, + 1.9114684537053108, + 1.9512700363993645, + 1.967313677072525, + 1.918452188372612, + 1.9862303733825684, + 1.9837543368339539, + 1.9692009016871452, + 1.9276557639241219, + 1.968496285378933, + 1.9896360710263252, + 1.9717752635478973, + 2.0082840025424957, + 1.976219892501831, + 2.003122739493847, + 1.9324949011206627, + 1.9459064528346062, + 1.9283039271831512, + 1.9475337788462639, + 1.9726758897304535, + 1.9253366738557816, + 1.9540219381451607, + 1.9781785532832146, + 1.9510220065712929, + 1.9512445107102394, + 1.9371732622385025, + 2.004064440727234, + 1.9872586727142334, + 1.9769022911787033, + 1.9801130145788193, + 1.9942347556352615, + 1.9987122938036919, + 1.9667104184627533, + 1.9370891526341438, + 1.9560095369815826, + 1.9681200459599495, + 1.9722236394882202, + 1.9898173063993454, + 1.935530185699463, + 1.8864601105451584, + 1.9630066454410553, + 1.9758243784308434, + 2.001752257347107, + 1.923615574836731, + 1.9472332820296288, + 1.9981622993946075, + 1.973135083913803, + 1.9509285241365433, + 1.885297603905201, + 2.0211476162075996, + 1.9071829989552498, + 1.9928844422101974, + 2.00692717730999, + 1.9842872321605682, + 2.00109251588583, + 1.9818494617938995, + 1.936644658446312, + 1.9584044367074966, + 1.9538073912262917, + 1.9085844308137894, + 2.034277558326721, + 1.9249130487442017, + 2.020281068980694, + 1.9819010198116302, + 2.000545024871826, + 1.9408598840236664, + 1.9750497564673424, + 1.9491553232073784, + 1.9623931050300598, + 1.9741264954209328, + 1.9875112175941467, + 1.9845987036824226, + 1.9898283928632736, + 2.0625792890787125, + 1.9260376617312431, + 1.9667504280805588, + 1.9489487260580063, + 1.938467875123024, + 1.9998233914375305, + 1.9764689356088638, + 1.95829788595438, + 1.9882329180836678, + 1.9973010048270226, + 1.9566485211253166, + 1.9661596342921257, + 1.908978708088398, + 1.9574913308024406, + 1.9623322263360023, + 1.9891720339655876, + 1.9324819296598434, + 1.9374082013964653, + 1.9508399590849876, + 1.9293567687273026, + 1.9535465314984322, + 1.9631175696849823, + 1.9789586067199707, + 1.9475156217813492, + 1.9747132360935211, + 1.9797033071517944, + 1.95493695884943, + 1.9521906226873398, + 2.002411127090454, + 1.9159511104226112, + 2.008476249873638, + 1.9105474799871445, + 1.9225244373083115, + 1.9776587411761284, + 1.9583446234464645, + 1.9479091688990593, + 1.9362523034214973, + 1.9082248732447624, + 1.9964978694915771, + 1.920926921069622, + 1.9167449623346329, + 1.9130030274391174, + 1.943030759692192, + 1.9833993837237358, + 1.9631337672472, + 1.9407937079668045, + 1.899339608848095, + 1.9470821097493172, + 1.965896598994732, + 1.9798924401402473, + 1.9170609563589096, + 1.9496226906776428, + 1.9600538164377213, + 1.8698056042194366, + 1.9805264547467232, + 1.941338837146759, + 1.9266857206821442, + 1.9516558200120926, + 1.9420401379466057, + 1.987006962299347, + 1.947738103568554, + 1.9706010818481445, + 1.947960503399372, + 1.9751052185893059, + 1.9319636449217796, + 1.9905009716749191, + 1.9436218664050102, + 1.940003663301468, + 1.9453891590237617, + 1.9715038686990738, + 2.00382898747921, + 1.9477650001645088, + 1.951663427054882, + 2.013366237282753, + 2.027686983346939, + 1.9522991999983788, + 1.9088076949119568, + 1.943908266723156, + 1.9589112028479576, + 1.9399935975670815, + 1.9466041550040245, + 1.9583351165056229, + 1.9711578041315079, + 1.972341500222683, + 1.9629981145262718, + 1.9255429431796074, + 1.9997840449213982, + 1.978601150214672, + 1.9555037021636963, + 1.9360194578766823, + 1.9074658080935478, + 1.9076490625739098, + 1.986734688282013, + 2.041659437119961, + 1.926809400320053, + 1.9595110565423965, + 1.895837664604187, + 1.926360733807087, + 1.9298450946807861, + 1.9697455242276192, + 1.9618245959281921, + 1.9282527714967728, + 1.9805611297488213, + 1.9747591242194176, + 1.9506695941090584, + 1.9602693691849709, + 1.951341800391674, + 1.9176730290055275, + 1.9396012648940086, + 1.9366328939795494, + 2.0281129106879234, + 1.985909678041935, + 1.998476967215538, + 1.9560641944408417, + 1.962735466659069, + 1.9411671832203865, + 1.9671385139226913, + 1.9731572046875954, + 1.9601443484425545, + 1.9375286549329758, + 1.9675467163324356, + 1.884014330804348, + 1.9762062057852745, + 1.9689671620726585, + 2.0042752251029015, + 2.0077598989009857, + 1.9941276460886002, + 1.9656305462121964, + 1.9406152665615082, + 1.9797475934028625, + 1.9954305738210678, + 1.9820226281881332, + 1.9354880154132843, + 1.967509187757969, + 1.929177775979042, + 1.944750152528286, + 1.9412413462996483, + 1.9266808107495308, + 1.9431317821145058, + 2.030749298632145, + 1.941776417195797, + 1.91646608710289, + 1.9930154085159302, + 1.9590728133916855, + 1.8970853313803673, + 1.9648876413702965, + 1.9867413640022278, + 1.9846311435103416, + 1.9910594746470451, + 1.9763456210494041, + 1.9735301807522774, + 1.9849067777395248, + 1.969221979379654, + 1.9305119514465332, + 1.9627868384122849, + 1.9417811781167984, + 1.987233228981495, + 2.0120777413249016, + 1.9673952162265778, + 1.9213355779647827, + 1.9359928369522095, + 1.994310885667801, + 1.9661416485905647, + 1.9612787440419197, + 1.9601325541734695, + 1.9784268364310265, + 1.9837863072752953, + 1.982413798570633, + 1.9604801908135414, + 1.948323592543602, + 1.9878549426794052, + 1.9397111311554909, + 1.9765613749623299, + 1.934636428952217, + 1.9459659308195114, + 1.8937472626566887, + 1.933688223361969, + 1.922104924917221, + 1.966235801577568, + 2.047500252723694, + 2.006844848394394, + 1.9184957072138786, + 2.0010698810219765, + 1.9549897909164429, + 1.9759212210774422, + 1.965776763856411, + 1.9849668964743614, + 1.9252301305532455, + 1.9623402655124664, + 1.9503879249095917, + 1.9871000200510025, + 1.966336041688919, + 1.9661304280161858, + 1.9785211235284805, + 1.9211297705769539, + 1.9326500967144966, + 1.975357860326767, + 1.9346897900104523, + 1.9733733832836151, + 1.9606999903917313, + 1.9550388529896736, + 1.9118377417325974, + 1.9777746498584747, + 1.9513186365365982, + 1.96848476678133, + 1.9783331826329231, + 1.9375919848680496, + 1.9432909786701202, + 1.9327085390686989, + 1.9649215042591095, + 1.9774200916290283, + 1.942120611667633, + 1.9161308333277702, + 1.9186554998159409, + 1.9556018263101578, + 1.9062335044145584, + 1.9645711109042168, + 1.9488106817007065, + 1.9659539014101028, + 1.9441681131720543, + 1.917533926665783, + 1.9678318575024605, + 1.9537648260593414, + 1.9461980387568474, + 1.9346711337566376, + 1.9433982968330383, + 1.9445807188749313, + 1.9976390153169632, + 1.914852425456047, + 1.9469269067049026, + 1.9758106619119644, + 1.9694431498646736, + 1.9617199674248695, + 1.9886367917060852, + 1.9734699428081512, + 1.9375826492905617, + 1.9974892064929008, + 1.9338751807808876, + 1.9639604911208153, + 1.9405117258429527, + 1.9407489448785782, + 1.9541997462511063, + 1.9630483016371727, + 2.009735144674778, + 1.9822835251688957, + 1.9571461379528046, + 1.9271455705165863, + 1.9271863549947739, + 1.8849033415317535, + 1.9545234143733978, + 1.9639728516340256, + 1.9170955196022987, + 1.9808934703469276, + 1.9735526740550995, + 1.9410394504666328, + 1.9564342498779297, + 1.9566906094551086 + ], + "val_steps": [ + 65000, + 70000, + 75000, + 80000, + 85000, + 90000, + 95000, + 100000, + 105000, + 110000, + 114000 + ], + "val_lm_loss": [ + 1.9414531254768372, + 1.9268969440460204, + 1.9145912384986878, + 1.9032046914100647, + 1.8934630823135377, + 1.8843062782287598, + 1.8810933542251587, + 1.8783321642875672, + 1.8769911456108093, + 1.8759204816818238, + 1.8741961073875428 + ], + "val_ppl": [ + 6.968870259477246, + 6.868164840920281, + 6.7841651246311985, + 6.707355039696364, + 6.64233183835151, + 6.581786929020317, + 6.560674082956543, + 6.542583802538608, + 6.533815955702289, + 6.52682417822976, + 6.515579188465557 + ] +} \ No newline at end of file diff --git a/075_cosmopedia_mol/mol_k5_top3_best.pt b/075_cosmopedia_mol/mol_k5_top3_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..125f91dc3f27e6df7ff18e7b365fa3f5efd76a3b --- /dev/null +++ b/075_cosmopedia_mol/mol_k5_top3_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bffb7a2ed17b463803a0f7eae86f3ebbb5d078a5124b6533e8ffbea50f9246d7 +size 416217874 diff --git a/075_cosmopedia_mol/mol_k5_top3_results.json b/075_cosmopedia_mol/mol_k5_top3_results.json new file mode 100644 index 0000000000000000000000000000000000000000..c2cdcd9c4b81e48722205d2b04a8547fd29583ea --- /dev/null +++ b/075_cosmopedia_mol/mol_k5_top3_results.json @@ -0,0 +1,4644 @@ +{ + "label": "mol_k5_top3", + "condition": "mol_k5_top3", + "total_params": 103996416, + "steps": [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, + 1000, + 1050, + 1100, + 1150, + 1200, + 1250, + 1300, + 1350, + 1400, + 1450, + 1500, + 1550, + 1600, + 1650, + 1700, + 1750, + 1800, + 1850, + 1900, + 1950, + 2000, + 2050, + 2100, + 2150, + 2200, + 2250, + 2300, + 2350, + 2400, + 2450, + 2500, + 2550, + 2600, + 2650, + 2700, + 2750, + 2800, + 2850, + 2900, + 2950, + 3000, + 3050, + 3100, + 3150, + 3200, + 3250, + 3300, + 3350, + 3400, + 3450, + 3500, + 3550, + 3600, + 3650, + 3700, + 3750, + 3800, + 3850, + 3900, + 3950, + 4000, + 4050, + 4100, + 4150, + 4200, + 4250, + 4300, + 4350, + 4400, + 4450, + 4500, + 4550, + 4600, + 4650, + 4700, + 4750, + 4800, + 4850, + 4900, + 4950, + 5000, + 5050, + 5100, + 5150, + 5200, + 5250, + 5300, + 5350, + 5400, + 5450, + 5500, + 5550, + 5600, + 5650, + 5700, + 5750, + 5800, + 5850, + 5900, + 5950, + 6000, + 6050, + 6100, + 6150, + 6200, + 6250, + 6300, + 6350, + 6400, + 6450, + 6500, + 6550, + 6600, + 6650, + 6700, + 6750, + 6800, + 6850, + 6900, + 6950, + 7000, + 7050, + 7100, + 7150, + 7200, + 7250, + 7300, + 7350, + 7400, + 7450, + 7500, + 7550, + 7600, + 7650, + 7700, + 7750, + 7800, + 7850, + 7900, + 7950, + 8000, + 8050, + 8100, + 8150, + 8200, + 8250, + 8300, + 8350, + 8400, + 8450, + 8500, + 8550, + 8600, + 8650, + 8700, + 8750, + 8800, + 8850, + 8900, + 8950, + 9000, + 9050, + 9100, + 9150, + 9200, + 9250, + 9300, + 9350, + 9400, + 9450, + 9500, + 9550, + 9600, + 9650, + 9700, + 9750, + 9800, + 9850, + 9900, + 9950, + 10000, + 10050, + 10100, + 10150, + 10200, + 10250, + 10300, + 10350, + 10400, + 10450, + 10500, + 10550, + 10600, + 10650, + 10700, + 10750, + 10800, + 10850, + 10900, + 10950, + 11000, + 11050, + 11100, + 11150, + 11200, + 11250, + 11300, + 11350, + 11400, + 11450, + 11500, + 11550, + 11600, + 11650, + 11700, + 11750, + 11800, + 11850, + 11900, + 11950, + 12000, + 12050, + 12100, + 12150, + 12200, + 12250, + 12300, + 12350, + 12400, + 12450, + 12500, + 12550, + 12600, + 12650, + 12700, + 12750, + 12800, + 12850, + 12900, + 12950, + 13000, + 13050, + 13100, + 13150, + 13200, + 13250, + 13300, + 13350, + 13400, + 13450, + 13500, + 13550, + 13600, + 13650, + 13700, + 13750, + 13800, + 13850, + 13900, + 13950, + 14000, + 14050, + 14100, + 14150, + 14200, + 14250, + 14300, + 14350, + 14400, + 14450, + 14500, + 14550, + 14600, + 14650, + 14700, + 14750, + 14800, + 14850, + 14900, + 14950, + 15000, + 15050, + 15100, + 15150, + 15200, + 15250, + 15300, + 15350, + 15400, + 15450, + 15500, + 15550, + 15600, + 15650, + 15700, + 15750, + 15800, + 15850, + 15900, + 15950, + 16000, + 16050, + 16100, + 16150, + 16200, + 16250, + 16300, + 16350, + 16400, + 16450, + 16500, + 16550, + 16600, + 16650, + 16700, + 16750, + 16800, + 16850, + 16900, + 16950, + 17000, + 17050, + 17100, + 17150, + 17200, + 17250, + 17300, + 17350, + 17400, + 17450, + 17500, + 17550, + 17600, + 17650, + 17700, + 17750, + 17800, + 17850, + 17900, + 17950, + 18000, + 18050, + 18100, + 18150, + 18200, + 18250, + 18300, + 18350, + 18400, + 18450, + 18500, + 18550, + 18600, + 18650, + 18700, + 18750, + 18800, + 18850, + 18900, + 18950, + 19000, + 19050, + 19100, + 19150, + 19200, + 19250, + 19300, + 19350, + 19400, + 19450, + 19500, + 19550, + 19600, + 19650, + 19700, + 19750, + 19800, + 19850, + 19900, + 19950, + 20000, + 20050, + 20100, + 20150, + 20200, + 20250, + 20300, + 20350, + 20400, + 20450, + 20500, + 20550, + 20600, + 20650, + 20700, + 20750, + 20800, + 20850, + 20900, + 20950, + 21000, + 21050, + 21100, + 21150, + 21200, + 21250, + 21300, + 21350, + 21400, + 21450, + 21500, + 21550, + 21600, + 21650, + 21700, + 21750, + 21800, + 21850, + 21900, + 21950, + 22000, + 22050, + 22100, + 22150, + 22200, + 22250, + 22300, + 22350, + 22400, + 22450, + 22500, + 22550, + 22600, + 22650, + 22700, + 22750, + 22800, + 22850, + 22900, + 22950, + 23000, + 23050, + 23100, + 23150, + 23200, + 23250, + 23300, + 23350, + 23400, + 23450, + 23500, + 23550, + 23600, + 23650, + 23700, + 23750, + 23800, + 23850, + 23900, + 23950, + 24000, + 24050, + 24100, + 24150, + 24200, + 24250, + 24300, + 24350, + 24400, + 24450, + 24500, + 24550, + 24600, + 24650, + 24700, + 24750, + 24800, + 24850, + 24900, + 24950, + 25000, + 25050, + 25100, + 25150, + 25200, + 25250, + 25300, + 25350, + 25400, + 25450, + 25500, + 25550, + 25600, + 25650, + 25700, + 25750, + 25800, + 25850, + 25900, + 25950, + 26000, + 26050, + 26100, + 26150, + 26200, + 26250, + 26300, + 26350, + 26400, + 26450, + 26500, + 26550, + 26600, + 26650, + 26700, + 26750, + 26800, + 26850, + 26900, + 26950, + 27000, + 27050, + 27100, + 27150, + 27200, + 27250, + 27300, + 27350, + 27400, + 27450, + 27500, + 27550, + 27600, + 27650, + 27700, + 27750, + 27800, + 27850, + 27900, + 27950, + 28000, + 28050, + 28100, + 28150, + 28200, + 28250, + 28300, + 28350, + 28400, + 28450, + 28500, + 28550, + 28600, + 28650, + 28700, + 28750, + 28800, + 28850, + 28900, + 28950, + 29000, + 29050, + 29100, + 29150, + 29200, + 29250, + 29300, + 29350, + 29400, + 29450, + 29500, + 29550, + 29600, + 29650, + 29700, + 29750, + 29800, + 29850, + 29900, + 29950, + 30000, + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000, + 50050, + 50100, + 50150, + 50200, + 50250, + 50300, + 50350, + 50400, + 50450, + 50500, + 50550, + 50600, + 50650, + 50700, + 50750, + 50800, + 50850, + 50900, + 50950, + 51000, + 51050, + 51100, + 51150, + 51200, + 51250, + 51300, + 51350, + 51400, + 51450, + 51500, + 51550, + 51600, + 51650, + 51700, + 51750, + 51800, + 51850, + 51900, + 51950, + 52000, + 52050, + 52100, + 52150, + 52200, + 52250, + 52300, + 52350, + 52400, + 52450, + 52500, + 52550, + 52600, + 52650, + 52700, + 52750, + 52800, + 52850, + 52900, + 52950, + 53000, + 53050, + 53100, + 53150, + 53200, + 53250, + 53300, + 53350, + 53400, + 53450, + 53500, + 53550, + 53600, + 53650, + 53700, + 53750, + 53800, + 53850, + 53900, + 53950, + 54000, + 54050, + 54100, + 54150, + 54200, + 54250, + 54300, + 54350, + 54400, + 54450, + 54500, + 54550, + 54600, + 54650, + 54700, + 54750, + 54800, + 54850, + 54900, + 54950, + 55000, + 55050, + 55100, + 55150, + 55200, + 55250, + 55300, + 55350, + 55400, + 55450, + 55500, + 55550, + 55600, + 55650, + 55700, + 55750, + 55800, + 55850, + 55900, + 55950, + 56000, + 56050, + 56100, + 56150, + 56200, + 56250, + 56300, + 56350, + 56400, + 56450, + 56500, + 56550, + 56600, + 56650, + 56700, + 56750, + 56800, + 56850, + 56900, + 56950, + 57000, + 57050, + 57100, + 57150, + 57200, + 57250, + 57300, + 57350, + 57400, + 57450, + 57500, + 57550, + 57600, + 57650, + 57700, + 57750, + 57800, + 57850, + 57900, + 57950, + 58000, + 58050, + 58100, + 58150, + 58200, + 58250, + 58300, + 58350, + 58400, + 58450, + 58500, + 58550, + 58600, + 58650, + 58700, + 58750, + 58800, + 58850, + 58900, + 58950, + 59000, + 59050, + 59100, + 59150, + 59200, + 59250, + 59300, + 59350, + 59400, + 59450, + 59500, + 59550, + 59600, + 59650, + 59700, + 59750, + 59800, + 59850, + 59900, + 59950, + 60000, + 60050, + 60100, + 60150, + 60200, + 60250, + 60300, + 60350, + 60400, + 60450, + 60500, + 60550, + 60600, + 60650, + 60700, + 60750, + 60800, + 60850, + 60900, + 60950, + 61000, + 61050, + 61100, + 61150, + 61200, + 61250, + 61300, + 61350, + 61400, + 61450, + 61500, + 61550, + 61600, + 61650, + 61700, + 61750, + 61800, + 61850, + 61900, + 61950, + 62000, + 62050, + 62100, + 62150, + 62200, + 62250, + 62300, + 62350, + 62400, + 62450, + 62500, + 62550, + 62600, + 62650, + 62700, + 62750, + 62800, + 62850, + 62900, + 62950, + 63000, + 63050, + 63100, + 63150, + 63200, + 63250, + 63300, + 63350, + 63400, + 63450, + 63500, + 63550, + 63600, + 63650, + 63700, + 63750, + 63800, + 63850, + 63900, + 63950, + 64000, + 64050, + 64100, + 64150, + 64200, + 64250, + 64300, + 64350, + 64400, + 64450, + 64500, + 64550, + 64600, + 64650, + 64700, + 64750, + 64800, + 64850, + 64900, + 64950, + 65000, + 65050, + 65100, + 65150, + 65200, + 65250, + 65300, + 65350, + 65400, + 65450, + 65500, + 65550, + 65600, + 65650, + 65700, + 65750, + 65800, + 65850, + 65900, + 65950, + 66000, + 66050, + 66100, + 66150, + 66200, + 66250, + 66300, + 66350, + 66400, + 66450, + 66500, + 66550, + 66600, + 66650, + 66700, + 66750, + 66800, + 66850, + 66900, + 66950, + 67000, + 67050, + 67100, + 67150, + 67200, + 67250, + 67300, + 67350, + 67400, + 67450, + 67500, + 67550, + 67600, + 67650, + 67700, + 67750, + 67800, + 67850, + 67900, + 67950, + 68000, + 68050, + 68100, + 68150, + 68200, + 68250, + 68300, + 68350, + 68400, + 68450, + 68500, + 68550, + 68600, + 68650, + 68700, + 68750, + 68800, + 68850, + 68900, + 68950, + 69000, + 69050, + 69100, + 69150, + 69200, + 69250, + 69300, + 69350, + 69400, + 69450, + 69500, + 69550, + 69600, + 69650, + 69700, + 69750, + 69800, + 69850, + 69900, + 69950, + 70000, + 70050, + 70100, + 70150, + 70200, + 70250, + 70300, + 70350, + 70400, + 70450, + 70500, + 70550, + 70600, + 70650, + 70700, + 70750, + 70800, + 70850, + 70900, + 70950, + 71000, + 71050, + 71100, + 71150, + 71200, + 71250, + 71300, + 71350, + 71400, + 71450, + 71500, + 71550, + 71600, + 71650, + 71700, + 71750, + 71800, + 71850, + 71900, + 71950, + 72000, + 72050, + 72100, + 72150, + 72200, + 72250, + 72300, + 72350, + 72400, + 72450, + 72500, + 72550, + 72600, + 72650, + 72700, + 72750, + 72800, + 72850, + 72900, + 72950, + 73000, + 73050, + 73100, + 73150, + 73200, + 73250, + 73300, + 73350, + 73400, + 73450, + 73500, + 73550, + 73600, + 73650, + 73700, + 73750, + 73800, + 73850, + 73900, + 73950, + 74000, + 74050, + 74100, + 74150, + 74200, + 74250, + 74300, + 74350, + 74400, + 74450, + 74500, + 74550, + 74600, + 74650, + 74700, + 74750, + 74800, + 74850, + 74900, + 74950, + 75000, + 75050, + 75100, + 75150, + 75200, + 75250, + 75300, + 75350, + 75400, + 75450, + 75500, + 75550, + 75600, + 75650, + 75700, + 75750, + 75800, + 75850, + 75900, + 75950, + 76000, + 76050, + 76100, + 76150, + 76200, + 76250, + 76300, + 76350, + 76400, + 76450, + 76500, + 76550, + 76600, + 76650, + 76700, + 76750, + 76800, + 76850, + 76900, + 76950, + 77000, + 77050, + 77100, + 77150, + 77200, + 77250, + 77300, + 77350, + 77400, + 77450, + 77500, + 77550, + 77600, + 77650, + 77700, + 77750, + 77800, + 77850, + 77900, + 77950, + 78000, + 78050, + 78100, + 78150, + 78200, + 78250, + 78300, + 78350, + 78400, + 78450, + 78500, + 78550, + 78600, + 78650, + 78700, + 78750, + 78800, + 78850, + 78900, + 78950, + 79000, + 79050, + 79100, + 79150, + 79200, + 79250, + 79300, + 79350, + 79400, + 79450, + 79500, + 79550, + 79600, + 79650, + 79700, + 79750, + 79800, + 79850, + 79900, + 79950, + 80000, + 80050, + 80100, + 80150, + 80200, + 80250, + 80300, + 80350, + 80400, + 80450, + 80500, + 80550, + 80600, + 80650, + 80700, + 80750, + 80800, + 80850, + 80900, + 80950, + 81000, + 81050, + 81100, + 81150, + 81200, + 81250, + 81300, + 81350, + 81400, + 81450, + 81500, + 81550, + 81600, + 81650, + 81700, + 81750, + 81800, + 81850, + 81900, + 81950, + 82000, + 82050, + 82100, + 82150, + 82200, + 82250, + 82300, + 82350, + 82400, + 82450, + 82500, + 82550, + 82600, + 82650, + 82700, + 82750, + 82800, + 82850, + 82900, + 82950, + 83000, + 83050, + 83100, + 83150, + 83200, + 83250, + 83300, + 83350, + 83400, + 83450, + 83500, + 83550, + 83600, + 83650, + 83700, + 83750, + 83800, + 83850, + 83900, + 83950, + 84000, + 84050, + 84100, + 84150, + 84200, + 84250, + 84300, + 84350, + 84400, + 84450, + 84500, + 84550, + 84600, + 84650, + 84700, + 84750, + 84800, + 84850, + 84900, + 84950, + 85000, + 85050, + 85100, + 85150, + 85200, + 85250, + 85300, + 85350, + 85400, + 85450, + 85500, + 85550, + 85600, + 85650, + 85700, + 85750, + 85800, + 85850, + 85900, + 85950, + 86000, + 86050, + 86100, + 86150, + 86200, + 86250, + 86300, + 86350, + 86400, + 86450, + 86500, + 86550, + 86600, + 86650, + 86700, + 86750, + 86800, + 86850, + 86900, + 86950, + 87000, + 87050, + 87100, + 87150, + 87200, + 87250, + 87300, + 87350, + 87400, + 87450, + 87500, + 87550, + 87600, + 87650, + 87700, + 87750, + 87800, + 87850, + 87900, + 87950, + 88000, + 88050, + 88100, + 88150, + 88200, + 88250, + 88300, + 88350, + 88400, + 88450, + 88500, + 88550, + 88600, + 88650, + 88700, + 88750, + 88800, + 88850, + 88900, + 88950, + 89000, + 89050, + 89100, + 89150, + 89200, + 89250, + 89300, + 89350, + 89400, + 89450, + 89500, + 89550, + 89600, + 89650, + 89700, + 89750, + 89800, + 89850, + 89900, + 89950, + 90000, + 90050, + 90100, + 90150, + 90200, + 90250, + 90300, + 90350, + 90400, + 90450, + 90500, + 90550, + 90600, + 90650, + 90700, + 90750, + 90800, + 90850, + 90900, + 90950, + 91000, + 91050, + 91100, + 91150, + 91200, + 91250, + 91300, + 91350, + 91400, + 91450, + 91500, + 91550, + 91600, + 91650, + 91700, + 91750, + 91800, + 91850, + 91900, + 91950, + 92000, + 92050, + 92100, + 92150, + 92200, + 92250, + 92300, + 92350, + 92400, + 92450, + 92500, + 92550, + 92600, + 92650, + 92700, + 92750, + 92800, + 92850, + 92900, + 92950, + 93000, + 93050, + 93100, + 93150, + 93200, + 93250, + 93300, + 93350, + 93400, + 93450, + 93500, + 93550, + 93600, + 93650, + 93700, + 93750, + 93800, + 93850, + 93900, + 93950, + 94000, + 94050, + 94100, + 94150, + 94200, + 94250, + 94300, + 94350, + 94400, + 94450, + 94500, + 94550, + 94600, + 94650, + 94700, + 94750, + 94800, + 94850, + 94900, + 94950, + 95000, + 95050, + 95100, + 95150, + 95200, + 95250, + 95300, + 95350, + 95400, + 95450, + 95500, + 95550, + 95600, + 95650, + 95700, + 95750, + 95800, + 95850, + 95900, + 95950, + 96000, + 96050, + 96100, + 96150, + 96200, + 96250, + 96300, + 96350, + 96400, + 96450, + 96500, + 96550, + 96600, + 96650, + 96700, + 96750, + 96800, + 96850, + 96900, + 96950, + 97000, + 97050, + 97100, + 97150, + 97200, + 97250, + 97300, + 97350, + 97400, + 97450, + 97500, + 97550, + 97600, + 97650, + 97700, + 97750, + 97800, + 97850, + 97900, + 97950, + 98000, + 98050, + 98100, + 98150, + 98200, + 98250, + 98300, + 98350, + 98400, + 98450, + 98500, + 98550, + 98600, + 98650, + 98700, + 98750, + 98800, + 98850, + 98900, + 98950, + 99000, + 99050, + 99100, + 99150, + 99200, + 99250, + 99300, + 99350, + 99400, + 99450, + 99500, + 99550, + 99600, + 99650, + 99700, + 99750, + 99800, + 99850, + 99900, + 99950, + 100000, + 100050, + 100100, + 100150, + 100200, + 100250, + 100300, + 100350, + 100400, + 100450, + 100500, + 100550, + 100600, + 100650, + 100700, + 100750, + 100800, + 100850, + 100900, + 100950, + 101000, + 101050, + 101100, + 101150, + 101200, + 101250, + 101300, + 101350, + 101400, + 101450, + 101500, + 101550, + 101600, + 101650, + 101700, + 101750, + 101800, + 101850, + 101900, + 101950, + 102000, + 102050, + 102100, + 102150, + 102200, + 102250, + 102300, + 102350, + 102400, + 102450, + 102500, + 102550, + 102600, + 102650, + 102700, + 102750, + 102800, + 102850, + 102900, + 102950, + 103000, + 103050, + 103100, + 103150, + 103200, + 103250, + 103300, + 103350, + 103400, + 103450, + 103500, + 103550, + 103600, + 103650, + 103700, + 103750, + 103800, + 103850, + 103900, + 103950, + 104000, + 104050, + 104100, + 104150, + 104200, + 104250, + 104300, + 104350, + 104400, + 104450, + 104500, + 104550, + 104600, + 104650, + 104700, + 104750, + 104800, + 104850, + 104900, + 104950, + 105000, + 105050, + 105100, + 105150, + 105200, + 105250, + 105300, + 105350, + 105400, + 105450, + 105500, + 105550, + 105600, + 105650, + 105700, + 105750, + 105800, + 105850, + 105900, + 105950, + 106000, + 106050, + 106100, + 106150, + 106200, + 106250, + 106300, + 106350, + 106400, + 106450, + 106500, + 106550, + 106600, + 106650, + 106700, + 106750, + 106800, + 106850, + 106900, + 106950, + 107000, + 107050, + 107100, + 107150, + 107200, + 107250, + 107300, + 107350, + 107400, + 107450, + 107500, + 107550, + 107600, + 107650, + 107700, + 107750, + 107800, + 107850, + 107900, + 107950, + 108000, + 108050, + 108100, + 108150, + 108200, + 108250, + 108300, + 108350, + 108400, + 108450, + 108500, + 108550, + 108600, + 108650, + 108700, + 108750, + 108800, + 108850, + 108900, + 108950, + 109000, + 109050, + 109100, + 109150, + 109200, + 109250, + 109300, + 109350, + 109400, + 109450, + 109500, + 109550, + 109600, + 109650, + 109700, + 109750, + 109800, + 109850, + 109900, + 109950, + 110000, + 110050, + 110100, + 110150, + 110200, + 110250, + 110300, + 110350, + 110400, + 110450, + 110500, + 110550, + 110600, + 110650, + 110700, + 110750, + 110800, + 110850, + 110900, + 110950, + 111000, + 111050, + 111100, + 111150, + 111200, + 111250, + 111300, + 111350, + 111400, + 111450, + 111500, + 111550, + 111600, + 111650, + 111700, + 111750, + 111800, + 111850, + 111900, + 111950, + 112000, + 112050, + 112100, + 112150, + 112200, + 112250, + 112300, + 112350, + 112400, + 112450, + 112500, + 112550, + 112600, + 112650, + 112700, + 112750, + 112800, + 112850, + 112900, + 112950, + 113000, + 113050, + 113100, + 113150, + 113200, + 113250, + 113300, + 113350, + 113400, + 113450, + 113500, + 113550, + 113600, + 113650, + 113700, + 113750, + 113800, + 113850, + 113900, + 113950, + 114000 + ], + "train_lm_loss": [ + 18.903329491615295, + 17.92984127998352, + 10.164313197135925, + 8.104424595832825, + 7.477854907512665, + 7.166016846895218, + 6.804580718278885, + 6.588657647371292, + 6.343842297792435, + 6.1530406177043915, + 6.03537654876709, + 5.883026510477066, + 5.765959143638611, + 5.63262152671814, + 5.503376245498657, + 5.457231432199478, + 5.3358118534088135, + 5.154331684112549, + 5.118495285511017, + 4.987202435731888, + 4.918202519416809, + 4.856857776641846, + 4.764546602964401, + 4.739320993423462, + 4.653553903102875, + 4.594469606876373, + 4.460053712129593, + 4.5042764544487, + 4.3352158814668655, + 4.367337346076965, + 4.301439195871353, + 4.245260179042816, + 4.1417907774448395, + 4.1692244708538055, + 4.094030827283859, + 4.075914442539215, + 4.063970133662224, + 4.015899688005447, + 3.9542290717363358, + 3.9417237490415573, + 3.862226963043213, + 3.8173378109931946, + 3.737475946545601, + 3.7468477189540863, + 3.696513816714287, + 3.6871324628591537, + 3.6475500762462616, + 3.644735813140869, + 3.671927288174629, + 3.647968351840973, + 3.5962390899658203, + 3.550555631518364, + 3.562489241361618, + 3.4957937002182007, + 3.50462144613266, + 3.578023299574852, + 3.4527556598186493, + 3.4544373154640198, + 3.350398063659668, + 3.3830062597990036, + 3.382434234023094, + 3.3270396888256073, + 3.2807713449001312, + 3.3077849447727203, + 3.266011416912079, + 3.260042205452919, + 3.2598999738693237, + 3.2155676782131195, + 3.2469393610954285, + 3.181729778647423, + 3.0961868166923523, + 3.1053847819566727, + 3.136656403541565, + 3.1520114690065384, + 3.1243404299020767, + 3.0733823627233505, + 3.111829712986946, + 3.0611568987369537, + 3.09712815284729, + 3.0544111281633377, + 3.0330445915460587, + 3.0836497992277145, + 3.05970498919487, + 3.061090111732483, + 3.0509516298770905, + 3.0231515914201736, + 2.9918859601020813, + 2.9983473122119904, + 2.9762284457683563, + 2.993172347545624, + 2.937165394425392, + 2.934217244386673, + 2.930036962032318, + 2.933642566204071, + 2.9177145212888718, + 2.9099421948194504, + 2.8811813294887543, + 2.9204109013080597, + 2.8727662563323975, + 2.867373511195183, + 2.8581928461790085, + 2.861521899700165, + 2.8654904961586, + 2.917135074734688, + 2.8218625634908676, + 2.7807117998600006, + 2.8882941603660583, + 2.8036590814590454, + 2.824520021677017, + 2.843007430434227, + 2.7606232315301895, + 2.8337128907442093, + 2.7995981127023697, + 2.753513351082802, + 2.7925665080547333, + 2.7115988731384277, + 2.7925430089235306, + 2.769044652581215, + 2.785498932003975, + 2.7525507509708405, + 2.7806119322776794, + 2.7130093425512314, + 2.778935343027115, + 2.741407096385956, + 2.7408120930194855, + 2.7623691111803055, + 2.758003354072571, + 2.7073383778333664, + 2.715796485543251, + 2.706937789916992, + 2.674252450466156, + 2.71977698802948, + 2.64662903547287, + 2.692355915904045, + 2.66234815120697, + 2.6434755474328995, + 2.6840816736221313, + 2.762632355093956, + 2.681123897433281, + 2.6680554002523422, + 2.656150981783867, + 2.6629453748464584, + 2.671376511454582, + 2.59904982149601, + 2.6131467670202255, + 2.588576927781105, + 2.6047462224960327, + 2.6906111240386963, + 2.622104585170746, + 2.6579268723726273, + 2.62265782058239, + 2.68343885242939, + 2.6053221821784973, + 2.6344454139471054, + 2.580280303955078, + 2.6195324063301086, + 2.5894234031438828, + 2.611226662993431, + 2.586986631155014, + 2.6138171702623367, + 2.560528114438057, + 2.535577654838562, + 2.574927195906639, + 2.5550353676080704, + 2.5868397653102875, + 2.5665783435106277, + 2.571004033088684, + 2.5946803241968155, + 2.5821424275636673, + 2.627265453338623, + 2.5654630064964294, + 2.5231086760759354, + 2.5666940212249756, + 2.569207727909088, + 2.54684017598629, + 2.5254932790994644, + 2.5157217234373093, + 2.56374292075634, + 2.548900321125984, + 2.569329857826233, + 2.5135190337896347, + 2.5780259370803833, + 2.5867731124162674, + 2.59375262260437, + 2.558076709508896, + 2.5007165521383286, + 2.498443692922592, + 2.5185642391443253, + 2.5822994709014893, + 2.4702048897743225, + 2.5288160741329193, + 2.520866259932518, + 2.5073431879281998, + 2.5073652267456055, + 2.545646086335182, + 2.50737564265728, + 2.5280023962259293, + 2.5668715238571167, + 2.528161585330963, + 2.5018108636140823, + 2.5364094972610474, + 2.5836428701877594, + 2.5294324308633804, + 2.4865193963050842, + 2.521015852689743, + 2.4429538398981094, + 2.4765315502882004, + 2.478725090622902, + 2.5066255927085876, + 2.4834312051534653, + 2.5159030705690384, + 2.4023068845272064, + 2.476969823241234, + 2.440930023789406, + 2.4585606902837753, + 2.495240181684494, + 2.4292139261960983, + 2.483469605445862, + 2.454894319176674, + 2.5337600111961365, + 2.4338560700416565, + 2.5047632604837418, + 2.444226711988449, + 2.464276298880577, + 2.445610925555229, + 2.4867041260004044, + 2.465359106659889, + 2.4517214000225067, + 2.463423863053322, + 2.44936665892601, + 2.436376914381981, + 2.4403330981731415, + 2.404223755002022, + 2.382946029305458, + 2.3788499385118484, + 2.413967415690422, + 2.44422148168087, + 2.437913179397583, + 2.4304381608963013, + 2.49213208258152, + 2.426941081881523, + 2.445514068007469, + 2.462341621518135, + 2.477852776646614, + 2.3807768523693085, + 2.405507430434227, + 2.458716496825218, + 2.4472174495458603, + 2.477419897913933, + 2.389219343662262, + 2.367404729127884, + 2.4385797530412674, + 2.413085624575615, + 2.4523219913244247, + 2.420377030968666, + 2.3971937149763107, + 2.363093540072441, + 2.382193237543106, + 2.4053917825222015, + 2.381447896361351, + 2.443738877773285, + 2.3652568385004997, + 2.419484868645668, + 2.417808562517166, + 2.366078183054924, + 2.35849666595459, + 2.420292004942894, + 2.397281527519226, + 2.408461943268776, + 2.4137678146362305, + 2.352091073989868, + 2.425624832510948, + 2.4354600608348846, + 2.403879329562187, + 2.4180456399917603, + 2.427777409553528, + 2.367577761411667, + 2.326912045478821, + 2.440141513943672, + 2.3276963233947754, + 2.3815340846776962, + 2.412144035100937, + 2.4011663645505905, + 2.397534102201462, + 2.2951226383447647, + 2.419630080461502, + 2.399062007665634, + 2.3924371004104614, + 2.4029073119163513, + 2.348469004034996, + 2.3680042177438736, + 2.313986226916313, + 2.403042048215866, + 2.3961091488599777, + 2.362778142094612, + 2.395942971110344, + 2.356927052140236, + 2.396364226937294, + 2.401548385620117, + 2.3737337440252304, + 2.4006219655275345, + 2.394235208630562, + 2.3756499737501144, + 2.3695772290229797, + 2.3487235605716705, + 2.3770404309034348, + 2.343890428543091, + 2.3433940708637238, + 2.331489749252796, + 2.293241024017334, + 2.39328832924366, + 2.332671597599983, + 2.3727834224700928, + 2.350191444158554, + 2.3654863387346268, + 2.320497900247574, + 2.3611690998077393, + 2.340107709169388, + 2.3577025681734085, + 2.334356799721718, + 2.3447548151016235, + 2.2891667038202286, + 2.3664010763168335, + 2.358737573027611, + 2.3565661758184433, + 2.352216213941574, + 2.3369773477315903, + 2.3925550431013107, + 2.3068550676107407, + 2.3956482261419296, + 2.3437786996364594, + 2.349457249045372, + 2.3070796877145767, + 2.334178015589714, + 2.2819222509860992, + 2.2938428223133087, + 2.3598936647176743, + 2.301691636443138, + 2.3393388241529465, + 2.314027965068817, + 2.3141153752803802, + 2.3115369379520416, + 2.3201946914196014, + 2.354948043823242, + 2.302939251065254, + 2.3038395643234253, + 2.3539970368146896, + 2.341263234615326, + 2.3251078724861145, + 2.314503848552704, + 2.3551515638828278, + 2.3610205203294754, + 2.3401401191949844, + 2.283906430006027, + 2.275030270218849, + 2.3227416574954987, + 2.370670661330223, + 2.3187233954668045, + 2.3102514818310738, + 2.3124470859766006, + 2.3197172582149506, + 2.2774096578359604, + 2.3112007081508636, + 2.2939581722021103, + 2.338823214173317, + 2.2810955718159676, + 2.28573527187109, + 2.32688806951046, + 2.3099788576364517, + 2.307595908641815, + 2.3257239013910294, + 2.3347322046756744, + 2.2502643167972565, + 2.2802968621253967, + 2.2775493264198303, + 2.337101072072983, + 2.313807964324951, + 2.2942542135715485, + 2.2853336334228516, + 2.2445603162050247, + 2.2688126116991043, + 2.2615013271570206, + 2.236464098095894, + 2.2930638417601585, + 2.278277099132538, + 2.336848109960556, + 2.2730683982372284, + 2.278511941432953, + 2.2956570982933044, + 2.3018685579299927, + 2.280346468091011, + 2.307181343436241, + 2.3087603002786636, + 2.2981488555669785, + 2.250538170337677, + 2.316019669175148, + 2.3652205243706703, + 2.3298290222883224, + 2.3198546916246414, + 2.299369364976883, + 2.2968240529298782, + 2.242356352508068, + 2.2576143741607666, + 2.302918091416359, + 2.2511720657348633, + 2.2736955136060715, + 2.296729937195778, + 2.266129791736603, + 2.227809302508831, + 2.3193642497062683, + 2.294358268380165, + 2.276749387383461, + 2.2462079972028732, + 2.228006437420845, + 2.277125261723995, + 2.2412228137254715, + 2.2871614545583725, + 2.264912784099579, + 2.2682378739118576, + 2.272270016372204, + 2.254054881632328, + 2.280806601047516, + 2.2175592854619026, + 2.2730967104434967, + 2.268938720226288, + 2.2500878870487213, + 2.2681543976068497, + 2.303349032998085, + 2.2494408562779427, + 2.3096469044685364, + 2.2683443874120712, + 2.283289149403572, + 2.2810079902410507, + 2.314850613474846, + 2.254880338907242, + 2.252894252538681, + 2.209537371993065, + 2.223983198404312, + 2.257752664387226, + 2.2003230154514313, + 2.2357040345668793, + 2.193962000310421, + 2.2435881793498993, + 2.2412393391132355, + 2.2001858428120613, + 2.2703884840011597, + 2.2950475588440895, + 2.292604148387909, + 2.2676920741796494, + 2.2631421089172363, + 2.2388795614242554, + 2.269433543086052, + 2.236208900809288, + 2.246996946632862, + 2.2523815482854843, + 2.2516969591379166, + 2.2929052263498306, + 2.2656468749046326, + 2.2620000392198563, + 2.2867502719163895, + 2.2627558037638664, + 2.2533846646547318, + 2.2338691353797913, + 2.2431216835975647, + 2.2347236573696136, + 2.267085909843445, + 2.2059026807546616, + 2.2257700338959694, + 2.2166503220796585, + 2.2074871212244034, + 2.216597706079483, + 2.257841318845749, + 2.1611669957637787, + 2.181058406829834, + 2.234190985560417, + 2.1827693805098534, + 2.20173442363739, + 2.2288148775696754, + 2.2690675482153893, + 2.2677790820598602, + 2.2590258568525314, + 2.265716999769211, + 2.2571099400520325, + 2.262490540742874, + 2.2365473359823227, + 2.2799617648124695, + 2.232344016432762, + 2.2474248111248016, + 2.270856410264969, + 2.21540667116642, + 2.2178446874022484, + 2.249391734600067, + 2.2124248445034027, + 2.229475721716881, + 2.252672553062439, + 2.2126733362674713, + 2.157659463584423, + 2.239472582936287, + 2.2784699350595474, + 2.214879758656025, + 2.2448720037937164, + 2.1729690954089165, + 2.228328727185726, + 2.22621800750494, + 2.189029924571514, + 2.233217880129814, + 2.145580753684044, + 2.198506698012352, + 2.205023154616356, + 2.231527917087078, + 2.2063027769327164, + 2.253859430551529, + 2.2434563636779785, + 2.179251067340374, + 2.1931580677628517, + 2.232054851949215, + 2.2475050687789917, + 2.224127873778343, + 2.1524645388126373, + 2.2569186836481094, + 2.213422253727913, + 2.213668644428253, + 2.1761105954647064, + 2.28291617333889, + 2.215282827615738, + 2.20790633559227, + 2.2279426380991936, + 2.1735685765743256, + 2.2136205509305, + 2.2289604172110558, + 2.2405152022838593, + 2.2308433204889297, + 2.1770718917250633, + 2.1632156670093536, + 2.19905449450016, + 2.1585259810090065, + 2.1823509708046913, + 2.169155031442642, + 2.189038634300232, + 2.219576358795166, + 2.2276630103588104, + 2.219713732600212, + 2.268963873386383, + 2.1647699773311615, + 2.1958242505788803, + 2.207914501428604, + 2.2724761962890625, + 2.245696932077408, + 2.2468663454055786, + 2.1697726249694824, + 2.238538347184658, + 2.1862332820892334, + 2.2197600081562996, + 2.218873843550682, + 2.220955975353718, + 2.1905034333467484, + 2.2028390392661095, + 2.215561591088772, + 2.165223181247711, + 2.1762961372733116, + 2.1765440702438354, + 2.1956514194607735, + 2.164561949670315, + 2.2205077409744263, + 2.2024796456098557, + 2.1882707476615906, + 2.2587804347276688, + 2.2122179567813873, + 2.198594719171524, + 2.1549447178840637, + 2.237223580479622, + 2.1631229743361473, + 2.178988017141819, + 2.2596794068813324, + 2.21021618694067, + 2.19253758341074, + 2.2257311418652534, + 2.176894061267376, + 2.1493232622742653, + 2.2323873192071915, + 2.2337165027856827, + 2.1717096269130707, + 2.1498996317386627, + 2.2211587578058243, + 2.177851267158985, + 2.1721040159463882, + 2.1937055811285973, + 2.1890237033367157, + 2.1341772601008415, + 2.1707007437944412, + 2.216708868741989, + 2.1998513862490654, + 2.1646661311388016, + 2.1798997819423676, + 2.1877292692661285, + 2.1109786853194237, + 2.241267830133438, + 2.1652140468358994, + 2.1646104976534843, + 2.1971552595496178, + 2.2108093053102493, + 2.164570063352585, + 2.177151210606098, + 2.1810258254408836, + 2.1812111884355545, + 2.2341349571943283, + 2.153850570321083, + 2.2208621203899384, + 2.1772093772888184, + 2.145630031824112, + 2.2013666555285454, + 2.2047812044620514, + 2.2124711871147156, + 2.173493131995201, + 2.207487791776657, + 2.2026387453079224, + 2.2026178166270256, + 2.1642300486564636, + 2.178011506795883, + 2.162189044058323, + 2.2150996327400208, + 2.1654092669487, + 2.1350086703896523, + 2.16745363175869, + 2.2100678980350494, + 2.1574818193912506, + 2.1974567994475365, + 2.1205168440937996, + 2.229145720601082, + 2.1699210554361343, + 2.2190853506326675, + 2.207837738096714, + 2.1236597448587418, + 2.1365120857954025, + 2.17224333435297, + 2.177599713206291, + 2.1722582578659058, + 2.166735403239727, + 2.1688311472535133, + 2.1665151938796043, + 2.2000377476215363, + 2.1241614818573, + 2.190988950431347, + 2.177071414887905, + 2.158554844558239, + 2.1792725697159767, + 2.1795786321163177, + 2.155384838581085, + 2.240721717476845, + 2.1811474412679672, + 2.1380989030003548, + 2.159171409904957, + 2.1597543880343437, + 2.155512683093548, + 2.122350588440895, + 2.1009815856814384, + 2.143064431846142, + 2.1870599389076233, + 2.1541583314538, + 2.1630136594176292, + 2.193787269294262, + 2.1658500730991364, + 2.2347401678562164, + 2.1253522336483, + 2.1999218985438347, + 2.132537193596363, + 2.178387477993965, + 2.2043971866369247, + 2.1849945560097694, + 2.1600457057356834, + 2.1994026377797127, + 2.185591146349907, + 2.199084535241127, + 2.174398221075535, + 2.1512517407536507, + 2.161075681447983, + 2.2051819190382957, + 2.1626291647553444, + 2.125160925090313, + 2.163478933274746, + 2.1828145384788513, + 2.1757472828030586, + 2.1600346714258194, + 2.155288726091385, + 2.1442637592554092, + 2.1290638968348503, + 2.0755981355905533, + 2.139959178864956, + 2.1365479081869125, + 2.161816328763962, + 2.2110435962677, + 2.152420751750469, + 2.174952954053879, + 2.1445241570472717, + 2.1913570314645767, + 2.186900295317173, + 2.1415829434990883, + 2.101001612842083, + 2.112829439342022, + 2.1752781942486763, + 2.1232960671186447, + 2.134253427386284, + 2.101384498178959, + 2.1788853108882904, + 2.1224107071757317, + 2.144082188606262, + 2.106220655143261, + 2.110179528594017, + 2.160745933651924, + 2.1815676987171173, + 2.1339770704507828, + 2.119999162852764, + 2.161237195134163, + 2.147463023662567, + 2.1371944174170494, + 2.1890452057123184, + 2.1391324028372765, + 2.1328689008951187, + 2.1506186053156853, + 2.1686890721321106, + 2.163981258869171, + 2.1295797377824783, + 2.1594068855047226, + 2.1675697416067123, + 2.111510209739208, + 2.1298627257347107, + 2.076423592865467, + 2.102194629609585, + 2.192333698272705, + 2.170065425336361, + 2.164506010711193, + 2.1127690374851227, + 2.1293860971927643, + 2.123198062181473, + 2.1284868642687798, + 2.194234147667885, + 2.1039896309375763, + 2.0893324688076973, + 2.1701092198491096, + 2.151445835828781, + 2.1303906068205833, + 2.1305025294423103, + 2.1243863850831985, + 2.1164629012346268, + 2.1155927926301956, + 2.1050413250923157, + 2.14733450114727, + 2.1330845803022385, + 2.1201723515987396, + 2.115355134010315, + 2.0583136826753616, + 2.1341800168156624, + 2.117356777191162, + 2.1395946219563484, + 2.1004986092448235, + 2.1642012745141983, + 2.1142408177256584, + 2.12865599244833, + 2.0919134318828583, + 2.094578228890896, + 2.159861743450165, + 2.161883383989334, + 2.1423541381955147, + 2.101611539721489, + 2.1816725209355354, + 2.1373454555869102, + 2.1026122495532036, + 2.1098494082689285, + 2.103230930864811, + 2.1375529915094376, + 2.117106281220913, + 2.2072098180651665, + 2.1700106412172318, + 2.1044202223420143, + 2.1456028148531914, + 2.1598452776670456, + 2.1589766144752502, + 2.1380355209112167, + 2.1547100841999054, + 2.1093590259552, + 2.106745921075344, + 2.1295643895864487, + 2.114450477063656, + 2.1235198378562927, + 2.1141681000590324, + 2.150572344660759, + 2.1241578236222267, + 2.1170715913176537, + 2.125470094382763, + 2.1533886194229126, + 2.1081865206360817, + 2.1127563640475273, + 2.0951920822262764, + 2.0712259709835052, + 2.1272462978959084, + 2.1235574185848236, + 2.1052190735936165, + 2.1102356985211372, + 2.0906480997800827, + 2.1139577850699425, + 2.0814698338508606, + 2.1558374911546707, + 2.117364503443241, + 2.1802947595715523, + 2.142713189125061, + 2.113558255136013, + 2.0402691438794136, + 2.130929544568062, + 2.089078940451145, + 2.2043967694044113, + 2.1170386895537376, + 2.13639272749424, + 2.1415015682578087, + 2.0776795223355293, + 2.1342485025525093, + 2.1097425892949104, + 2.117808111011982, + 2.150445282459259, + 2.129983074963093, + 2.109442450106144, + 2.1425175592303276, + 2.119787745177746, + 2.082450196146965, + 2.111189991235733, + 2.1056854128837585, + 2.1215207800269127, + 2.1303368881344795, + 2.0367309898138046, + 2.1687429323792458, + 2.1307670399546623, + 2.108369380235672, + 2.1307860016822815, + 2.073788307607174, + 2.160708025097847, + 2.142209179699421, + 2.1126782223582268, + 2.16403104364872, + 2.087421253323555, + 2.1046360433101654, + 2.1017247810959816, + 2.14090858399868, + 2.163697227835655, + 2.1066840142011642, + 2.12765783816576, + 2.1165453121066093, + 2.1311854273080826, + 2.139414869248867, + 2.155309021472931, + 2.1117357686161995, + 2.130001053214073, + 2.113844022154808, + 2.1869060695171356, + 2.1142632961273193, + 2.1235010623931885, + 2.149813875555992, + 2.127158507704735, + 2.1460615172982216, + 2.1065470054745674, + 2.1485074907541275, + 2.1642863750457764, + 2.1088403090834618, + 2.0614844411611557, + 2.096938505768776, + 2.0993761271238327, + 2.1268388777971268, + 2.1201959773898125, + 2.039766699075699, + 2.1795507967472076, + 2.1327605322003365, + 2.1265581473708153, + 2.1701936796307564, + 2.1656615883111954, + 2.0997896268963814, + 2.097021706402302, + 2.1082813516259193, + 2.1179957166314125, + 2.1102853789925575, + 2.1123270764946938, + 2.0752890035510063, + 2.1712440103292465, + 2.1059444546699524, + 2.082631528377533, + 2.155254051089287, + 2.132354252040386, + 2.0806171968579292, + 2.089174285531044, + 2.1124488338828087, + 2.134492836892605, + 2.1800086200237274, + 2.0582835748791695, + 2.1230070367455482, + 2.120060108602047, + 2.155825227499008, + 2.0796189829707146, + 2.106152907013893, + 2.073783218860626, + 2.124751552939415, + 2.114295646548271, + 2.1129709482192993, + 2.092460535466671, + 2.089748166501522, + 2.0904882848262787, + 2.088655136525631, + 2.067375175654888, + 2.0756298676133156, + 2.1036627516150475, + 2.114393875002861, + 2.111477240920067, + 2.1680192351341248, + 2.1017357259988785, + 2.0622358471155167, + 2.0821987688541412, + 2.0488307774066925, + 2.06732839345932, + 2.1078705862164497, + 2.080307863652706, + 2.057394355535507, + 2.117069624364376, + 2.107779800891876, + 2.041841745376587, + 2.158811643719673, + 2.0825853422284126, + 2.1538227647542953, + 2.145830862224102, + 2.1114887967705727, + 2.084949806332588, + 2.104806549847126, + 2.1323858201503754, + 2.139321431517601, + 2.0892001390457153, + 2.080063223838806, + 2.072810709476471, + 2.0968208387494087, + 2.1224145740270615, + 2.07036916911602, + 2.1199173033237457, + 2.095970056951046, + 2.0579488649964333, + 2.1239101737737656, + 2.104670539498329, + 2.0772128850221634, + 2.1079971939325333, + 2.086605206131935, + 2.0648358315229416, + 2.0531172156333923, + 2.0467510521411896, + 2.1234483644366264, + 2.1175953298807144, + 2.1099019944667816, + 2.0861279293894768, + 2.0996555238962173, + 2.081056594848633, + 2.1226171478629112, + 2.101033128798008, + 2.0928540229797363, + 2.158732630312443, + 2.127871535718441, + 2.0735312774777412, + 2.094828501343727, + 2.0676101744174957, + 2.134289972484112, + 2.092088997364044, + 2.102558746933937, + 2.1168838813900948, + 2.098923936486244, + 2.118324786424637, + 2.1180190816521645, + 2.1135298758745193, + 2.0586284026503563, + 2.1071862876415253, + 2.1288518980145454, + 2.0651752576231956, + 2.090782232582569, + 2.068972058594227, + 2.0997697189450264, + 2.125876158475876, + 2.09501513838768, + 2.069745220243931, + 2.094430059194565, + 2.056264042854309, + 2.006237842142582, + 2.0894191190600395, + 2.1284607723355293, + 2.0320154279470444, + 2.0951108038425446, + 2.077903464436531, + 2.1040961742401123, + 2.0871993377804756, + 2.0956984907388687, + 2.1017756536602974, + 2.060873366892338, + 2.042510025203228, + 2.0851129814982414, + 2.1336601823568344, + 2.0999446138739586, + 2.1083776131272316, + 2.0746087953448296, + 2.107918918132782, + 2.146236315369606, + 2.086410917341709, + 2.1746934205293655, + 2.1266499534249306, + 2.100032962858677, + 2.0394300371408463, + 2.0712085217237473, + 2.0760195031762123, + 2.0573936626315117, + 2.0751214548945427, + 2.051611326634884, + 2.0655000284314156, + 2.0939380452036858, + 2.08581306040287, + 2.114070549607277, + 2.0514139756560326, + 2.07100610435009, + 2.0959163904190063, + 2.114249676465988, + 2.0740491524338722, + 2.067357785999775, + 2.095892608165741, + 2.0467053204774857, + 2.1220770701766014, + 2.0695320144295692, + 2.094291113317013, + 2.1279467791318893, + 2.0780918076634407, + 2.080880895256996, + 2.0533206835389137, + 2.0690599009394646, + 2.1075809448957443, + 2.0120058953762054, + 2.03410554677248, + 2.0841410234570503, + 2.1072199642658234, + 2.095383048057556, + 2.075208894908428, + 2.0644234716892242, + 2.095093533396721, + 2.084298774600029, + 2.046740561723709, + 2.113687813282013, + 2.06196378916502, + 2.0542866215109825, + 2.0929630622267723, + 2.06613989174366, + 2.0345097109675407, + 2.046288311481476, + 2.1052807569503784, + 2.1026474311947823, + 2.0737393721938133, + 2.0774945095181465, + 2.0461520925164223, + 2.1186364144086838, + 2.0642650574445724, + 2.0406417176127434, + 2.0366513282060623, + 2.061113901436329, + 2.0412654355168343, + 2.0816094651818275, + 2.0801395177841187, + 1.973021112382412, + 2.1163234263658524, + 2.0927261263132095, + 2.065478414297104, + 2.0802832692861557, + 2.095717489719391, + 2.0356909707188606, + 2.061442978680134, + 2.0868303328752518, + 2.0912823900580406, + 2.0386205688118935, + 2.055590495467186, + 2.036224842071533, + 2.0768974497914314, + 2.0479916259646416, + 2.0896254032850266, + 2.106969438493252, + 2.0857290774583817, + 2.075026720762253, + 2.0454948768019676, + 2.103655233979225, + 2.058486208319664, + 2.082231029868126, + 2.0596854984760284, + 2.0744275972247124, + 2.089168205857277, + 2.0618015602231026, + 2.062759004533291, + 2.1119192242622375, + 2.0257438123226166, + 2.131607338786125, + 2.105978339910507, + 2.0712039098143578, + 2.107598215341568, + 2.073985904455185, + 1.9439823925495148, + 2.1214686930179596, + 2.053619183599949, + 2.0343048945069313, + 2.090387798845768, + 2.0901281908154488, + 2.057362049818039, + 2.028391756117344, + 2.044340781867504, + 2.034681186079979, + 2.154541462659836, + 2.044111907482147, + 2.0870151668787003, + 2.0508432760834694, + 2.021634578704834, + 1.9909860789775848, + 2.0883549004793167, + 2.0783542096614838, + 2.094413161277771, + 2.0635452270507812, + 2.009696125984192, + 2.060945324599743, + 2.065837986767292, + 2.045767404139042, + 2.1231731548905373, + 2.087479390203953, + 2.0548707097768784, + 2.0070531219244003, + 2.0952226668596268, + 2.053274042904377, + 2.0579733550548553, + 2.0662709400057793, + 2.0401426032185555, + 2.118391662836075, + 2.083898901939392, + 2.0495484843850136, + 2.0736409053206444, + 2.0465185716748238, + 2.043108068406582, + 2.0914372950792313, + 2.1048558354377747, + 2.086051322519779, + 2.103169411420822, + 2.0821699798107147, + 2.0962017625570297, + 2.0667935609817505, + 2.0556842386722565, + 2.052866041660309, + 2.097395785152912, + 2.099081337451935, + 2.072004847228527, + 2.046247959136963, + 2.0544236227869987, + 2.0655177906155586, + 2.035923056304455, + 2.036447249352932, + 2.0230825394392014, + 2.1244082674384117, + 2.0761948972940445, + 2.0493563786149025, + 2.0060083344578743, + 1.980415165424347, + 2.087464764714241, + 2.0665166676044464, + 2.0220499336719513, + 2.0718472227454185, + 2.0150156170129776, + 2.0033219754695892, + 2.039699725806713, + 2.065089337527752, + 2.0431280359625816, + 2.0452684611082077, + 2.1440927386283875, + 2.0421895757317543, + 2.026904582977295, + 2.0509841442108154, + 2.031520150601864, + 2.053785152733326, + 2.026868887245655, + 2.0266001001000404, + 2.0763982087373734, + 2.0821193009614944, + 2.0517464578151703, + 2.038862131536007, + 1.9990024790167809, + 2.052116461098194, + 2.069176696240902, + 2.0059439167380333, + 2.051148548722267, + 2.1174005568027496, + 2.0578740760684013, + 2.0617022067308426, + 2.015684887766838, + 1.9918065145611763, + 2.0433847904205322, + 2.038047455251217, + 2.035690873861313, + 1.9894048646092415, + 2.0666913390159607, + 2.0927188396453857, + 2.0595704466104507, + 2.0294402465224266, + 2.0924503430724144, + 2.042292058467865, + 2.0762484446167946, + 2.0819596350193024, + 2.028313100337982, + 2.1216435581445694, + 2.0415269136428833, + 2.0225378051400185, + 2.0822734981775284, + 2.04546420276165, + 2.0576097294688225, + 2.0659835040569305, + 2.038278214633465, + 2.014960892498493, + 2.078406870365143, + 2.0646080672740936, + 2.031374618411064, + 2.028431385755539, + 2.0506748110055923, + 2.049563556909561, + 1.9979069605469704, + 2.101785719394684, + 2.0181030184030533, + 2.0758405551314354, + 2.046645201742649, + 2.0639835745096207, + 2.074028044939041, + 2.0259329080581665, + 2.009568430483341, + 2.050681047141552, + 2.041439190506935, + 2.0444538816809654, + 2.0874293223023415, + 2.0261741131544113, + 2.078333355486393, + 2.0452976897358894, + 2.0620628148317337, + 2.0709852054715157, + 2.0197107568383217, + 2.055909439921379, + 2.098265066742897, + 2.083114340901375, + 2.0075256898999214, + 2.043776646256447, + 2.070661909878254, + 2.025929741561413, + 2.0542023107409477, + 2.065564140677452, + 2.030430994927883, + 2.045175977051258, + 2.06366029381752, + 2.0467558428645134, + 2.0359495729207993, + 1.9779925495386124, + 2.058270663022995, + 2.053260251879692, + 2.0753167867660522, + 2.0301400423049927, + 2.066460058093071, + 2.0390331745147705, + 2.0441869720816612, + 2.0333017334342003, + 2.0589302107691765, + 2.061717316508293, + 2.018419571220875, + 2.0592330545186996, + 2.054318457841873, + 2.0327121540904045, + 2.038090243935585, + 1.9943738132715225, + 2.09998270124197, + 2.0050804913043976, + 2.02004237473011, + 2.032327212393284, + 1.9805154502391815, + 2.025603234767914, + 2.1022657230496407, + 2.077101729810238, + 2.0055205300450325, + 2.043917514383793, + 2.0846188068389893, + 1.993859738111496, + 2.0336728990077972, + 2.018345944583416, + 2.093410685658455, + 2.0349952951073647, + 2.000280112028122, + 2.099334791302681, + 2.0366057381033897, + 2.019442543387413, + 2.062258392572403, + 2.0316184237599373, + 2.0475651994347572, + 2.061204232275486, + 2.0284619480371475, + 2.021746553480625, + 2.0331611931324005, + 2.0249457210302353, + 2.018635742366314, + 2.066042974591255, + 2.082159712910652, + 2.042824052274227, + 2.0800784900784492, + 2.0167581662535667, + 2.0486266538500786, + 1.996762491762638, + 2.041399009525776, + 2.0362974032759666, + 2.040357820689678, + 2.0294687151908875, + 2.038611054420471, + 2.0051754638552666, + 2.0573702082037926, + 2.016111023724079, + 2.0441753417253494, + 2.055411145091057, + 2.0245402604341507, + 2.0399163961410522, + 2.0376088991761208, + 2.0255334973335266, + 2.047636367380619, + 2.0299688652157784, + 1.9944815933704376, + 1.9840778037905693, + 1.9920043870806694, + 2.006505750119686, + 2.0150836631655693, + 2.07187607139349, + 2.0489368066191673, + 2.0115428641438484, + 2.0362210497260094, + 2.0835664495825768, + 2.0072920992970467, + 2.040283627808094, + 2.0194199681282043, + 2.0088608786463737, + 2.0220408886671066, + 2.020657353103161, + 2.0466532930731773, + 1.9871886074543, + 2.0450963154435158, + 2.0290394574403763, + 2.0269337818026543, + 2.018285796046257, + 2.014351896941662, + 2.054477922618389, + 2.081388480961323, + 2.0829474478960037, + 2.000673420727253, + 1.989341102540493, + 1.9869554713368416, + 2.0683125853538513, + 2.051385208964348, + 2.004777282476425, + 2.0540613383054733, + 2.0306272506713867, + 2.054213710129261, + 2.0909192115068436, + 1.9950163140892982, + 2.0189238637685776, + 1.9717024639248848, + 2.0215673968195915, + 2.0102554634213448, + 2.0087046027183533, + 2.0078384280204773, + 2.0031341165304184, + 2.0453209280967712, + 2.0194076895713806, + 2.0731295496225357, + 2.0332191213965416, + 2.0771984979510307, + 2.0049709007143974, + 1.983339510858059, + 1.9977687746286392, + 1.9959268867969513, + 2.0336559414863586, + 2.034532368183136, + 2.0372586846351624, + 2.034123055636883, + 2.0328898951411247, + 2.0162342861294746, + 2.04398512840271, + 2.0257643088698387, + 2.0306036472320557, + 2.028611548244953, + 2.0396213680505753, + 2.0329549610614777, + 2.058601036667824, + 2.0165525004267693, + 2.021343804895878, + 2.030618853867054, + 2.021678000688553, + 1.9532994329929352, + 2.010654531419277, + 1.995180383324623, + 1.9909831434488297, + 2.0606606528162956, + 2.0753395184874535, + 2.0661752745509148, + 2.0520168766379356, + 1.9698584452271461, + 1.983627326786518, + 1.9799969792366028, + 1.9967233091592789, + 1.9965864345431328, + 2.0288119837641716, + 2.0555076375603676, + 1.9973373264074326, + 2.0527151450514793, + 2.0230872184038162, + 2.0143861919641495, + 2.033418372273445, + 1.9680943936109543, + 2.029505178332329, + 2.023883618414402, + 2.0157910361886024, + 2.0043200477957726, + 1.9752991572022438, + 2.024109371006489, + 1.9964541718363762, + 2.0231613740324974, + 2.076086886227131, + 1.9728534147143364, + 2.0514397025108337, + 2.032817631959915, + 2.0368765518069267, + 1.9881208837032318, + 2.0268020629882812, + 2.0463944151997566, + 2.0377945974469185, + 1.9644491225481033, + 1.9581931754946709, + 1.965641811490059, + 2.07379350066185, + 1.9951844438910484, + 2.035319358110428, + 2.0216955840587616, + 2.064442925155163, + 1.974889151751995, + 2.041388913989067, + 2.006249211728573, + 2.0404616221785545, + 2.0108172968029976, + 2.0389703288674355, + 2.0193275585770607, + 1.9674829915165901, + 2.008486270904541, + 2.0874203965067863, + 2.0504836663603783, + 2.0212441608309746, + 2.0409564077854156, + 1.9510135129094124, + 2.062198728322983, + 2.024104528129101, + 1.9584163576364517, + 2.009763613343239, + 1.9961022660136223, + 1.9712582156062126, + 2.0379042476415634, + 2.036837212741375, + 2.0692937448620796, + 2.0347658842802048, + 1.9805351048707962, + 2.038200728595257, + 1.9973745793104172, + 2.073351353406906, + 2.074919417500496, + 2.0417734310030937, + 2.0678286030888557, + 2.020480155944824, + 2.044561006128788, + 2.026425912976265, + 2.0266231074929237, + 2.036422647535801, + 2.0130963400006294, + 2.0719946771860123, + 1.9848330616950989, + 2.043527767062187, + 2.0016317144036293, + 2.005630813539028, + 2.0042879432439804, + 2.0316048189997673, + 2.013679839670658, + 2.0566568076610565, + 1.9774751737713814, + 2.021688900887966, + 2.0076692178845406, + 2.0341063663363457, + 2.012077420949936, + 2.036433346569538, + 2.0010046288371086, + 2.055697374045849, + 2.0256293192505836, + 2.0265951603651047, + 2.034500204026699, + 2.023943714797497, + 2.015304684638977, + 2.033477582037449, + 2.086182191967964, + 1.9988139420747757, + 2.0091537460684776, + 2.0034321695566177, + 2.017543204128742, + 2.0128481462597847, + 2.003162629902363, + 2.0155480429530144, + 2.041213095188141, + 1.9569048210978508, + 1.9870126023888588, + 2.0178354009985924, + 2.0380600318312645, + 1.948934867978096, + 2.0492620542645454, + 2.037949301302433, + 1.9849823787808418, + 1.991340309381485, + 1.9829950332641602, + 2.048638328909874, + 1.9717241749167442, + 1.9793516471982002, + 1.9903881400823593, + 1.9942143112421036, + 2.0376163944602013, + 1.9838189631700516, + 2.047453962266445, + 2.01197549700737, + 1.9918284863233566, + 1.998724214732647, + 1.9798468574881554, + 2.0005411207675934, + 2.0205050632357597, + 2.0413459315896034, + 2.0026202648878098, + 1.9912406876683235, + 2.030923068523407, + 2.024840719997883, + 1.9870098903775215, + 1.9923313707113266, + 2.0519390404224396, + 2.0736250057816505, + 2.0273465663194656, + 1.968103714287281, + 2.0015460550785065, + 2.0036661103367805, + 2.0370395109057426, + 2.0211618691682816, + 1.948953777551651, + 1.9964265078306198, + 2.0219241231679916, + 1.9657308012247086, + 1.9990552365779877, + 2.0142288506031036, + 1.9972298443317413, + 1.9732386022806168, + 2.005556493997574, + 2.0266085267066956, + 1.9268651828169823, + 2.0050573721528053, + 1.976886324584484, + 1.993572473526001, + 2.00119611620903, + 2.0345426723361015, + 1.9907111302018166, + 1.9933981150388718, + 1.9761326983571053, + 1.9705879390239716, + 1.9728923216462135, + 1.9994847401976585, + 1.8986326828598976, + 2.006401889026165, + 2.0060112103819847, + 2.0280862376093864, + 2.024144686758518, + 2.0293940156698227, + 2.0279960557818413, + 1.9956505298614502, + 1.9991819933056831, + 1.9962808191776276, + 2.0448773950338364, + 1.981645405292511, + 1.9757319167256355, + 2.0039198845624924, + 1.9489812925457954, + 2.002955824136734, + 2.007725238800049, + 2.003944233059883, + 1.9523543566465378, + 1.9954290241003036, + 1.9891649633646011, + 2.024536870419979, + 2.027180589735508, + 1.9512684121727943, + 2.036916509270668, + 2.0256539061665535, + 2.015167571604252, + 2.032428167760372, + 2.0370295867323875, + 1.979444868862629, + 2.0007887184619904, + 1.9735669791698456, + 1.9563447758555412, + 1.9758893847465515, + 2.0330212265253067, + 1.9894602820277214, + 1.992725446820259, + 2.023265965282917, + 2.0457792207598686, + 1.9979688972234726, + 1.9990206509828568, + 1.984873354434967, + 2.0049352645874023, + 1.973696619272232, + 1.9904580414295197, + 1.9953944236040115, + 1.9911752566695213, + 2.0298176407814026, + 2.0263891518115997, + 2.022261656820774, + 2.0638676062226295, + 2.0133154913783073, + 1.9484881684184074, + 1.9680012539029121, + 1.9467031210660934, + 2.039751686155796, + 2.0094179064035416, + 1.95301865786314, + 1.9679397642612457, + 1.9636483564972878, + 1.9707100987434387, + 2.02432419359684, + 2.0303526297211647, + 2.005777418613434, + 2.0000901594758034, + 1.9618289172649384, + 2.0007731318473816, + 2.031101204454899, + 1.9639815166592598, + 1.994271196424961, + 2.003559358417988, + 1.9770219773054123, + 1.9872236400842667, + 2.012238673865795, + 1.9968409314751625, + 1.9998448491096497, + 1.9774624556303024, + 2.0154098868370056, + 2.003876380622387, + 2.021783709526062, + 2.003754325211048, + 1.9549158066511154, + 1.9960304573178291, + 1.9874075129628181, + 1.982538640499115, + 1.990470603108406, + 1.9728527814149857, + 2.0487864911556244, + 2.02922485768795, + 2.015368454158306, + 1.9615913331508636, + 1.991979405283928, + 1.9482854530215263, + 1.9672145023941994, + 2.03170345723629, + 1.984251357614994, + 1.9683262333273888, + 1.978220447897911, + 1.9764154702425003, + 1.993741236627102, + 2.009088933467865, + 1.9958978667855263, + 1.9576800614595413, + 1.9465628564357758, + 2.029682047665119, + 2.0702997967600822, + 2.051579639315605, + 1.9916836023330688, + 2.0293788462877274, + 1.9564949348568916, + 2.0251080617308617, + 1.936678521335125, + 1.9872208535671234, + 1.957603357732296, + 1.967326581478119, + 1.9598493725061417, + 1.9948946833610535, + 1.9967625960707664, + 2.0043041333556175, + 1.9574492052197456, + 1.9797451049089432, + 1.9778640121221542, + 1.9840318337082863, + 1.9467475935816765, + 2.0238067656755447, + 1.9428724125027657, + 1.9986260458827019, + 1.9569962099194527, + 2.015150584280491, + 1.9796911776065826, + 1.950767457485199, + 2.0166414231061935, + 2.0097014158964157, + 1.979643538594246, + 1.994995005428791, + 2.020479880273342, + 1.9659815579652786, + 1.9736223369836807, + 1.9904606491327286, + 1.9569835886359215, + 2.056053549051285, + 1.985175646841526, + 1.9431265369057655, + 1.9906971231102943, + 1.9903093874454498, + 2.0202900394797325, + 2.0131694301962852, + 2.015625797212124, + 1.9715702012181282, + 1.987017460167408, + 1.997785709798336, + 1.991996556520462, + 2.0179220363497734, + 1.9681666046380997, + 1.9917064979672432, + 1.9613301753997803, + 2.0276446118950844, + 2.001225285232067, + 2.0260725766420364, + 1.970255047082901, + 2.0572341680526733, + 2.0005247592926025, + 1.979899249970913, + 2.0252081751823425, + 1.9811146557331085, + 1.9936858415603638, + 2.0070173367857933, + 2.000417113304138, + 1.9843238219618797, + 2.009116806089878, + 2.011754162609577, + 1.9624217972159386, + 1.9713948518037796, + 2.00575952231884, + 2.0025568082928658, + 1.9416814967989922, + 1.9992287531495094, + 1.9812031090259552, + 2.0190754383802414, + 1.9646539464592934, + 1.956482507288456, + 1.902027577161789, + 1.9882357940077782, + 1.9575849771499634, + 2.023204579949379, + 1.9557916447520256, + 2.018774338066578, + 1.9791523143649101, + 1.9972052872180939, + 1.9881841242313385, + 1.9837678149342537, + 1.961767591536045, + 2.014382690191269, + 2.0177138820290565, + 1.9982723519206047, + 1.9900426045060158, + 1.983957827091217, + 1.9878977686166763, + 1.9984715059399605, + 2.036802761256695, + 1.9794442653656006, + 1.9995977729558945, + 1.9674545004963875, + 1.9862236231565475, + 1.9719116613268852, + 1.9790240228176117, + 1.9823661297559738, + 1.9796909540891647, + 1.9670779705047607, + 2.0369744151830673, + 1.9903996959328651, + 1.990893915295601, + 1.9673370718955994, + 2.0059223249554634, + 1.9891259372234344, + 1.994106762111187, + 1.9646711498498917, + 1.9905275851488113, + 2.0162282809615135, + 2.029538244009018, + 1.998750016093254, + 1.9592677056789398, + 2.0252536088228226, + 1.9900496453046799, + 1.9578977078199387, + 1.9234263524413109, + 1.978588305413723, + 2.0019769445061684, + 2.005474239587784, + 2.0035881474614143, + 1.9983802735805511, + 1.9955928698182106, + 1.9970834031701088, + 1.9034947976469994, + 1.9595503956079483, + 2.0048756301403046, + 1.9618002101778984, + 1.9449108764529228, + 1.9705002829432487, + 2.041468247771263, + 2.0014069005846977, + 2.023065611720085, + 1.9905713871121407, + 1.9397542774677277, + 2.0069906190037727, + 2.01557806879282, + 1.976302094757557, + 1.967396765947342, + 1.9634771347045898, + 1.964187890291214, + 2.0086662396788597, + 2.018429212272167, + 1.9346887543797493, + 1.9636751040816307, + 1.96481703966856, + 1.9898859784007072, + 1.9792080074548721, + 2.010906793177128, + 1.9683189392089844, + 1.9295216724276543, + 1.9609934613108635, + 2.0228690803050995, + 1.9802682399749756, + 2.0107511579990387, + 1.9930797442793846, + 1.9663060531020164, + 1.9452548697590828, + 1.9434687420725822, + 1.9490374699234962, + 1.980733536183834, + 1.976844422519207, + 1.9295583292841911, + 1.9546903148293495, + 1.9829406291246414, + 1.9520632401108742, + 1.9840684235095978, + 1.955265261232853, + 2.020340710878372, + 2.0037192702293396, + 1.9423299729824066, + 1.9702577367424965, + 1.9444866552948952, + 1.9658300131559372, + 1.9548269659280777, + 1.9772934392094612, + 1.9639210104942322, + 1.968404084444046, + 2.0227527543902397, + 1.9456632137298584, + 1.9776961877942085, + 1.9549111127853394, + 1.9825699627399445, + 1.9591833800077438, + 1.9687852263450623, + 1.9846654683351517, + 1.956328697502613, + 1.9657792448997498, + 2.022035025060177, + 1.9525629505515099, + 1.945475995540619, + 1.9783242270350456, + 1.9732216745615005, + 2.0019629076123238, + 1.973229058086872, + 2.0032474398612976, + 1.9999478310346603, + 1.9716782420873642, + 2.0129379853606224, + 1.9254481047391891, + 1.996189571917057, + 1.9045229628682137, + 2.003628581762314, + 1.9823297709226608, + 2.05001550167799, + 1.9673197120428085, + 1.9754755198955536, + 1.9839229732751846, + 1.9973993822932243, + 2.017169304192066, + 1.9599768742918968, + 2.0175496637821198, + 2.0020358189940453, + 1.9334690421819687, + 1.9591052904725075, + 1.951500043272972, + 1.9726936891674995, + 1.9952700659632683, + 1.953581415116787, + 1.948911540210247, + 1.9880067259073257, + 1.9874287545681, + 1.9622489586472511, + 1.9482117593288422, + 2.0667017325758934, + 1.9500126764178276, + 1.9877309873700142, + 1.9725619032979012, + 1.9913336113095284, + 1.971820168197155, + 2.002478413283825, + 1.9907185435295105, + 2.002168409526348, + 1.9689177200198174, + 1.9600966051220894, + 1.92982167750597, + 2.0011792108416557, + 1.9596682041883469, + 2.0478461533784866, + 1.9893052577972412, + 1.969656489789486, + 1.9665499404072762, + 2.0110422894358635, + 1.98421248793602, + 2.0103980600833893, + 2.0189016610383987, + 1.95450921356678, + 2.0029066652059555, + 1.9843851327896118, + 1.9809149205684662, + 2.005268655717373, + 1.9965302273631096, + 1.9705824851989746, + 2.0524618327617645, + 1.977734848856926, + 1.9765473082661629, + 1.958316594362259, + 1.9700078815221786, + 1.9730227813124657, + 1.9455704614520073, + 1.9587862119078636, + 1.9694848135113716, + 1.9445673674345016, + 1.9781478121876717, + 2.052105948328972, + 2.0197309404611588, + 1.9605534598231316, + 1.9754693806171417, + 1.9756909608840942, + 1.9701308757066727, + 2.0271742716431618, + 2.023346893489361, + 1.955320566892624, + 2.00396341830492, + 1.9993778243660927, + 2.0284929797053337, + 1.9712548479437828, + 1.965102195739746, + 1.9473783448338509, + 1.9866988956928253, + 1.9785229340195656, + 2.0360209345817566, + 2.0187632739543915, + 2.008522257208824, + 1.9897810071706772, + 1.94232277572155, + 1.9922220259904861, + 1.9487124308943748, + 1.9948014989495277, + 1.9335628598928452, + 2.0040488094091415, + 1.924577385187149, + 1.984047032892704, + 1.9879765138030052, + 1.9674692675471306, + 2.013790614902973, + 1.9668302312493324, + 1.996106281876564, + 2.018061675131321, + 2.001836620271206, + 2.0218796357512474, + 2.006892628967762, + 2.0037938207387924, + 1.9662308767437935, + 1.9752113372087479, + 1.9817914813756943, + 1.9738308489322662, + 2.0083541721105576, + 1.9911008179187775, + 1.977576732635498, + 1.9234645292162895, + 1.9948003962635994, + 1.9194355905056, + 2.05178065598011, + 1.9455625414848328, + 2.0101326629519463, + 1.9932969361543655, + 2.026337757706642, + 1.9410244822502136, + 2.0123127698898315, + 1.9828352257609367, + 1.9750860705971718, + 1.9307838156819344, + 1.9326386451721191, + 2.0205856040120125, + 1.9951538816094398, + 1.9778669774532318, + 1.9693284556269646, + 1.942105732858181, + 1.9559845849871635, + 2.031798392534256, + 1.9397209957242012, + 2.0114766284823418, + 2.0145903825759888, + 2.031409628689289, + 2.0102570205926895, + 1.9748994633555412, + 1.9773231595754623, + 1.9733703956007957, + 1.954077087342739, + 1.9415976330637932, + 1.8984617590904236, + 1.9607841596007347, + 2.003305606544018, + 1.9547590091824532, + 2.010063759982586, + 2.0084265992045403, + 2.0554540902376175, + 1.9404931887984276, + 1.9929765090346336, + 1.9988348335027695, + 1.9845892190933228, + 1.970971241593361, + 1.917422465980053, + 2.024563029408455, + 1.9139663726091385, + 1.9083149805665016, + 2.003641039133072, + 1.9876406788825989, + 1.9894047379493713, + 2.020521506667137, + 1.940741941332817, + 1.8833247050642967, + 1.9695348963141441, + 2.0037611722946167, + 1.985296331346035, + 2.0092520490288734, + 1.9391512721776962, + 1.9642057493329048, + 1.961976408958435, + 1.9449979588389397, + 1.9672669768333435, + 2.0159821957349777, + 1.963540531694889, + 1.9474770426750183, + 1.9431106969714165, + 1.9637756645679474, + 1.9616743847727776, + 1.955384761095047, + 1.9960447996854782, + 1.9806016385555267, + 1.9477500393986702, + 1.9895000979304314, + 1.960922859609127, + 1.9479193612933159, + 2.017876476049423, + 1.9992086663842201, + 2.011783942580223, + 2.0214675813913345, + 1.947587288916111, + 2.027778685092926, + 2.031375251710415, + 2.00665033608675, + 1.9077134653925896, + 1.977152295410633, + 1.9879475682973862, + 1.9815393760800362, + 1.9220975041389465, + 1.9515141546726227, + 2.0029659792780876, + 1.9514135122299194, + 1.977848306298256, + 1.9494537264108658, + 1.98582124710083, + 1.9859148934483528, + 1.9964262694120407, + 1.9774816557765007, + 1.9603338986635208, + 2.0022129863500595, + 2.012118346989155, + 1.9560701921582222, + 1.9739028811454773, + 1.9211419746279716, + 1.945909895002842, + 1.9479344859719276, + 1.936737671494484, + 1.9935224279761314, + 1.9783772006630898, + 1.9869036749005318, + 2.011808879673481, + 1.996253363788128, + 1.930686816573143, + 2.01967366784811, + 2.006540074944496, + 1.9827731773257256, + 1.9988197907805443, + 1.9683834314346313, + 1.9723219722509384, + 1.966482363641262, + 1.9693551063537598, + 2.0197072625160217, + 1.9641033485531807, + 1.923660583794117, + 2.006109729409218, + 1.9945014491677284, + 1.9725365340709686, + 2.001401573419571, + 1.9551436230540276, + 2.020927242934704, + 1.9589046016335487, + 2.0071070566773415, + 1.9800099655985832, + 2.0243884697556496, + 1.9440942332148552, + 1.996916301548481, + 2.009444333612919, + 1.9957198575139046, + 1.9644110053777695, + 1.9388806745409966, + 2.016555368900299, + 2.0338662937283516, + 1.938504971563816, + 1.9598388001322746, + 1.986661545932293, + 2.0027903616428375, + 1.9505780786275864, + 1.9265750870108604, + 1.9967865943908691, + 1.986883021891117, + 1.970155008137226, + 1.9828371554613113, + 1.9342463836073875, + 2.0036663189530373, + 1.9826207011938095, + 1.9979740679264069, + 1.9895241037011147, + 1.9097932800650597, + 1.9843325689435005, + 1.972087986767292, + 2.007706291973591, + 1.9789492562413216, + 1.9675401151180267, + 1.9833966717123985, + 2.021587237715721, + 2.013091519474983, + 1.9968589171767235, + 2.008657656610012, + 1.9655629247426987, + 1.9516348391771317, + 2.0179964900016785, + 2.005268208682537, + 1.9539107084274292, + 1.9967302158474922, + 1.9568492397665977, + 1.9734862297773361, + 1.9781926646828651, + 1.9959547892212868, + 1.934755727648735, + 1.9478243067860603, + 1.9319412037730217, + 1.9343886822462082, + 1.957096479833126, + 1.9985850304365158, + 1.9493222385644913, + 1.9835767149925232, + 1.9606860503554344, + 1.973264180123806, + 1.909697063267231, + 1.9920822232961655, + 1.9794139489531517, + 1.9243657514452934, + 2.0234353616833687, + 1.927987590432167, + 1.9461112692952156, + 1.9626985937356949, + 1.9795078784227371, + 2.0012419298291206, + 1.9698284193873405, + 1.991179920732975, + 1.9713560566306114, + 2.0154649540781975, + 1.9456562027335167, + 2.0245508700609207, + 2.004884473979473, + 1.9430117756128311, + 1.9530266225337982, + 1.960635781288147, + 1.9638905301690102, + 1.936725914478302, + 1.957448735833168, + 2.032605789601803, + 1.942757785320282, + 1.9967185854911804, + 1.94998599588871, + 1.9293048828840256, + 1.988465316593647, + 1.9418746456503868, + 1.989843674004078, + 1.9676333591341972, + 1.9751940295100212, + 1.9640126004815102, + 1.9811243563890457, + 1.9960966184735298, + 1.9456981346011162, + 2.048684313893318, + 2.029345892369747, + 2.009684279561043, + 1.9996330440044403, + 1.990261934697628, + 1.9563634023070335, + 1.9646337255835533, + 1.9971077516674995, + 2.0252646431326866, + 1.969106025993824, + 2.0258323028683662, + 1.9593088403344154, + 1.984964370727539, + 2.0020723417401314, + 1.948498584330082, + 1.9960700795054436, + 1.9450861737132072, + 1.973616048693657, + 1.8961909785866737, + 1.9728698655962944, + 1.9818176478147507, + 1.9935768321156502, + 1.983077384531498, + 1.960403434932232, + 1.9727842062711716, + 2.016776457428932, + 1.9792862758040428, + 1.981849916279316, + 1.975904420018196, + 1.9750661998987198, + 1.9559002071619034, + 2.008531726896763, + 2.023620679974556, + 2.0092128440737724, + 1.939083307981491, + 1.9724336192011833, + 1.9977569580078125, + 1.9886582866311073, + 1.973270982503891, + 1.9654527753591537, + 2.0069473311305046, + 1.9550659209489822, + 1.9312019050121307, + 1.9664997607469559, + 1.9914808347821236, + 1.9681864902377129, + 1.9273158460855484, + 2.0083284601569176, + 2.0012303441762924, + 1.990386962890625, + 1.9962528347969055, + 1.9876855239272118, + 1.9505937844514847, + 2.010381869971752, + 1.9350207075476646, + 1.9865258187055588, + 2.0243826806545258, + 1.9579422771930695, + 1.9344551041722298, + 1.9476393461227417, + 2.0106156691908836, + 1.9337513595819473, + 1.9508503377437592, + 1.9603656008839607, + 1.9764313697814941, + 1.9999417290091515, + 1.9665289297699928, + 2.004588097333908, + 1.9054122269153595, + 2.0233445912599564, + 1.9976691454648972, + 1.9702251255512238 + ], + "val_steps": [ + 5000, + 10000, + 15000, + 20000, + 25000, + 30000, + 35000, + 40000, + 45000, + 50000, + 55000, + 60000, + 65000, + 70000, + 75000, + 80000, + 85000, + 90000, + 95000, + 100000, + 105000, + 110000, + 114000 + ], + "val_lm_loss": [ + 2.7975464153289793, + 2.4204524326324464, + 2.2778771257400514, + 2.193556618690491, + 2.1345069503784178, + 2.0942814230918883, + 2.06017117023468, + 2.032040586471558, + 2.0095694231987, + 1.9881365370750428, + 1.967874310016632, + 1.9517910623550414, + 1.9354958248138427, + 1.9221543312072753, + 1.9093537545204162, + 1.8973060131072998, + 1.8883024740219116, + 1.8794119453430176, + 1.8752654528617858, + 1.8732275485992431, + 1.8715489673614503, + 1.8698759746551514, + 1.869820020198822 + ], + "val_ppl": [ + 16.40434789642657, + 11.2509484597766, + 9.755947760529361, + 8.967048840876455, + 8.452877791057968, + 8.11960431325781, + 7.8473129217592446, + 7.6296394236869896, + 7.46010450774543, + 7.301914230055081, + 7.155450042246805, + 7.041287678732635, + 6.927478019485029, + 6.835668912879081, + 6.748726055275397, + 6.667906970171607, + 6.608141662522236, + 6.54965217604342, + 6.522550320192605, + 6.509271522147634, + 6.498354346345478, + 6.487491735988663, + 6.487128742071292 + ] +} \ No newline at end of file diff --git a/076_moe_ffn_baseline/moe_ffn_3of5_dff256_best.pt b/076_moe_ffn_baseline/moe_ffn_3of5_dff256_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..edc76b1b834faca61f588e827466bd422fdd8320 --- /dev/null +++ b/076_moe_ffn_baseline/moe_ffn_3of5_dff256_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ad127e2fee3a0b63fa66709882b46384149018f1bb103dbc4f6ca92428bf45b +size 349663992 diff --git a/076_moe_ffn_baseline/moe_ffn_3of5_dff256_results.json b/076_moe_ffn_baseline/moe_ffn_3of5_dff256_results.json new file mode 100644 index 0000000000000000000000000000000000000000..2098c805701dae47b9f4b45022fa43041dd89077 --- /dev/null +++ b/076_moe_ffn_baseline/moe_ffn_3of5_dff256_results.json @@ -0,0 +1,839 @@ +{ + "label": "moe_ffn_3of5_dff256", + "total_params": 87404544, + "active_params": 79015936, + "steps": [ + 30050, + 30100, + 30150, + 30200, + 30250, + 30300, + 30350, + 30400, + 30450, + 30500, + 30550, + 30600, + 30650, + 30700, + 30750, + 30800, + 30850, + 30900, + 30950, + 31000, + 31050, + 31100, + 31150, + 31200, + 31250, + 31300, + 31350, + 31400, + 31450, + 31500, + 31550, + 31600, + 31650, + 31700, + 31750, + 31800, + 31850, + 31900, + 31950, + 32000, + 32050, + 32100, + 32150, + 32200, + 32250, + 32300, + 32350, + 32400, + 32450, + 32500, + 32550, + 32600, + 32650, + 32700, + 32750, + 32800, + 32850, + 32900, + 32950, + 33000, + 33050, + 33100, + 33150, + 33200, + 33250, + 33300, + 33350, + 33400, + 33450, + 33500, + 33550, + 33600, + 33650, + 33700, + 33750, + 33800, + 33850, + 33900, + 33950, + 34000, + 34050, + 34100, + 34150, + 34200, + 34250, + 34300, + 34350, + 34400, + 34450, + 34500, + 34550, + 34600, + 34650, + 34700, + 34750, + 34800, + 34850, + 34900, + 34950, + 35000, + 35050, + 35100, + 35150, + 35200, + 35250, + 35300, + 35350, + 35400, + 35450, + 35500, + 35550, + 35600, + 35650, + 35700, + 35750, + 35800, + 35850, + 35900, + 35950, + 36000, + 36050, + 36100, + 36150, + 36200, + 36250, + 36300, + 36350, + 36400, + 36450, + 36500, + 36550, + 36600, + 36650, + 36700, + 36750, + 36800, + 36850, + 36900, + 36950, + 37000, + 37050, + 37100, + 37150, + 37200, + 37250, + 37300, + 37350, + 37400, + 37450, + 37500, + 37550, + 37600, + 37650, + 37700, + 37750, + 37800, + 37850, + 37900, + 37950, + 38000, + 38050, + 38100, + 38150, + 38200, + 38250, + 38300, + 38350, + 38400, + 38450, + 38500, + 38550, + 38600, + 38650, + 38700, + 38750, + 38800, + 38850, + 38900, + 38950, + 39000, + 39050, + 39100, + 39150, + 39200, + 39250, + 39300, + 39350, + 39400, + 39450, + 39500, + 39550, + 39600, + 39650, + 39700, + 39750, + 39800, + 39850, + 39900, + 39950, + 40000, + 40050, + 40100, + 40150, + 40200, + 40250, + 40300, + 40350, + 40400, + 40450, + 40500, + 40550, + 40600, + 40650, + 40700, + 40750, + 40800, + 40850, + 40900, + 40950, + 41000, + 41050, + 41100, + 41150, + 41200, + 41250, + 41300, + 41350, + 41400, + 41450, + 41500, + 41550, + 41600, + 41650, + 41700, + 41750, + 41800, + 41850, + 41900, + 41950, + 42000, + 42050, + 42100, + 42150, + 42200, + 42250, + 42300, + 42350, + 42400, + 42450, + 42500, + 42550, + 42600, + 42650, + 42700, + 42750, + 42800, + 42850, + 42900, + 42950, + 43000, + 43050, + 43100, + 43150, + 43200, + 43250, + 43300, + 43350, + 43400, + 43450, + 43500, + 43550, + 43600, + 43650, + 43700, + 43750, + 43800, + 43850, + 43900, + 43950, + 44000, + 44050, + 44100, + 44150, + 44200, + 44250, + 44300, + 44350, + 44400, + 44450, + 44500, + 44550, + 44600, + 44650, + 44700, + 44750, + 44800, + 44850, + 44900, + 44950, + 45000, + 45050, + 45100, + 45150, + 45200, + 45250, + 45300, + 45350, + 45400, + 45450, + 45500, + 45550, + 45600, + 45650, + 45700, + 45750, + 45800, + 45850, + 45900, + 45950, + 46000, + 46050, + 46100, + 46150, + 46200, + 46250, + 46300, + 46350, + 46400, + 46450, + 46500, + 46550, + 46600, + 46650, + 46700, + 46750, + 46800, + 46850, + 46900, + 46950, + 47000, + 47050, + 47100, + 47150, + 47200, + 47250, + 47300, + 47350, + 47400, + 47450, + 47500, + 47550, + 47600, + 47650, + 47700, + 47750, + 47800, + 47850, + 47900, + 47950, + 48000, + 48050, + 48100, + 48150, + 48200, + 48250, + 48300, + 48350, + 48400, + 48450, + 48500, + 48550, + 48600, + 48650, + 48700, + 48750, + 48800, + 48850, + 48900, + 48950, + 49000, + 49050, + 49100, + 49150, + 49200, + 49250, + 49300, + 49350, + 49400, + 49450, + 49500, + 49550, + 49600, + 49650, + 49700, + 49750, + 49800, + 49850, + 49900, + 49950, + 50000 + ], + "train_lm_loss": [ + 3.563804864883423, + 3.278953790664673, + 3.5623297691345215, + 3.4435513019561768, + 3.3232429027557373, + 3.376445770263672, + 3.460474967956543, + 3.326740026473999, + 3.399210214614868, + 3.445897102355957, + 3.308715343475342, + 3.4225432872772217, + 3.3138537406921387, + 3.361447334289551, + 3.4871137142181396, + 3.357971668243408, + 3.2375316619873047, + 3.334530830383301, + 3.5028464794158936, + 3.2810473442077637, + 3.4990322589874268, + 3.400071382522583, + 3.342735767364502, + 3.4445061683654785, + 3.3828654289245605, + 3.464329719543457, + 3.3749518394470215, + 3.377721071243286, + 3.4031801223754883, + 3.3678784370422363, + 3.423732280731201, + 3.261235475540161, + 3.444387674331665, + 3.310911178588867, + 3.327232599258423, + 3.338757276535034, + 3.3904919624328613, + 3.394727945327759, + 3.301560401916504, + 3.3940155506134033, + 3.3593709468841553, + 3.4552934169769287, + 3.28324818611145, + 3.4560368061065674, + 3.330399751663208, + 3.3377573490142822, + 3.3304498195648193, + 3.4039270877838135, + 3.284109115600586, + 3.4228010177612305, + 3.3947324752807617, + 3.4360289573669434, + 3.411677360534668, + 3.3706538677215576, + 3.291677951812744, + 3.3225202560424805, + 3.308684825897217, + 3.2982428073883057, + 3.5312726497650146, + 3.2852046489715576, + 3.4486637115478516, + 3.3542795181274414, + 3.3382856845855713, + 3.263730525970459, + 3.336940050125122, + 3.3787221908569336, + 3.3259074687957764, + 3.321169137954712, + 3.346818208694458, + 3.432842969894409, + 3.3987374305725098, + 3.3513565063476562, + 3.269444704055786, + 3.4437289237976074, + 3.368438720703125, + 3.369935989379883, + 3.2923645973205566, + 3.323976993560791, + 3.2908380031585693, + 3.4067652225494385, + 3.3439860343933105, + 3.3173038959503174, + 3.4356274604797363, + 3.409071683883667, + 3.3455493450164795, + 3.26070237159729, + 3.380394697189331, + 3.4453370571136475, + 3.3277487754821777, + 3.3322010040283203, + 3.3518121242523193, + 3.3313708305358887, + 3.423877716064453, + 3.3594136238098145, + 3.3791768550872803, + 3.412203311920166, + 3.2941324710845947, + 3.218994140625, + 3.3484444618225098, + 3.3631579875946045, + 3.314756155014038, + 3.4110107421875, + 3.273794651031494, + 3.4085464477539062, + 3.333404064178467, + 3.292609453201294, + 3.308562755584717, + 3.410182476043701, + 3.3349578380584717, + 3.3910064697265625, + 3.3315494060516357, + 3.330045461654663, + 3.3921728134155273, + 3.407144069671631, + 3.3401424884796143, + 3.2615597248077393, + 3.3752145767211914, + 3.354963541030884, + 3.3568596839904785, + 3.290346384048462, + 3.423976182937622, + 3.2897026538848877, + 3.2321279048919678, + 3.369631290435791, + 3.254794120788574, + 3.33634614944458, + 3.2513961791992188, + 3.2370402812957764, + 3.3215737342834473, + 3.1930110454559326, + 3.0078656673431396, + 3.3025505542755127, + 3.3319010734558105, + 3.1920247077941895, + 3.3551557064056396, + 3.2945005893707275, + 3.21807861328125, + 3.223259449005127, + 3.3539586067199707, + 3.1743240356445312, + 3.227973699569702, + 3.1944806575775146, + 3.3556573390960693, + 3.296186685562134, + 3.345808744430542, + 3.3952560424804688, + 3.265185594558716, + 3.3033244609832764, + 3.287456512451172, + 3.1666293144226074, + 3.352288007736206, + 3.173133134841919, + 3.264657735824585, + 3.191490650177002, + 3.339628219604492, + 3.1896955966949463, + 3.3484535217285156, + 3.2611541748046875, + 3.2327940464019775, + 3.293515920639038, + 3.19201397895813, + 3.2317349910736084, + 3.2758946418762207, + 3.240844249725342, + 3.2551076412200928, + 3.1146223545074463, + 3.185821533203125, + 3.3301453590393066, + 3.164832830429077, + 3.3018596172332764, + 3.314164400100708, + 3.2803235054016113, + 3.3761088848114014, + 3.164076566696167, + 3.2444121837615967, + 3.300957679748535, + 3.2789506912231445, + 3.3124749660491943, + 3.295945882797241, + 3.3125643730163574, + 3.339808225631714, + 3.219149589538574, + 3.1262362003326416, + 3.2410941123962402, + 3.212085723876953, + 3.290534496307373, + 3.419210433959961, + 3.4621052742004395, + 3.3872482776641846, + 3.2450947761535645, + 3.1357240676879883, + 3.17799711227417, + 3.2847416400909424, + 3.2299628257751465, + 3.2263615131378174, + 3.2974936962127686, + 3.377227544784546, + 3.231602907180786, + 3.1733319759368896, + 3.1608126163482666, + 3.3993875980377197, + 3.3992111682891846, + 3.3848352432250977, + 3.370238780975342, + 3.376953363418579, + 3.4569344520568848, + 3.436572313308716, + 3.3462557792663574, + 3.4041082859039307, + 3.402430772781372, + 3.4192593097686768, + 3.2824933528900146, + 3.392512559890747, + 3.4523303508758545, + 3.334235191345215, + 3.3162081241607666, + 3.3854854106903076, + 3.4571454524993896, + 3.374213457107544, + 3.2841849327087402, + 3.2749431133270264, + 3.4356698989868164, + 3.2580740451812744, + 3.48087739944458, + 3.4065074920654297, + 3.289424419403076, + 3.402974843978882, + 3.4865026473999023, + 3.3656866550445557, + 3.3412692546844482, + 3.3183670043945312, + 3.2817490100860596, + 3.366673707962036, + 3.446760892868042, + 3.453284502029419, + 3.490241765975952, + 3.472956895828247, + 3.439767837524414, + 3.4039323329925537, + 3.3113694190979004, + 3.3156254291534424, + 3.2967824935913086, + 3.492008924484253, + 3.367037773132324, + 3.2754697799682617, + 3.4444549083709717, + 3.320282220840454, + 3.432253360748291, + 3.3519420623779297, + 3.342878580093384, + 3.389129638671875, + 3.4697265625, + 3.4052629470825195, + 3.2069036960601807, + 3.374483585357666, + 3.3137757778167725, + 3.2922708988189697, + 3.3051207065582275, + 3.462017297744751, + 3.522639036178589, + 3.314390182495117, + 3.3067116737365723, + 3.300877332687378, + 3.295384645462036, + 3.360240936279297, + 3.3959569931030273, + 3.19234561920166, + 3.3220736980438232, + 3.2341387271881104, + 3.3321948051452637, + 3.328888416290283, + 3.240748405456543, + 3.14154052734375, + 3.310379981994629, + 3.4054882526397705, + 3.367121696472168, + 3.4285407066345215, + 3.185914993286133, + 3.3699944019317627, + 3.3626081943511963, + 3.2664384841918945, + 3.2367708683013916, + 3.143813371658325, + 3.285569906234741, + 3.3083183765411377, + 3.228227138519287, + 3.2812962532043457, + 3.2035131454467773, + 3.334287643432617, + 3.2900993824005127, + 3.3940629959106445, + 3.3148038387298584, + 3.2855677604675293, + 3.3293700218200684, + 3.384108543395996, + 3.191138505935669, + 3.3901753425598145, + 3.3232994079589844, + 3.3526904582977295, + 3.222489356994629, + 3.2362403869628906, + 3.2396228313446045, + 3.208895444869995, + 3.2508628368377686, + 3.340911388397217, + 3.378321886062622, + 3.2798516750335693, + 3.265139579772949, + 3.299147844314575, + 3.330254077911377, + 3.333657741546631, + 3.307037591934204, + 3.3217413425445557, + 3.172126054763794, + 3.2476608753204346, + 3.24674391746521, + 3.3503530025482178, + 3.313633680343628, + 3.254999876022339, + 3.3858096599578857, + 3.241959810256958, + 3.212409734725952, + 3.3270366191864014, + 3.253938674926758, + 3.3379480838775635, + 3.307971477508545, + 3.2831509113311768, + 3.2191169261932373, + 3.2664880752563477, + 3.1779143810272217, + 3.1966681480407715, + 3.332584857940674, + 3.2375266551971436, + 3.2973828315734863, + 3.2981882095336914, + 3.295456647872925, + 3.2749907970428467, + 3.284789562225342, + 3.1910319328308105, + 3.3059797286987305, + 3.3442046642303467, + 3.368237257003784, + 3.2633304595947266, + 3.3078296184539795, + 3.2871522903442383, + 3.323009490966797, + 3.308948278427124, + 3.36444354057312, + 3.3399009704589844, + 3.313957691192627, + 3.310342311859131, + 3.181002616882324, + 3.225287675857544, + 3.2064695358276367, + 3.4027445316314697, + 3.252173900604248, + 3.3642208576202393, + 3.2207465171813965, + 3.2343547344207764, + 3.2980523109436035, + 3.225893974304199, + 3.2349159717559814, + 3.314257860183716, + 3.2192132472991943, + 3.207495927810669, + 3.3802340030670166, + 3.2462940216064453, + 3.321593761444092, + 3.293653726577759, + 3.286876678466797, + 3.2406649589538574, + 3.1920993328094482, + 3.103175640106201, + 3.2689037322998047, + 3.2525572776794434, + 3.191192150115967, + 3.174307107925415, + 3.256286859512329, + 3.2933032512664795, + 3.167811155319214, + 3.0858194828033447, + 3.2874794006347656, + 3.266204833984375, + 3.267522096633911, + 3.2534892559051514, + 3.2359585762023926, + 3.2013349533081055, + 3.1932015419006348, + 3.054973840713501, + 3.114339828491211, + 3.110930919647217, + 3.20746111869812, + 3.2242369651794434, + 3.2217493057250977, + 3.333747386932373, + 3.1327788829803467, + 3.320770502090454, + 3.1562256813049316, + 3.313164472579956, + 3.1076581478118896 + ], + "val_steps": [ + 32500, + 35000, + 37500, + 40000, + 42500, + 45000, + 47500, + 50000 + ], + "val_lm_loss": [ + 3.503723314830235, + 3.4844370569501604, + 3.481419427054269, + 3.4712675980159213, + 3.4497692244393483, + 3.4543867792401994, + 3.4496961321149553, + 3.4574152401515414 + ], + "val_ppl": [ + 33.238981038199206, + 32.60406770938233, + 32.505828998517785, + 32.177504745361894, + 31.493123626801466, + 31.63888111360529, + 31.490821805316955, + 31.734843464106348 + ] +} \ No newline at end of file diff --git a/077_cosmopedia_moe_ffn/moe_ffn_cosmopedia_best.pt b/077_cosmopedia_moe_ffn/moe_ffn_cosmopedia_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..0bdd183d46be7274f7222c2a2a1ffb54f9f76a4e --- /dev/null +++ b/077_cosmopedia_moe_ffn/moe_ffn_cosmopedia_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b5a0f4e3f6049c1b762ad1d759b14fb7ef46bcc9c86fabddd975031bf7d49bb +size 424444533 diff --git a/077_cosmopedia_moe_ffn/moe_ffn_cosmopedia_results.json b/077_cosmopedia_moe_ffn/moe_ffn_cosmopedia_results.json new file mode 100644 index 0000000000000000000000000000000000000000..d842703d728cfb9a1e6309a235a33a599441f662 --- /dev/null +++ b/077_cosmopedia_moe_ffn/moe_ffn_cosmopedia_results.json @@ -0,0 +1,1599 @@ +{ + "label": "moe_ffn_cosmopedia", + "total_params": 106099712, + "active_params": 97711104, + "steps": [ + 75050, + 75100, + 75150, + 75200, + 75250, + 75300, + 75350, + 75400, + 75450, + 75500, + 75550, + 75600, + 75650, + 75700, + 75750, + 75800, + 75850, + 75900, + 75950, + 76000, + 76050, + 76100, + 76150, + 76200, + 76250, + 76300, + 76350, + 76400, + 76450, + 76500, + 76550, + 76600, + 76650, + 76700, + 76750, + 76800, + 76850, + 76900, + 76950, + 77000, + 77050, + 77100, + 77150, + 77200, + 77250, + 77300, + 77350, + 77400, + 77450, + 77500, + 77550, + 77600, + 77650, + 77700, + 77750, + 77800, + 77850, + 77900, + 77950, + 78000, + 78050, + 78100, + 78150, + 78200, + 78250, + 78300, + 78350, + 78400, + 78450, + 78500, + 78550, + 78600, + 78650, + 78700, + 78750, + 78800, + 78850, + 78900, + 78950, + 79000, + 79050, + 79100, + 79150, + 79200, + 79250, + 79300, + 79350, + 79400, + 79450, + 79500, + 79550, + 79600, + 79650, + 79700, + 79750, + 79800, + 79850, + 79900, + 79950, + 80000, + 80050, + 80100, + 80150, + 80200, + 80250, + 80300, + 80350, + 80400, + 80450, + 80500, + 80550, + 80600, + 80650, + 80700, + 80750, + 80800, + 80850, + 80900, + 80950, + 81000, + 81050, + 81100, + 81150, + 81200, + 81250, + 81300, + 81350, + 81400, + 81450, + 81500, + 81550, + 81600, + 81650, + 81700, + 81750, + 81800, + 81850, + 81900, + 81950, + 82000, + 82050, + 82100, + 82150, + 82200, + 82250, + 82300, + 82350, + 82400, + 82450, + 82500, + 82550, + 82600, + 82650, + 82700, + 82750, + 82800, + 82850, + 82900, + 82950, + 83000, + 83050, + 83100, + 83150, + 83200, + 83250, + 83300, + 83350, + 83400, + 83450, + 83500, + 83550, + 83600, + 83650, + 83700, + 83750, + 83800, + 83850, + 83900, + 83950, + 84000, + 84050, + 84100, + 84150, + 84200, + 84250, + 84300, + 84350, + 84400, + 84450, + 84500, + 84550, + 84600, + 84650, + 84700, + 84750, + 84800, + 84850, + 84900, + 84950, + 85000, + 85050, + 85100, + 85150, + 85200, + 85250, + 85300, + 85350, + 85400, + 85450, + 85500, + 85550, + 85600, + 85650, + 85700, + 85750, + 85800, + 85850, + 85900, + 85950, + 86000, + 86050, + 86100, + 86150, + 86200, + 86250, + 86300, + 86350, + 86400, + 86450, + 86500, + 86550, + 86600, + 86650, + 86700, + 86750, + 86800, + 86850, + 86900, + 86950, + 87000, + 87050, + 87100, + 87150, + 87200, + 87250, + 87300, + 87350, + 87400, + 87450, + 87500, + 87550, + 87600, + 87650, + 87700, + 87750, + 87800, + 87850, + 87900, + 87950, + 88000, + 88050, + 88100, + 88150, + 88200, + 88250, + 88300, + 88350, + 88400, + 88450, + 88500, + 88550, + 88600, + 88650, + 88700, + 88750, + 88800, + 88850, + 88900, + 88950, + 89000, + 89050, + 89100, + 89150, + 89200, + 89250, + 89300, + 89350, + 89400, + 89450, + 89500, + 89550, + 89600, + 89650, + 89700, + 89750, + 89800, + 89850, + 89900, + 89950, + 90000, + 90050, + 90100, + 90150, + 90200, + 90250, + 90300, + 90350, + 90400, + 90450, + 90500, + 90550, + 90600, + 90650, + 90700, + 90750, + 90800, + 90850, + 90900, + 90950, + 91000, + 91050, + 91100, + 91150, + 91200, + 91250, + 91300, + 91350, + 91400, + 91450, + 91500, + 91550, + 91600, + 91650, + 91700, + 91750, + 91800, + 91850, + 91900, + 91950, + 92000, + 92050, + 92100, + 92150, + 92200, + 92250, + 92300, + 92350, + 92400, + 92450, + 92500, + 92550, + 92600, + 92650, + 92700, + 92750, + 92800, + 92850, + 92900, + 92950, + 93000, + 93050, + 93100, + 93150, + 93200, + 93250, + 93300, + 93350, + 93400, + 93450, + 93500, + 93550, + 93600, + 93650, + 93700, + 93750, + 93800, + 93850, + 93900, + 93950, + 94000, + 94050, + 94100, + 94150, + 94200, + 94250, + 94300, + 94350, + 94400, + 94450, + 94500, + 94550, + 94600, + 94650, + 94700, + 94750, + 94800, + 94850, + 94900, + 94950, + 95000, + 95050, + 95100, + 95150, + 95200, + 95250, + 95300, + 95350, + 95400, + 95450, + 95500, + 95550, + 95600, + 95650, + 95700, + 95750, + 95800, + 95850, + 95900, + 95950, + 96000, + 96050, + 96100, + 96150, + 96200, + 96250, + 96300, + 96350, + 96400, + 96450, + 96500, + 96550, + 96600, + 96650, + 96700, + 96750, + 96800, + 96850, + 96900, + 96950, + 97000, + 97050, + 97100, + 97150, + 97200, + 97250, + 97300, + 97350, + 97400, + 97450, + 97500, + 97550, + 97600, + 97650, + 97700, + 97750, + 97800, + 97850, + 97900, + 97950, + 98000, + 98050, + 98100, + 98150, + 98200, + 98250, + 98300, + 98350, + 98400, + 98450, + 98500, + 98550, + 98600, + 98650, + 98700, + 98750, + 98800, + 98850, + 98900, + 98950, + 99000, + 99050, + 99100, + 99150, + 99200, + 99250, + 99300, + 99350, + 99400, + 99450, + 99500, + 99550, + 99600, + 99650, + 99700, + 99750, + 99800, + 99850, + 99900, + 99950, + 100000, + 100050, + 100100, + 100150, + 100200, + 100250, + 100300, + 100350, + 100400, + 100450, + 100500, + 100550, + 100600, + 100650, + 100700, + 100750, + 100800, + 100850, + 100900, + 100950, + 101000, + 101050, + 101100, + 101150, + 101200, + 101250, + 101300, + 101350, + 101400, + 101450, + 101500, + 101550, + 101600, + 101650, + 101700, + 101750, + 101800, + 101850, + 101900, + 101950, + 102000, + 102050, + 102100, + 102150, + 102200, + 102250, + 102300, + 102350, + 102400, + 102450, + 102500, + 102550, + 102600, + 102650, + 102700, + 102750, + 102800, + 102850, + 102900, + 102950, + 103000, + 103050, + 103100, + 103150, + 103200, + 103250, + 103300, + 103350, + 103400, + 103450, + 103500, + 103550, + 103600, + 103650, + 103700, + 103750, + 103800, + 103850, + 103900, + 103950, + 104000, + 104050, + 104100, + 104150, + 104200, + 104250, + 104300, + 104350, + 104400, + 104450, + 104500, + 104550, + 104600, + 104650, + 104700, + 104750, + 104800, + 104850, + 104900, + 104950, + 105000, + 105050, + 105100, + 105150, + 105200, + 105250, + 105300, + 105350, + 105400, + 105450, + 105500, + 105550, + 105600, + 105650, + 105700, + 105750, + 105800, + 105850, + 105900, + 105950, + 106000, + 106050, + 106100, + 106150, + 106200, + 106250, + 106300, + 106350, + 106400, + 106450, + 106500, + 106550, + 106600, + 106650, + 106700, + 106750, + 106800, + 106850, + 106900, + 106950, + 107000, + 107050, + 107100, + 107150, + 107200, + 107250, + 107300, + 107350, + 107400, + 107450, + 107500, + 107550, + 107600, + 107650, + 107700, + 107750, + 107800, + 107850, + 107900, + 107950, + 108000, + 108050, + 108100, + 108150, + 108200, + 108250, + 108300, + 108350, + 108400, + 108450, + 108500, + 108550, + 108600, + 108650, + 108700, + 108750, + 108800, + 108850, + 108900, + 108950, + 109000, + 109050, + 109100, + 109150, + 109200, + 109250, + 109300, + 109350, + 109400, + 109450, + 109500, + 109550, + 109600, + 109650, + 109700, + 109750, + 109800, + 109850, + 109900, + 109950, + 110000, + 110050, + 110100, + 110150, + 110200, + 110250, + 110300, + 110350, + 110400, + 110450, + 110500, + 110550, + 110600, + 110650, + 110700, + 110750, + 110800, + 110850, + 110900, + 110950, + 111000, + 111050, + 111100, + 111150, + 111200, + 111250, + 111300, + 111350, + 111400, + 111450, + 111500, + 111550, + 111600, + 111650, + 111700, + 111750, + 111800, + 111850, + 111900, + 111950, + 112000, + 112050, + 112100, + 112150, + 112200, + 112250, + 112300, + 112350, + 112400, + 112450, + 112500, + 112550, + 112600, + 112650, + 112700, + 112750, + 112800, + 112850, + 112900, + 112950, + 113000, + 113050, + 113100, + 113150, + 113200, + 113250, + 113300, + 113350, + 113400, + 113450, + 113500, + 113550, + 113600, + 113650, + 113700, + 113750, + 113800, + 113850, + 113900, + 113950, + 114000 + ], + "train_lm_loss": [ + 2.056634306907654, + 2.042723536491394, + 2.0778759196400642, + 2.0830744579434395, + 1.9930649995803833, + 2.0000117123126984, + 2.0135350301861763, + 2.0823443606495857, + 2.0362224876880646, + 2.0275216549634933, + 2.0408994257450104, + 2.0489819198846817, + 2.122226595878601, + 2.0003229677677155, + 2.0586161389946938, + 2.0525637790560722, + 1.9753141850233078, + 2.1086814552545547, + 2.0775692388415337, + 2.024793840944767, + 2.01783799380064, + 2.110067553818226, + 2.04203662276268, + 2.01266897469759, + 2.023089751601219, + 2.0539930760860443, + 2.0137308463454247, + 2.0166506245732307, + 2.1163174137473106, + 1.9960755929350853, + 2.022157236933708, + 2.0499612241983414, + 2.071295388042927, + 2.005870223045349, + 2.058467037975788, + 2.053957872092724, + 2.060522846877575, + 2.0845325216650963, + 2.045583076775074, + 2.054957367479801, + 2.0581462308764458, + 2.0381749495863914, + 2.054376445710659, + 2.061845049262047, + 2.052484408020973, + 2.076095536351204, + 2.0067398995161057, + 1.9807870090007782, + 2.0265163481235504, + 2.0744279250502586, + 2.1030154153704643, + 2.0101450979709625, + 2.030967026948929, + 2.0731155946850777, + 2.046240784227848, + 2.027009814977646, + 2.0564359426498413, + 2.0733155086636543, + 2.04118499904871, + 2.078469194471836, + 1.9847282320261002, + 2.0448949858546257, + 2.0439492613077164, + 2.020939476788044, + 2.0658759772777557, + 2.064076080918312, + 2.0405943766236305, + 2.057353913784027, + 2.097002826631069, + 2.016542933881283, + 2.0314745903015137, + 2.030610665678978, + 2.05759859085083, + 1.9752275571227074, + 2.010603964328766, + 2.001951642334461, + 2.0455672666430473, + 2.0550715625286102, + 2.0270790085196495, + 2.0788628831505775, + 2.0231399685144424, + 2.0644757002592087, + 1.9939524456858635, + 2.013077735900879, + 2.0274162143468857, + 2.008769951760769, + 2.070554405450821, + 2.048803336918354, + 2.0322345569729805, + 2.007083460688591, + 2.042867660522461, + 2.080493688583374, + 2.0038804784417152, + 2.007066532969475, + 2.0516715571284294, + 1.9676597490906715, + 2.056869439780712, + 2.112566404044628, + 2.073686435818672, + 2.0658074766397476, + 1.9892392680048943, + 2.018832355737686, + 2.030092403292656, + 2.018119126558304, + 2.0001730397343636, + 2.028693236410618, + 2.0626792535185814, + 1.9976045712828636, + 2.087723657488823, + 2.054127149283886, + 2.060315616428852, + 2.0666304528713226, + 2.0736063346266747, + 2.014582760632038, + 2.04131481051445, + 2.054950699210167, + 2.0510181337594986, + 2.056834138929844, + 2.0046820640563965, + 2.0005365163087845, + 2.020183280110359, + 2.0755788385868073, + 2.0150781720876694, + 2.043303929269314, + 2.023023843765259, + 2.025325119495392, + 2.071675457060337, + 2.0370849817991257, + 2.0071223750710487, + 2.0913905054330826, + 1.9789845570921898, + 1.9967784136533737, + 2.009564682841301, + 2.0321762412786484, + 2.0756276473402977, + 2.056501515209675, + 2.106865257024765, + 2.0431954488158226, + 2.048810638487339, + 2.0154840871691704, + 2.0170809999108315, + 2.023192435503006, + 2.075350470840931, + 2.086780607700348, + 2.0370775386691093, + 2.026880592107773, + 1.9967768862843513, + 2.026461586356163, + 2.0389491245150566, + 2.0297572314739227, + 2.0632974207401276, + 2.054554082453251, + 2.1074495390057564, + 2.0386071130633354, + 2.0307971462607384, + 2.0548529401421547, + 2.02689990401268, + 2.0519658848643303, + 1.9971879869699478, + 2.028899662196636, + 2.0973974391818047, + 2.01481045037508, + 2.0050823017954826, + 2.097275488078594, + 1.9929770454764366, + 2.028214007616043, + 1.999344453215599, + 2.0400782972574234, + 1.9731034487485886, + 2.034611590206623, + 2.055023580789566, + 2.113173097372055, + 2.0707920119166374, + 1.968406155705452, + 2.0161736086010933, + 2.0175761729478836, + 1.9456613436341286, + 2.025512419641018, + 2.0173728242516518, + 2.002131372690201, + 2.027958668768406, + 2.028186723589897, + 2.024326004087925, + 2.006623923778534, + 2.0592111870646477, + 2.0624540373682976, + 1.9958266839385033, + 2.058178588747978, + 2.0080627650022507, + 2.0284092351794243, + 1.9656677693128586, + 1.997840791940689, + 2.037178121507168, + 2.039845585823059, + 2.0380933731794357, + 2.037462830543518, + 1.9928299635648727, + 2.041373521089554, + 2.048455759882927, + 2.0482649579644203, + 1.9649118706583977, + 2.038865327835083, + 1.9856999292969704, + 1.9928498640656471, + 2.053557924926281, + 2.0573812127113342, + 2.0177955329418182, + 2.005293495953083, + 2.05987635999918, + 1.9818605855107307, + 2.0210581719875336, + 1.9926248267292976, + 2.0219970792531967, + 2.0252958983182907, + 1.9931257739663124, + 2.0770480781793594, + 2.035587899386883, + 2.059744007885456, + 1.9876118376851082, + 2.015064537525177, + 2.0056181102991104, + 2.043839320540428, + 1.9911372438073158, + 1.9533571377396584, + 1.9806547164916992, + 2.0519709438085556, + 1.9967614337801933, + 2.034730389714241, + 2.0285056605935097, + 1.9913482442498207, + 1.971978947520256, + 2.0412209779024124, + 1.998925007879734, + 2.016299068927765, + 2.0251488015055656, + 2.057107239961624, + 1.9864870458841324, + 2.018696151673794, + 2.0692814514040947, + 1.9714944809675217, + 1.9489599615335464, + 2.0801594480872154, + 2.0098919793963432, + 2.0348945036530495, + 2.0279467180371284, + 2.0472500696778297, + 1.9726373925805092, + 2.0253152176737785, + 1.996934823691845, + 2.083639018237591, + 2.0529337599873543, + 2.040751986205578, + 2.0296598747372627, + 2.0773798674345016, + 2.0657101422548294, + 2.039336070418358, + 2.0391072630882263, + 1.9907563626766205, + 2.0053477957844734, + 2.026754543185234, + 2.0244283080101013, + 2.0705743357539177, + 2.0099001824855804, + 1.941980816423893, + 2.0280580446124077, + 1.9881991371512413, + 2.034957066178322, + 2.0269225910305977, + 2.040909558534622, + 1.9953540414571762, + 1.9754233583807945, + 2.036652311682701, + 2.019350364804268, + 1.96830265969038, + 2.0110625699162483, + 2.004908323287964, + 1.9876765385270119, + 2.0693057104945183, + 1.9977481588721275, + 2.0330845490098, + 2.050627402961254, + 2.0730484053492546, + 1.9978075474500656, + 2.0478819981217384, + 2.104213126003742, + 2.0886075720191, + 2.0021906718611717, + 2.0610282942652702, + 2.0317507535219193, + 2.079721048474312, + 2.033554658293724, + 1.987454392015934, + 1.9962974563241005, + 2.0132867097854614, + 2.036820888519287, + 1.9981699213385582, + 2.037012502551079, + 2.0385916978120804, + 2.017819456756115, + 1.994986891746521, + 2.059018053114414, + 2.013323910534382, + 2.030646361410618, + 2.0077463760972023, + 2.028165988624096, + 1.984341025352478, + 2.0004177019000053, + 2.0792005956172943, + 2.0306731685996056, + 2.011068768799305, + 1.9943395182490349, + 2.017463155090809, + 2.0459240153431892, + 2.0929026529192924, + 1.9865126386284828, + 2.011636734008789, + 2.045724295079708, + 1.9700713455677032, + 2.0791539177298546, + 2.0104187726974487, + 1.9717603400349617, + 1.9735903516411781, + 1.9918054267764091, + 1.9767247512936592, + 2.0353101193904877, + 2.0309352427721024, + 2.0095142871141434, + 2.054037109017372, + 2.0038552582263947, + 1.9603231251239777, + 2.0280587002635, + 2.021761015057564, + 2.0577462390065193, + 2.0200418680906296, + 2.021873116493225, + 1.9955458790063858, + 1.9984805136919022, + 1.9903134778141975, + 2.065230466425419, + 1.9808414950966835, + 2.0298884138464928, + 2.0350377559661865, + 2.017307959496975, + 2.049398608505726, + 2.0638146325945854, + 1.9516903534531593, + 2.003455877304077, + 2.0011551305651665, + 2.0095087736845016, + 1.962717279791832, + 2.0315073430538177, + 2.0279544964432716, + 2.022437058389187, + 1.9994832873344421, + 2.0043377950787544, + 2.0201037228107452, + 2.019993782043457, + 2.016444258391857, + 1.9985512271523476, + 1.9535131081938744, + 2.063063085079193, + 1.9515295922756195, + 2.0678438916802406, + 1.982693150639534, + 2.0215494632720947, + 1.9728758707642555, + 2.0275094732642174, + 1.9770054519176483, + 2.0104024559259415, + 2.0215619057416916, + 2.0038098469376564, + 2.046553313732147, + 2.005864642560482, + 2.047754980623722, + 1.949441023170948, + 1.9653135985136032, + 1.994687706232071, + 1.9873066172003746, + 2.0299218371510506, + 1.9706338867545128, + 1.9661610126495361, + 2.026771605014801, + 2.0297220200300217, + 2.050032302737236, + 2.0081716403365135, + 2.013861708343029, + 2.0395672991871834, + 2.052990272641182, + 1.9850402176380157, + 1.989513337612152, + 1.960232987999916, + 2.0061413422226906, + 2.0567664578557014, + 2.030290089547634, + 2.0557455122470856, + 2.0140588507056236, + 2.072558142244816, + 1.9878016114234924, + 2.02736859023571, + 2.0065028443932533, + 2.036472201347351, + 2.001084513962269, + 1.98545540869236, + 2.0553133860230446, + 1.9872069358825684, + 2.038708209991455, + 2.0499140843749046, + 2.0185500010848045, + 1.9799701198935509, + 2.011422038078308, + 2.040590465068817, + 2.0203530490398407, + 2.075219251215458, + 2.0278448089957237, + 2.014893338084221, + 2.135450631380081, + 2.05527476221323, + 2.0279663503170013, + 2.0841950699687004, + 1.9493668153882027, + 2.0596803054213524, + 2.0111391693353653, + 2.0656978711485863, + 2.0392111018300056, + 2.021916538476944, + 2.013943672180176, + 2.079200394451618, + 2.0083263888955116, + 2.0559271723031998, + 2.04885470867157, + 2.038327008485794, + 2.0435815677046776, + 2.035163089632988, + 1.9752878919243813, + 2.0332395657896996, + 2.0053645372390747, + 1.9989453107118607, + 2.049259513616562, + 1.969650886952877, + 1.9982682690024376, + 2.0629043132066727, + 2.0158949345350266, + 1.9888503476977348, + 1.9839312508702278, + 2.017633505165577, + 2.011141672730446, + 2.0931195318698883, + 2.0391959100961685, + 2.021135702729225, + 2.028821773827076, + 2.021524839103222, + 2.0400652661919594, + 2.0405413061380386, + 2.026640363037586, + 2.0422092005610466, + 1.991449624300003, + 2.024348594248295, + 2.064819075167179, + 2.0163105502724648, + 2.011019468307495, + 2.0293231084942818, + 2.0299551486968994, + 2.008136697113514, + 1.9970572739839554, + 2.0481806248426437, + 1.9912889003753662, + 2.0442994460463524, + 1.9910322204232216, + 2.0266743898391724, + 1.9999784976243973, + 2.008312463760376, + 2.0625249668955803, + 2.0617890283465385, + 1.9771273508667946, + 2.0605148002505302, + 1.9777631536126137, + 2.01988922059536, + 1.9905713722109795, + 2.0190222561359406, + 2.0388213768601418, + 2.0411465615034103, + 2.068674147129059, + 2.030368998646736, + 2.039605528116226, + 1.9835192263126373, + 2.0012989565730095, + 1.993776299059391, + 2.0418330281972885, + 2.091486506164074, + 2.025251179933548, + 2.1135023087263107, + 2.0531981959939003, + 2.040508560836315, + 2.0219036042690277, + 2.0429776310920715, + 2.072138376533985, + 2.047173611819744, + 2.040925867855549, + 2.061343252658844, + 2.065835639834404, + 2.0260923206806183, + 2.0099723264575005, + 2.032868094742298, + 2.047256000339985, + 2.013136602938175, + 2.024530202150345, + 1.9815638214349747, + 2.046955421566963, + 2.0083775520324707, + 2.042802944779396, + 2.0298011302948, + 2.008853979408741, + 2.045887239277363, + 1.974074810743332, + 2.030057430267334, + 2.0552425235509872, + 2.009108781814575, + 1.9691296592354774, + 2.0960229635238647, + 2.0708085894584656, + 2.023347459733486, + 2.0174287036061287, + 2.0328001976013184, + 2.038503296673298, + 2.0535950884222984, + 2.0224699676036835, + 2.016728200018406, + 2.0424589589238167, + 2.067999340593815, + 2.0636712685227394, + 2.027644008398056, + 2.037797659635544, + 2.0440652072429657, + 2.06362247467041, + 2.0903051495552063, + 2.0289282724261284, + 2.0238478034734726, + 2.038326323032379, + 2.0659397020936012, + 1.9835526123642921, + 2.0464139506220818, + 2.0298902690410614, + 2.0075491219758987, + 1.9918037056922913, + 1.935358613729477, + 2.038253530859947, + 1.9977165162563324, + 2.031728096306324, + 2.016676388680935, + 1.9825623780488968, + 2.017787344753742, + 2.081353187561035, + 1.9890751987695694, + 2.0533317998051643, + 2.0397534742951393, + 1.9640254825353622, + 2.0543231815099716, + 2.0452915877103806, + 2.0376105308532715, + 2.009821891784668, + 2.008651554584503, + 2.031516470015049, + 1.996335081756115, + 2.0109291672706604, + 2.0274588242173195, + 2.0583384186029434, + 2.0084821954369545, + 2.026429161429405, + 2.021872766315937, + 2.04173581302166, + 2.0261140540242195, + 2.0420551151037216, + 2.0374939516186714, + 2.009368844330311, + 2.005404196679592, + 1.9873533099889755, + 2.0475507229566574, + 1.9757003784179688, + 2.0142135471105576, + 2.0276147797703743, + 2.027939647436142, + 2.0262202844023705, + 1.9901736974716187, + 2.002513214945793, + 1.9887317568063736, + 2.051794335246086, + 2.0440256148576736, + 2.031392253935337, + 2.0642052441835403, + 1.981400966644287, + 2.031738832592964, + 1.9977665320038795, + 2.0391125604510307, + 2.0238738134503365, + 2.0292960330843925, + 2.01607283949852, + 1.9920953214168549, + 2.0227116718888283, + 1.9972597286105156, + 2.0829573571681976, + 2.025190256536007, + 2.0211936831474304, + 2.0426851511001587, + 1.9520322233438492, + 1.991441287100315, + 2.0408439561724663, + 2.007566913962364, + 2.0851309299468994, + 2.0086240246891975, + 2.0378425866365433, + 2.060731403529644, + 2.023512676358223, + 2.0667620599269867, + 1.961341917514801, + 2.0657320097088814, + 2.0476117357611656, + 2.0198022723197937, + 2.0121247842907906, + 2.044725254178047, + 2.0381190925836563, + 2.02546913176775, + 2.066698834300041, + 2.0614675283432007, + 2.050039619207382, + 1.984857328236103, + 2.0314126759767532, + 2.0533106848597527, + 2.029734507203102, + 2.0084728598594666, + 2.060582682490349, + 2.063216805458069, + 2.044323168694973, + 1.9942221343517303, + 2.0202794671058655, + 2.0204631313681602, + 1.980076052248478, + 2.097091242671013, + 2.0674255564808846, + 2.0105798169970512, + 2.0106528401374817, + 2.0577104166150093, + 1.9912999048829079, + 2.0104000568389893, + 1.9756155088543892, + 2.010587587952614, + 2.0208412930369377, + 1.9335214346647263, + 2.031992807984352, + 1.965048462152481, + 2.009037971496582, + 2.012113496661186, + 2.022594891488552, + 2.044933095574379, + 2.020239531993866, + 2.073130324482918, + 2.035692073404789, + 2.0699029117822647, + 2.0188287645578384, + 2.0019903108477592, + 2.0263382866978645, + 2.0275150761008263, + 2.022184520959854, + 2.0207660868763924, + 2.0063309893012047, + 2.0305333733558655, + 2.0107240453362465, + 2.028130866587162, + 1.975446455180645, + 2.0501552149653435, + 2.0180558636784554, + 1.986439771950245, + 2.0227156206965446, + 1.993635855615139, + 2.023946598172188, + 2.0611615106463432, + 2.0947342067956924, + 2.0055320858955383, + 2.0370763167738914, + 2.084538370370865, + 2.0160897597670555, + 1.997576355934143, + 2.029329776763916, + 2.068680487573147, + 2.0523690432310104, + 2.005965031683445, + 2.029499404132366, + 2.007810615003109, + 2.037222683429718, + 2.011931635439396, + 1.984961397945881, + 2.023342914879322, + 2.0604945346713066, + 2.0376061722636223, + 2.0744302719831467, + 2.059051401913166, + 1.9760089740157127, + 1.9803574457764626, + 2.0115713700652122, + 2.0062870234251022, + 2.042583502829075, + 2.036208339035511, + 1.9985091984272003, + 2.070684716105461, + 2.0290838927030563, + 2.0447983890771866, + 2.0667333975434303, + 2.016111932694912, + 2.0321648567914963, + 1.9935902655124664, + 2.0798029229044914, + 2.03485057502985, + 2.0299564003944397, + 2.030798487365246, + 2.060362547636032, + 2.059497833251953, + 1.9581471532583237, + 2.0300786569714546, + 2.058981314301491, + 2.005778916180134, + 2.025949642062187, + 1.9964686334133148, + 2.015907183289528, + 1.9647531881928444, + 2.0187485218048096, + 2.0766205117106438, + 2.08968548476696, + 2.051788903772831, + 2.044918030500412, + 2.0060421228408813, + 1.9996809363365173, + 2.024820551276207, + 2.026817761361599, + 2.021583080291748, + 1.9945007115602493, + 2.0135195031762123, + 1.9913892671465874, + 2.077386289834976, + 1.9710986390709877, + 1.9990082308650017, + 2.0222596898674965, + 2.004252813756466, + 1.9753919020295143, + 2.0320939123630524, + 2.0590453520417213, + 1.9890214502811432, + 2.0398473888635635, + 2.057387128472328, + 2.049452692270279, + 2.0349615961313248, + 2.00402595102787, + 1.9933370351791382, + 1.9967706054449081, + 2.065375506877899, + 2.0008651688694954, + 2.0524847432971, + 2.0375261902809143, + 2.024942561984062, + 2.040322780609131, + 2.053848847746849, + 2.014146313071251, + 1.9670651704072952, + 2.0595770105719566, + 2.0058394446969032, + 2.047056145966053, + 1.996963307261467, + 2.0037085488438606, + 2.042675793170929, + 2.0155977979302406, + 1.993407003581524, + 2.0283112078905106, + 2.0111302882432938, + 2.01959590613842, + 2.0031564980745316, + 2.027013510465622, + 2.0050100162625313, + 2.0700744166970253, + 2.046190246939659, + 2.050894156098366, + 2.004172556102276, + 2.067686155438423 + ], + "val_steps": [ + 80000, + 85000, + 90000, + 95000, + 100000, + 105000, + 110000, + 114000 + ], + "val_lm_loss": [ + 1.9557997846603394, + 1.9476701760292052, + 1.9409800553321839, + 1.9376245760917663, + 1.9355368399620057, + 1.9344543790817261, + 1.933139328956604, + 1.930645122528076 + ], + "val_ppl": [ + 7.069570897618006, + 7.012331037113062, + 6.965574274692993, + 6.942240604532866, + 6.927762156849322, + 6.920267182566716, + 6.911172665525928, + 6.893956253802674 + ] +} \ No newline at end of file diff --git a/078_cosmopedia_dense_deltanet/dense_deltanet_cosmopedia_best.pt b/078_cosmopedia_dense_deltanet/dense_deltanet_cosmopedia_best.pt new file mode 100644 index 0000000000000000000000000000000000000000..f276615cac0dec4f9f0181768db22feaf2a52c24 --- /dev/null +++ b/078_cosmopedia_dense_deltanet/dense_deltanet_cosmopedia_best.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57274ce5d513207e8389f646ea89926e193588a1d73273750021eea945dde77d +size 413742234 diff --git a/078_cosmopedia_dense_deltanet/dense_deltanet_cosmopedia_results.json b/078_cosmopedia_dense_deltanet/dense_deltanet_cosmopedia_results.json new file mode 100644 index 0000000000000000000000000000000000000000..54fc938a51d9421b737100b6ea8d81d903df9984 --- /dev/null +++ b/078_cosmopedia_dense_deltanet/dense_deltanet_cosmopedia_results.json @@ -0,0 +1,1192 @@ +{ + "label": "dense_deltanet_cosmopedia", + "total_params": 103421696, + "steps": [ + 85050, + 85100, + 85150, + 85200, + 85250, + 85300, + 85350, + 85400, + 85450, + 85500, + 85550, + 85600, + 85650, + 85700, + 85750, + 85800, + 85850, + 85900, + 85950, + 86000, + 86050, + 86100, + 86150, + 86200, + 86250, + 86300, + 86350, + 86400, + 86450, + 86500, + 86550, + 86600, + 86650, + 86700, + 86750, + 86800, + 86850, + 86900, + 86950, + 87000, + 87050, + 87100, + 87150, + 87200, + 87250, + 87300, + 87350, + 87400, + 87450, + 87500, + 87550, + 87600, + 87650, + 87700, + 87750, + 87800, + 87850, + 87900, + 87950, + 88000, + 88050, + 88100, + 88150, + 88200, + 88250, + 88300, + 88350, + 88400, + 88450, + 88500, + 88550, + 88600, + 88650, + 88700, + 88750, + 88800, + 88850, + 88900, + 88950, + 89000, + 89050, + 89100, + 89150, + 89200, + 89250, + 89300, + 89350, + 89400, + 89450, + 89500, + 89550, + 89600, + 89650, + 89700, + 89750, + 89800, + 89850, + 89900, + 89950, + 90000, + 90050, + 90100, + 90150, + 90200, + 90250, + 90300, + 90350, + 90400, + 90450, + 90500, + 90550, + 90600, + 90650, + 90700, + 90750, + 90800, + 90850, + 90900, + 90950, + 91000, + 91050, + 91100, + 91150, + 91200, + 91250, + 91300, + 91350, + 91400, + 91450, + 91500, + 91550, + 91600, + 91650, + 91700, + 91750, + 91800, + 91850, + 91900, + 91950, + 92000, + 92050, + 92100, + 92150, + 92200, + 92250, + 92300, + 92350, + 92400, + 92450, + 92500, + 92550, + 92600, + 92650, + 92700, + 92750, + 92800, + 92850, + 92900, + 92950, + 93000, + 93050, + 93100, + 93150, + 93200, + 93250, + 93300, + 93350, + 93400, + 93450, + 93500, + 93550, + 93600, + 93650, + 93700, + 93750, + 93800, + 93850, + 93900, + 93950, + 94000, + 94050, + 94100, + 94150, + 94200, + 94250, + 94300, + 94350, + 94400, + 94450, + 94500, + 94550, + 94600, + 94650, + 94700, + 94750, + 94800, + 94850, + 94900, + 94950, + 95000, + 95050, + 95100, + 95150, + 95200, + 95250, + 95300, + 95350, + 95400, + 95450, + 95500, + 95550, + 95600, + 95650, + 95700, + 95750, + 95800, + 95850, + 95900, + 95950, + 96000, + 96050, + 96100, + 96150, + 96200, + 96250, + 96300, + 96350, + 96400, + 96450, + 96500, + 96550, + 96600, + 96650, + 96700, + 96750, + 96800, + 96850, + 96900, + 96950, + 97000, + 97050, + 97100, + 97150, + 97200, + 97250, + 97300, + 97350, + 97400, + 97450, + 97500, + 97550, + 97600, + 97650, + 97700, + 97750, + 97800, + 97850, + 97900, + 97950, + 98000, + 98050, + 98100, + 98150, + 98200, + 98250, + 98300, + 98350, + 98400, + 98450, + 98500, + 98550, + 98600, + 98650, + 98700, + 98750, + 98800, + 98850, + 98900, + 98950, + 99000, + 99050, + 99100, + 99150, + 99200, + 99250, + 99300, + 99350, + 99400, + 99450, + 99500, + 99550, + 99600, + 99650, + 99700, + 99750, + 99800, + 99850, + 99900, + 99950, + 100000, + 100050, + 100100, + 100150, + 100200, + 100250, + 100300, + 100350, + 100400, + 100450, + 100500, + 100550, + 100600, + 100650, + 100700, + 100750, + 100800, + 100850, + 100900, + 100950, + 101000, + 101050, + 101100, + 101150, + 101200, + 101250, + 101300, + 101350, + 101400, + 101450, + 101500, + 101550, + 101600, + 101650, + 101700, + 101750, + 101800, + 101850, + 101900, + 101950, + 102000, + 102050, + 102100, + 102150, + 102200, + 102250, + 102300, + 102350, + 102400, + 102450, + 102500, + 102550, + 102600, + 102650, + 102700, + 102750, + 102800, + 102850, + 102900, + 102950, + 103000, + 103050, + 103100, + 103150, + 103200, + 103250, + 103300, + 103350, + 103400, + 103450, + 103500, + 103550, + 103600, + 103650, + 103700, + 103750, + 103800, + 103850, + 103900, + 103950, + 104000, + 104050, + 104100, + 104150, + 104200, + 104250, + 104300, + 104350, + 104400, + 104450, + 104500, + 104550, + 104600, + 104650, + 104700, + 104750, + 104800, + 104850, + 104900, + 104950, + 105000, + 105050, + 105100, + 105150, + 105200, + 105250, + 105300, + 105350, + 105400, + 105450, + 105500, + 105550, + 105600, + 105650, + 105700, + 105750, + 105800, + 105850, + 105900, + 105950, + 106000, + 106050, + 106100, + 106150, + 106200, + 106250, + 106300, + 106350, + 106400, + 106450, + 106500, + 106550, + 106600, + 106650, + 106700, + 106750, + 106800, + 106850, + 106900, + 106950, + 107000, + 107050, + 107100, + 107150, + 107200, + 107250, + 107300, + 107350, + 107400, + 107450, + 107500, + 107550, + 107600, + 107650, + 107700, + 107750, + 107800, + 107850, + 107900, + 107950, + 108000, + 108050, + 108100, + 108150, + 108200, + 108250, + 108300, + 108350, + 108400, + 108450, + 108500, + 108550, + 108600, + 108650, + 108700, + 108750, + 108800, + 108850, + 108900, + 108950, + 109000, + 109050, + 109100, + 109150, + 109200, + 109250, + 109300, + 109350, + 109400, + 109450, + 109500, + 109550, + 109600, + 109650, + 109700, + 109750, + 109800, + 109850, + 109900, + 109950, + 110000, + 110050, + 110100, + 110150, + 110200, + 110250, + 110300, + 110350, + 110400, + 110450, + 110500, + 110550, + 110600, + 110650, + 110700, + 110750, + 110800, + 110850, + 110900, + 110950, + 111000, + 111050, + 111100, + 111150, + 111200, + 111250, + 111300, + 111350, + 111400, + 111450, + 111500, + 111550, + 111600, + 111650, + 111700, + 111750, + 111800, + 111850, + 111900, + 111950, + 112000, + 112050, + 112100, + 112150, + 112200, + 112250, + 112300, + 112350, + 112400, + 112450, + 112500, + 112550, + 112600, + 112650, + 112700, + 112750, + 112800, + 112850, + 112900, + 112950, + 113000, + 113050, + 113100, + 113150, + 113200, + 113250, + 113300, + 113350, + 113400, + 113450, + 113500, + 113550, + 113600, + 113650, + 113700, + 113750, + 113800, + 113850, + 113900, + 113950, + 114000 + ], + "train_lm_loss": [ + 1.8993509486317635, + 1.8913578018546104, + 1.9462169110774994, + 1.9175557047128677, + 1.9137339890003204, + 1.9544687196612358, + 1.9069128707051277, + 1.8673961013555527, + 1.869547076523304, + 1.9640633314847946, + 1.9536357820034027, + 1.9478488638997078, + 1.9097196236252785, + 1.9306947216391563, + 1.8944845348596573, + 1.9364046305418015, + 1.9163610190153122, + 1.925491839647293, + 1.9320097044110298, + 1.9357634037733078, + 1.9794237166643143, + 1.9183538407087326, + 1.9409931749105453, + 1.8990054950118065, + 1.8842628300189972, + 1.9040403813123703, + 1.8916754573583603, + 1.931191235780716, + 1.955015629529953, + 1.9656467884778976, + 1.938490629196167, + 1.9075695052742958, + 1.934816874563694, + 1.914453960955143, + 1.9214404076337814, + 1.9295536801218987, + 1.8667733818292618, + 1.9518729969859123, + 1.9155190661549568, + 1.892993614077568, + 1.881025180220604, + 1.9391252249479294, + 1.9093966633081436, + 1.9575285986065865, + 1.9584178403019905, + 1.9545228332281113, + 1.9138535782694817, + 1.908821202814579, + 1.9805713072419167, + 1.8685731887817383, + 1.9154701605439186, + 1.9193433001637459, + 1.9195127487182617, + 1.9363009929656982, + 1.9430114924907684, + 1.9045645371079445, + 1.9380651265382767, + 1.926978461444378, + 1.892722524702549, + 1.9235548004508018, + 1.919811762869358, + 1.9339845329523087, + 1.8977335095405579, + 1.9145575538277626, + 1.925274796783924, + 1.9165361151099205, + 1.9550993889570236, + 1.9241130650043488, + 1.9582602381706238, + 1.898716539144516, + 1.8845161721110344, + 1.923081248998642, + 1.9606261625885963, + 1.8739067390561104, + 1.9380826726555824, + 1.9113118797540665, + 1.9251898303627968, + 1.9373723864555359, + 1.9121614694595337, + 1.9631621986627579, + 1.9083297550678253, + 1.978742316365242, + 1.903444305062294, + 1.8448917493224144, + 1.8934637606143951, + 1.8895453661680222, + 1.8804322928190231, + 1.8743648901581764, + 1.9113469198346138, + 1.967365637421608, + 1.8718299716711044, + 1.9645467847585678, + 1.9218448475003242, + 1.9183731079101562, + 1.934196837246418, + 1.8810397312045097, + 1.9277633801102638, + 1.9130713194608688, + 1.8807211890816689, + 1.896452210843563, + 1.9432829022407532, + 1.8884995952248573, + 1.933840200304985, + 1.9630557149648666, + 1.9507884085178375, + 1.8763100057840347, + 1.9175261035561562, + 1.8705361410975456, + 1.9018020629882812, + 1.9253737851977348, + 1.9198969900608063, + 1.9381261989474297, + 1.9565170779824257, + 1.893360584974289, + 1.883187673985958, + 1.9414309412240982, + 1.9159487783908844, + 1.9368500038981438, + 1.8728197813034058, + 1.9077808409929276, + 1.9001966416835785, + 1.9080384150147438, + 1.9019716829061508, + 1.9590949416160583, + 1.8895363807678223, + 1.9233874008059502, + 1.949795663356781, + 1.930750921368599, + 1.908883884549141, + 1.9362715110182762, + 1.8879271745681763, + 1.898847907781601, + 1.910587765276432, + 1.9504061341285706, + 1.9122270420193672, + 1.9210331588983536, + 1.9284740164875984, + 1.9254558756947517, + 1.9682474806904793, + 1.9411419034004211, + 1.892907328903675, + 1.9345186352729797, + 1.9760702550411224, + 1.921644426882267, + 1.901418924331665, + 1.8836387321352959, + 1.9482372999191284, + 1.9143284559249878, + 1.9006356820464134, + 1.9141264632344246, + 1.8703678995370865, + 1.9014236107468605, + 1.8465059772133827, + 1.9256030023097992, + 1.9119556993246078, + 1.9078045412898064, + 1.8767316713929176, + 1.9498994797468185, + 1.93028112500906, + 1.9073882550001144, + 1.9652492925524712, + 1.8768964484333992, + 1.9309115707874298, + 1.966635562479496, + 1.8941730037331581, + 1.912079133093357, + 1.9274219498038292, + 1.8622140064835548, + 1.921373799443245, + 2.0029648914933205, + 1.9270873740315437, + 1.9761667102575302, + 1.904133528470993, + 1.9303027838468552, + 1.8826880380511284, + 1.9193763136863708, + 1.8673029094934464, + 1.9282169789075851, + 1.9577817246317863, + 1.900231197476387, + 1.9152159839868546, + 1.8413608372211456, + 1.9019211903214455, + 1.9042233973741531, + 1.8960270956158638, + 1.9004351124167442, + 1.8634918853640556, + 1.8724862337112427, + 1.9253037348389626, + 1.9277152493596077, + 1.8585775345563889, + 1.9248638153076172, + 1.9604734182357788, + 1.902750387787819, + 1.9081002995371819, + 1.9178642183542252, + 1.8830500468611717, + 1.914499655365944, + 1.864804059267044, + 1.9307364150881767, + 1.9723359793424606, + 1.892360620200634, + 1.9091215208172798, + 1.959268942475319, + 1.9246518686413765, + 1.8888720199465752, + 1.8985977917909622, + 1.938220627605915, + 1.919536367058754, + 1.9377781376242638, + 1.9629558101296425, + 1.9773217216134071, + 1.9329802468419075, + 1.864185817539692, + 1.9490762054920197, + 1.9255956262350082, + 1.8956649005413055, + 1.9586830958724022, + 1.886858806014061, + 1.931414283812046, + 1.9340692982077599, + 1.9321967735886574, + 1.9272165596485138, + 1.870995618402958, + 1.9056110605597496, + 1.9220571219921112, + 1.8988176882266998, + 1.9122308418154716, + 1.8800499960780144, + 1.9179411232471466, + 1.9398182034492493, + 1.885853111743927, + 1.9240068420767784, + 1.9119306653738022, + 1.8853746578097343, + 1.924406073987484, + 2.0380795300006866, + 1.908091977238655, + 1.8952049016952515, + 1.9279706701636314, + 1.8943633362650871, + 1.8946423828601837, + 1.8851810544729233, + 1.9473041966557503, + 1.9180490896105766, + 1.9306344091892242, + 1.9187883511185646, + 1.9449745044112206, + 1.898177444934845, + 1.9148311242461205, + 1.9068198278546333, + 1.9000388905405998, + 1.886426068842411, + 1.9520436972379684, + 1.8781077340245247, + 1.8975373655557632, + 1.8854669257998466, + 1.914646990597248, + 1.9557287320494652, + 1.8834244385361671, + 1.8732532411813736, + 1.9175236225128174, + 1.864779956638813, + 1.9449376910924911, + 1.9433217868208885, + 1.9660806953907013, + 1.9061253145337105, + 1.9273666664958, + 1.9289216920733452, + 1.932887852191925, + 1.9443712085485458, + 1.9066361412405968, + 1.9036856219172478, + 1.8937370255589485, + 1.8797177225351334, + 1.8579362481832504, + 1.9156440272927284, + 1.9158053249120712, + 1.9209210947155952, + 1.908517800271511, + 1.939871110022068, + 1.9323527291417122, + 1.9154017195105553, + 1.9313650205731392, + 1.942598208785057, + 1.9063016697764397, + 1.935642160475254, + 1.892142429947853, + 1.935759037733078, + 1.889526180922985, + 1.9476546570658684, + 1.9313478022813797, + 1.8494428247213364, + 1.9719826951622963, + 1.9558906853199005, + 1.9163338616490364, + 1.9428957179188728, + 1.9422083422541618, + 1.97281876206398, + 1.88697399944067, + 1.8989820331335068, + 1.8817318007349968, + 1.909569926559925, + 1.8959646373987198, + 1.9271970763802528, + 1.9146582931280136, + 1.9107000902295113, + 1.9647833928465843, + 1.9092654660344124, + 1.9069137498736382, + 1.910465955734253, + 1.8663298040628433, + 1.9589476138353348, + 1.9399111568927765, + 1.9630805999040604, + 1.911949373781681, + 1.9587303921580315, + 1.936435028910637, + 1.9118696451187134, + 1.9399891719222069, + 1.9275632128119469, + 1.9110327884554863, + 1.8798263370990753, + 1.9473915323615074, + 1.9085867628455162, + 1.8986310213804245, + 1.882401593029499, + 1.9370605796575546, + 1.897752046585083, + 1.9300150945782661, + 1.9551788195967674, + 1.925673209130764, + 1.8873746916651726, + 1.9184299930930138, + 1.8885671496391296, + 1.901699721813202, + 1.879435122013092, + 1.8834273740649223, + 1.913024663925171, + 1.907528579235077, + 1.9088138788938522, + 1.895873785018921, + 1.939902976155281, + 1.898641623556614, + 1.9300665110349655, + 1.8815483376383781, + 1.9182746559381485, + 1.8881271705031395, + 1.9552011340856552, + 1.9337511360645294, + 1.9161063954234123, + 1.8807982727885246, + 1.9300132542848587, + 1.891732208430767, + 1.92069873213768, + 1.886291265487671, + 1.930952101945877, + 1.8904972299933434, + 1.9271034598350525, + 1.8599367812275887, + 1.8821983933448792, + 1.8715904876589775, + 1.8978794142603874, + 1.926045373082161, + 1.8841818794608116, + 1.882009282708168, + 1.8901946321129799, + 1.902532733976841, + 1.925018846988678, + 1.9204386472702026, + 1.9490227401256561, + 1.9126435369253159, + 1.8817402124404907, + 1.9466321393847466, + 1.950209617614746, + 1.8716078028082848, + 1.9574666619300842, + 1.8667886182665825, + 1.9202954545617104, + 1.8853767737746239, + 1.9312357977032661, + 1.8735490962862968, + 1.9151965901255608, + 1.8502085134387016, + 1.9485214725136757, + 1.9381404146552086, + 1.9555385038256645, + 1.8869276121258736, + 1.8923894241452217, + 1.956413209438324, + 1.954947255551815, + 1.9026309251785278, + 1.8926212191581726, + 1.9339983239769936, + 1.910304605960846, + 1.886472798883915, + 1.9216883480548859, + 1.9259201884269714, + 1.907299742102623, + 1.9254958108067513, + 1.9278681799769402, + 1.9134644716978073, + 1.8993704989552498, + 1.9418271780014038, + 1.904234193265438, + 1.9021797478199005, + 1.9108466506004333, + 1.922763466835022, + 1.8955828323960304, + 1.8516435250639915, + 1.9629215896129608, + 1.8605827316641808, + 1.9490898475050926, + 1.9186706840991974, + 1.9251472651958466, + 1.8754642754793167, + 1.8800168931484222, + 1.920440100133419, + 1.879056639969349, + 1.9094001576304436, + 1.9049727097153664, + 1.92082279920578, + 1.9117316901683807, + 1.9137994199991226, + 1.9492627307772636, + 1.8872556760907173, + 1.897338218986988, + 1.8933943286538124, + 1.8796545267105103, + 1.884909637272358, + 1.8477187380194664, + 1.886008895933628, + 1.917771689593792, + 1.8980033323168755, + 1.904886744916439, + 1.940460167825222, + 1.9593870267271996, + 1.9063709750771523, + 1.9491114169359207, + 1.9575831145048141, + 1.925810120999813, + 1.909021444618702, + 1.8995770514011383, + 1.9267735332250595, + 1.9498321115970612, + 1.9102553576231003, + 1.8842810094356537, + 1.878807120025158, + 1.943179875612259, + 1.9826108664274216, + 1.9257171154022217, + 1.969157062470913, + 1.880656398832798, + 1.9254154935479164, + 1.9331290423870087, + 1.929471343755722, + 1.9237162545323372, + 1.8972548246383667, + 1.9232234954833984, + 1.9082463905215263, + 1.9373218342661858, + 1.899821750819683, + 1.9204484447836876, + 1.8715757429599762, + 1.9577690809965134, + 1.9380791634321213, + 1.884612761437893, + 1.8424571081995964, + 1.958386093378067, + 1.8756610378623009, + 1.9407548531889915, + 1.9505297094583511, + 1.9322156310081482, + 1.8537162616848946, + 1.9141738563776016, + 1.8954143971204758, + 1.9069680497050285, + 1.9131209552288055, + 2.00232145935297, + 1.9557161182165146, + 1.9430338069796562, + 1.947689525783062, + 1.9209743663668633, + 1.9085825681686401, + 1.8796041905879974, + 1.909756138920784, + 1.8746148869395256, + 1.9251394644379616, + 1.8356220424175262, + 1.875704102218151, + 1.9033051133155823, + 1.8664770349860191, + 1.8842499181628227, + 1.9258777052164078, + 1.925527036190033, + 1.9369518086314201, + 1.9471411108970642, + 1.918559543788433, + 1.8639959692955017, + 1.9545784890651703, + 1.9116480723023415, + 1.9543559476733208, + 1.8378403186798096, + 1.8954553306102753, + 1.9908026456832886, + 1.950305089354515, + 1.8856734707951546, + 1.9260368198156357, + 1.90554741024971, + 1.9515779837965965, + 1.9466176182031631, + 1.9012921079993248, + 1.8885494321584702, + 1.9372845441102982, + 1.916036918759346, + 1.9563937112689018, + 1.9305287301540375, + 1.9488045871257782, + 1.9827722012996674, + 1.9040860682725906, + 1.8820802122354507, + 1.9018264934420586, + 1.942785069346428, + 1.894459456205368, + 1.8718074634671211, + 1.919627882540226, + 1.950400948524475, + 1.9353228732943535, + 1.8757927939295769, + 1.9327723160386086, + 1.913128912448883, + 1.8761787861585617, + 1.9128396287560463, + 1.9332176446914673, + 1.8786796033382416, + 1.884487360715866, + 1.912053443491459, + 1.9359993562102318, + 1.9456013068556786, + 1.9846030175685883, + 1.918519288301468, + 1.9419968724250793, + 1.8911189883947372, + 1.9296466931700706, + 1.9572546035051346, + 1.9783612787723541, + 1.937604658305645, + 1.9310293942689896, + 1.9563584327697754, + 1.888058990240097, + 1.950521856546402, + 1.91691505163908, + 1.9417264387011528, + 1.9405465573072433, + 1.891113519668579, + 1.8557255044579506, + 1.9667765125632286, + 1.8780313059687614, + 1.926493525505066, + 1.8736927956342697, + 1.9464675784111023, + 1.8911215737462044, + 1.931305430829525, + 1.9426800087094307, + 1.9019600674510002, + 1.9311100244522095, + 1.939404547214508, + 1.877443477511406, + 1.9473941177129745, + 1.9533710554242134, + 1.9531152918934822, + 1.905884012579918, + 1.941638521850109, + 1.9659530594944954, + 1.9137183874845505, + 1.9397465661168098, + 1.9514736086130142, + 1.8991069570183754, + 1.9203240275382996, + 1.9382868856191635 + ], + "val_steps": [ + 90000, + 95000, + 100000, + 105000, + 110000, + 114000 + ], + "val_lm_loss": [ + 1.899927294254303, + 1.8965708994865418, + 1.8956851434707642, + 1.895426366329193, + 1.894984676837921, + 1.8928855013847352 + ], + "val_ppl": [ + 6.685408357009003, + 6.663007102136407, + 6.657107916517787, + 6.655385432039437, + 6.652446467336432, + 6.638496461885675 + ] +} \ No newline at end of file