Franok / source /docs /evaluation /pretraining.jsonl
yunusemrejr's picture
Publish complete source and evaluations from 22d18d084b18
c41b3ce verified
Raw History Blame Contribute Delete
11.5 kB
{"hardware": "NVIDIA L4", "torch": "2.14.1+cu130", "parameters": 314061312, "options": {"steps": 3200, "batch": 24, "length": 256, "minutes": 65.0, "resume": false}}
{"step": 0, "validation": {"en-chat": 9.105072021484375, "en-text": 9.099853515625, "tr-chat": 9.097759246826172, "tr-text": 9.09213638305664}}
{"step": 1, "loss": 9.0839, "lr": 1.0625e-05, "elapsed": 0.5}
{"step": 25, "loss": 7.7957, "lr": 0.00026559403362119564, "elapsed": 4.6}
{"step": 50, "loss": 7.2962, "lr": 0.0005309918771610558, "elapsed": 8.9}
{"step": 75, "loss": 7.2138, "lr": 0.0007959921606096713, "elapsed": 13.2}
{"step": 100, "loss": 6.9397, "lr": 0.0008483151296340993, "elapsed": 17.6}
{"step": 125, "loss": 6.7901, "lr": 0.0008473579260698794, "elapsed": 21.9}
{"step": 150, "loss": 6.4237, "lr": 0.0008461872705412913, "elapsed": 26.3}
{"step": 175, "loss": 6.4766, "lr": 0.0008448038682074518, "elapsed": 30.6}
{"step": 200, "loss": 6.3541, "lr": 0.000843208552378198, "elapsed": 35.0}
{"step": 225, "loss": 6.2811, "lr": 0.0008414022840121328, "elapsed": 39.4}
{"step": 250, "loss": 6.0852, "lr": 0.0008393861511377792, "elapsed": 43.7}
{"step": 275, "loss": 5.9495, "lr": 0.0008371613681981927, "elapsed": 48.1}
{"step": 300, "loss": 5.8963, "lr": 0.000834729275319426, "elapsed": 52.5}
{"step": 325, "loss": 5.9049, "lr": 0.0008320913375032876, "elapsed": 56.9}
{"step": 350, "loss": 5.8683, "lr": 0.000829249143744879, "elapsed": 61.3}
{"step": 375, "loss": 5.6889, "lr": 0.0008262044060754438, "elapsed": 65.7}
{"step": 400, "loss": 5.7129, "lr": 0.0008229589585311038, "elapsed": 70.1}
{"step": 400, "validation": {"en-chat": 5.799419641494751, "en-text": 5.806256055831909, "tr-chat": 5.46284282207489, "tr-text": 5.790651082992554}}
{"step": 425, "loss": 5.7627, "lr": 0.000819514756048105, "elapsed": 85.7}
{"step": 450, "loss": 5.7606, "lr": 0.000815873873285236, "elapsed": 90.1}
{"step": 475, "loss": 5.7249, "lr": 0.0008120385033741316, "elapsed": 94.5}
{"step": 500, "loss": 5.468, "lr": 0.0008080109565982126, "elapsed": 98.9}
{"step": 525, "loss": 5.618, "lr": 0.0008037936590010574, "elapsed": 103.4}
{"step": 550, "loss": 5.4839, "lr": 0.0007993891509250433, "elapsed": 107.8}
{"step": 575, "loss": 5.5849, "lr": 0.0007948000854811401, "elapsed": 112.3}
{"step": 600, "loss": 5.3814, "lr": 0.0007900292269507753, "elapsed": 116.7}
{"step": 625, "loss": 5.5435, "lr": 0.0007850794491207333, "elapsed": 121.2}
{"step": 650, "loss": 5.4361, "lr": 0.0007799537335520953, "elapsed": 125.7}
{"step": 675, "loss": 5.3268, "lr": 0.0007746551677842565, "elapsed": 130.1}
{"step": 700, "loss": 5.2794, "lr": 0.0007691869434751096, "elapsed": 134.6}
{"step": 725, "loss": 5.524, "lr": 0.0007635523544785085, "elapsed": 139.1}
{"step": 750, "loss": 5.4445, "lr": 0.0007577547948601734, "elapsed": 143.6}
{"step": 775, "loss": 5.252, "lr": 0.0007517977568532341, "elapsed": 148.1}
{"step": 800, "loss": 5.2484, "lr": 0.0007456848287546378, "elapsed": 152.6}
{"step": 800, "validation": {"en-chat": 5.338170051574707, "en-text": 5.376357793807983, "tr-chat": 4.89131224155426, "tr-text": 5.257007360458374}}
{"step": 825, "loss": 5.239, "lr": 0.000739419692763694, "elapsed": 168.6}
{"step": 850, "loss": 5.1703, "lr": 0.0007330061227640552, "elapsed": 173.1}
{"step": 875, "loss": 5.0804, "lr": 0.0007264479820504703, "elapsed": 177.6}
{"step": 900, "loss": 5.3154, "lr": 0.0007197492210016805, "elapsed": 182.1}
{"step": 925, "loss": 5.2842, "lr": 0.0007129138747008589, "elapsed": 186.6}
{"step": 950, "loss": 5.1809, "lr": 0.0007059460605050266, "elapsed": 191.1}
{"step": 975, "loss": 5.3406, "lr": 0.0006988499755649124, "elapsed": 195.6}
{"step": 1000, "loss": 5.2758, "lr": 0.0006916298942967442, "elapsed": 200.1}
{"step": 1025, "loss": 5.1067, "lr": 0.0006842901658075018, "elapsed": 204.5}
{"step": 1050, "loss": 5.2393, "lr": 0.0006768352112751769, "elapsed": 209.0}
{"step": 1075, "loss": 5.1331, "lr": 0.0006692695212856212, "elapsed": 213.5}
{"step": 1100, "loss": 5.1425, "lr": 0.0006615976531275849, "elapsed": 218.0}
{"step": 1125, "loss": 5.0878, "lr": 0.0006538242280475769, "elapsed": 222.4}
{"step": 1150, "loss": 4.8275, "lr": 0.0006459539284661982, "elapsed": 226.9}
{"step": 1175, "loss": 5.1303, "lr": 0.0006379914951576274, "elapsed": 231.4}
{"step": 1200, "loss": 4.9823, "lr": 0.0006299417243939547, "elapsed": 235.8}
{"step": 1200, "validation": {"en-chat": 5.076594591140747, "en-text": 5.0938804149627686, "tr-chat": 4.597304105758667, "tr-text": 4.935847759246826}}
{"step": 1225, "loss": 4.9232, "lr": 0.0006218094650560881, "elapsed": 252.0}
{"step": 1250, "loss": 5.0304, "lr": 0.0006135996157129677, "elapsed": 256.5}
{"step": 1275, "loss": 4.8968, "lr": 0.0006053171216708522, "elapsed": 261.0}
{"step": 1300, "loss": 4.9225, "lr": 0.0005969669719944494, "elapsed": 265.5}
{"step": 1325, "loss": 4.919, "lr": 0.000588554196501693, "elapsed": 270.0}
{"step": 1350, "loss": 5.029, "lr": 0.0005800838627339652, "elapsed": 274.5}
{"step": 1375, "loss": 4.8, "lr": 0.000571561072903602, "elapsed": 279.0}
{"step": 1400, "loss": 4.7834, "lr": 0.0005629909608205095, "elapsed": 283.5}
{"step": 1425, "loss": 4.8984, "lr": 0.0005543786887997491, "elapsed": 288.0}
{"step": 1450, "loss": 4.855, "lr": 0.000545729444551953, "elapsed": 292.5}
{"step": 1475, "loss": 4.818, "lr": 0.0005370484380584395, "elapsed": 297.0}
{"step": 1500, "loss": 4.7386, "lr": 0.0005283408984329169, "elapsed": 301.5}
{"step": 1525, "loss": 4.7805, "lr": 0.0005196120707716596, "elapsed": 305.9}
{"step": 1550, "loss": 4.8542, "lr": 0.0005108672129940587, "elapsed": 310.4}
{"step": 1575, "loss": 4.8574, "lr": 0.0005021115926754486, "elapsed": 314.9}
{"step": 1600, "loss": 4.8696, "lr": 0.0004933504838741151, "elapsed": 319.3}
{"step": 1600, "validation": {"en-chat": 4.8916977643966675, "en-text": 4.843601703643799, "tr-chat": 4.3593608140945435, "tr-text": 4.710019469261169}}
{"step": 1625, "loss": 4.6734, "lr": 0.0004845891639544012, "elapsed": 335.4}
{"step": 1650, "loss": 4.7368, "lr": 0.0004758329104078198, "elapsed": 339.9}
{"step": 1675, "loss": 4.6734, "lr": 0.0004670869976740899, "elapsed": 344.3}
{"step": 1700, "loss": 4.6777, "lr": 0.00045835669396401157, "elapsed": 348.8}
{"step": 1725, "loss": 4.6383, "lr": 0.00044964725808609266, "elapsed": 353.3}
{"step": 1750, "loss": 4.4583, "lr": 0.0004409639362788392, "elapsed": 357.8}
{"step": 1775, "loss": 4.6482, "lr": 0.0004323119590506176, "elapsed": 362.3}
{"step": 1800, "loss": 4.401, "lr": 0.000423696538028992, "elapsed": 366.8}
{"step": 1825, "loss": 4.6571, "lr": 0.0004151228628214344, "elapsed": 371.3}
{"step": 1850, "loss": 4.6726, "lr": 0.0004065960978892994, "elapsed": 375.8}
{"step": 1875, "loss": 4.6911, "lr": 0.0003981213794369458, "elapsed": 380.3}
{"step": 1900, "loss": 4.7001, "lr": 0.0003897038123178793, "elapsed": 384.8}
{"step": 1925, "loss": 4.7697, "lr": 0.0003813484669597794, "elapsed": 389.3}
{"step": 1950, "loss": 4.3841, "lr": 0.000373060376310263, "elapsed": 393.8}
{"step": 1975, "loss": 4.5125, "lr": 0.00036484453280522596, "elapsed": 398.3}
{"step": 2000, "loss": 4.5179, "lr": 0.00035670588536158555, "elapsed": 402.7}
{"step": 2000, "validation": {"en-chat": 4.6717376708984375, "en-text": 4.66798460483551, "tr-chat": 4.170347452163696, "tr-text": 4.531984567642212}}
{"step": 2025, "loss": 4.5235, "lr": 0.0003486493363962381, "elapsed": 418.8}
{"step": 2050, "loss": 4.5919, "lr": 0.00034067973887302655, "elapsed": 423.3}
{"step": 2075, "loss": 4.5392, "lr": 0.00033280189337949655, "elapsed": 427.7}
{"step": 2100, "loss": 4.578, "lr": 0.0003250205452352015, "elapsed": 432.2}
{"step": 2125, "loss": 4.4564, "lr": 0.00031734038163330053, "elapsed": 436.7}
{"step": 2150, "loss": 4.4431, "lr": 0.0003097660288171675, "elapsed": 441.2}
{"step": 2175, "loss": 4.5285, "lr": 0.0003023020492937155, "elapsed": 445.7}
{"step": 2200, "loss": 4.6282, "lr": 0.0002949529390851133, "elapsed": 450.2}
{"step": 2225, "loss": 4.555, "lr": 0.00028772312502054967, "elapsed": 454.7}
{"step": 2250, "loss": 4.4448, "lr": 0.00028061696206967677, "elapsed": 459.2}
{"step": 2275, "loss": 4.5926, "lr": 0.00027363873071933984, "elapsed": 463.7}
{"step": 2300, "loss": 4.512, "lr": 0.0002667926343951715, "elapsed": 468.2}
{"step": 2325, "loss": 4.2647, "lr": 0.0002600827969296053, "elapsed": 472.7}
{"step": 2350, "loss": 4.5518, "lr": 0.0002535132600778334, "elapsed": 477.2}
{"step": 2375, "loss": 4.6265, "lr": 0.0002470879810832046, "elapsed": 481.6}
{"step": 2400, "loss": 4.3257, "lr": 0.00024081083029352878, "elapsed": 486.1}
{"step": 2400, "validation": {"en-chat": 4.51096785068512, "en-text": 4.488943457603455, "tr-chat": 4.005213916301727, "tr-text": 4.354970812797546}}
{"step": 2425, "loss": 4.4052, "lr": 0.0002346855888297236, "elapsed": 501.8}
{"step": 2450, "loss": 4.2672, "lr": 0.0002287159463082095, "elapsed": 506.3}
{"step": 2475, "loss": 4.1985, "lr": 0.00022290549861842202, "elapsed": 510.7}
{"step": 2500, "loss": 4.3785, "lr": 0.00021725774575678282, "elapsed": 515.2}
{"step": 2525, "loss": 4.1344, "lr": 0.00021177608971843277, "elapsed": 519.7}
{"step": 2550, "loss": 4.2312, "lr": 0.00020646383244799708, "elapsed": 524.2}
{"step": 2575, "loss": 4.3202, "lr": 0.0002013241738506173, "elapsed": 528.7}
{"step": 2600, "loss": 4.2824, "lr": 0.00019636020986444827, "elapsed": 533.2}
{"step": 2625, "loss": 4.0665, "lr": 0.0001915749305957807, "elapsed": 537.7}
{"step": 2650, "loss": 4.2915, "lr": 0.00018697121851791285, "elapsed": 542.2}
{"step": 2675, "loss": 4.0365, "lr": 0.0001825518467348567, "elapsed": 546.7}
{"step": 2700, "loss": 4.0437, "lr": 0.0001783194773109233, "elapsed": 551.2}
{"step": 2725, "loss": 4.1994, "lr": 0.0001742766596671952, "elapsed": 555.7}
{"step": 2750, "loss": 4.1352, "lr": 0.00017042582904585024, "elapsed": 560.2}
{"step": 2775, "loss": 4.0371, "lr": 0.00016676930504326283, "elapsed": 564.6}
{"step": 2800, "loss": 3.9858, "lr": 0.00016330929021276608, "elapsed": 569.1}
{"step": 2800, "validation": {"en-chat": 4.332608222961426, "en-text": 4.401367783546448, "tr-chat": 3.803794264793396, "tr-text": 4.304149270057678}}
{"step": 2825, "loss": 4.0682, "lr": 0.00016004786873791542, "elapsed": 585.2}
{"step": 2850, "loss": 3.949, "lr": 0.000156987005177055, "elapsed": 589.7}
{"step": 2875, "loss": 3.9821, "lr": 0.00015412854327994074, "elapsed": 594.1}
{"step": 2900, "loss": 3.9945, "lr": 0.0001514742048771347, "elapsed": 598.6}
{"step": 2925, "loss": 4.1055, "lr": 0.0001490255888428387, "elapsed": 603.1}
{"step": 2950, "loss": 3.9349, "lr": 0.00014678417013179207, "elapsed": 607.6}
{"step": 2975, "loss": 4.2881, "lr": 0.0001447512988908144, "elapsed": 612.1}
{"step": 3000, "loss": 3.8894, "lr": 0.00014292819964552717, "elapsed": 616.6}
{"step": 3025, "loss": 4.0608, "lr": 0.00014131597056274572, "elapsed": 621.1}
{"step": 3050, "loss": 4.0052, "lr": 0.0001399155827889844, "elapsed": 625.6}
{"step": 3075, "loss": 3.8881, "lr": 0.00013872787986547426, "elapsed": 630.1}
{"step": 3100, "loss": 4.1351, "lr": 0.0001377535772200454, "elapsed": 634.6}
{"step": 3125, "loss": 3.9423, "lr": 0.0001369932617361798, "elapsed": 639.1}
{"step": 3150, "loss": 3.9433, "lr": 0.00013644739139949494, "elapsed": 643.6}
{"step": 3175, "loss": 3.9781, "lr": 0.0001361162950218703, "elapsed": 648.0}
{"step": 3200, "loss": 4.0099, "lr": 0.00013600017204338323, "elapsed": 652.5}
{"step": 3200, "validation": {"en-chat": 4.227522850036621, "en-text": 4.345997452735901, "tr-chat": 3.6909888982772827, "tr-text": 4.251839280128479}}