| { |
| "HuggingFaceFW/fineweb-edu-classifier": { |
| "en_llama3": { |
| "n": 6000, |
| "spearman": 0.7811745959348265, |
| "pearson": 0.8254685326164869, |
| "f1_ge3": 0.7167381974248928, |
| "acc_round": 0.7331666666666666, |
| "mean_pred": 1.303708251953125 |
| }, |
| "fw2_gemma": { |
| "n": 9750, |
| "spearman": 0.31332828330222384, |
| "pearson": 0.3201400964584369, |
| "f1_ge3": 0.014141414141414142, |
| "acc_round": 0.4175384615384615, |
| "mean_pred": 0.7903328074919872, |
| "by_lang": { |
| "afr_Latn": 0.4783886151902705, |
| "amh_Ethi": 0.0035800511195597565, |
| "arb_Arab": 0.18433856457178072, |
| "azj_Latn": 0.09337074285835445, |
| "ben_Beng": -0.09760323618540227, |
| "bul_Cyrl": 0.20503335722132798, |
| "cat_Latn": 0.6595302771997552, |
| "ces_Latn": 0.3599752431781002, |
| "cmn_Hani": 0.36116549630361733, |
| "dan_Latn": 0.6001897248450279, |
| "deu_Latn": 0.6149203082616248, |
| "ekk_Latn": 0.3708713726345026, |
| "ell_Grek": 0.2641394141294525, |
| "eus_Latn": 0.3617396078726179, |
| "fas_Arab": 0.25725247538293716, |
| "fil_Latn": 0.5192007594329661, |
| "fin_Latn": 0.4594319524209811, |
| "fra_Latn": 0.7030699852374831, |
| "glg_Latn": 0.46222411627307064, |
| "guj_Gujr": 0.17410758233215715, |
| "heb_Hebr": 0.2552741558144449, |
| "hin_Deva": 0.21684813093532157, |
| "hrv_Latn": 0.3901958638082959, |
| "hun_Latn": 0.4679083623541295, |
| "hye_Armn": 0.17378206897576237, |
| "ind_Latn": 0.5425439672915935, |
| "isl_Latn": 0.36410277837656496, |
| "ita_Latn": 0.7114280686997687, |
| "jpn_Jpan": 0.35637333571825525, |
| "kan_Knda": 0.2159835575450683, |
| "kat_Geor": 0.20181702902943727, |
| "kaz_Cyrl": 0.07028438701894076, |
| "khm_Khmr": 0.19854930856895855, |
| "kor_Hang": 0.36423478651136526, |
| "lit_Latn": 0.44426372175382817, |
| "lvs_Latn": 0.42898220014392424, |
| "mal_Mlym": 0.254422574383917, |
| "mar_Deva": -0.02052601043908013, |
| "mkd_Cyrl": 0.17931674439965872, |
| "mya_Mymr": 0.060353023381646796, |
| "nld_Latn": 0.7176838890206896, |
| "nob_Latn": 0.6039381224420028, |
| "npi_Deva": 0.24364382637822735, |
| "pan_Guru": 0.3141409775437518, |
| "pol_Latn": 0.5030397252962449, |
| "por_Latn": 0.6909333335761909, |
| "ron_Latn": 0.5864636995970052, |
| "rus_Cyrl": 0.2812129159232862, |
| "sin_Sinh": 0.3592815174301788, |
| "slk_Latn": 0.4598330329453881, |
| "slv_Latn": 0.5057813338679621, |
| "spa_Latn": 0.6448157099816524, |
| "srp_Cyrl": 0.23893350817449613, |
| "swe_Latn": 0.6506862369110045, |
| "swh_Latn": 0.25299473872794553, |
| "tam_Taml": 0.22364087822978565, |
| "tel_Telu": 0.1593746895000516, |
| "tha_Thai": 0.25103745018468, |
| "tur_Latn": 0.5612733688350385, |
| "ukr_Cyrl": 0.2528674145533893, |
| "urd_Arab": 0.15879523016824826, |
| "uzn_Latn": 0.5122183422461761, |
| "vie_Latn": 0.23123533417387576, |
| "yue_Hani": 0.13726957886753305, |
| "zsm_Latn": 0.6246551025461468 |
| }, |
| "lang_mean_spearman": 0.3550279141795836 |
| } |
| }, |
| "nvidia/nemocurator-fineweb-nemotron-4-edu-classifier": { |
| "en_llama3": { |
| "n": 6000, |
| "spearman": 0.6900049364432077, |
| "pearson": 0.6947466119935968, |
| "f1_ge3": 0.5593952483801296, |
| "acc_round": 0.5658333333333333, |
| "mean_pred": 1.387032918294271 |
| }, |
| "fw2_gemma": { |
| "n": 9750, |
| "spearman": 0.2895303544363772, |
| "pearson": 0.2906974159907195, |
| "f1_ge3": 0.004081632653061225, |
| "acc_round": 0.45066666666666666, |
| "mean_pred": 0.640254885942508, |
| "by_lang": { |
| "afr_Latn": 0.4125513088566815, |
| "amh_Ethi": 0.018373213098583384, |
| "arb_Arab": 0.0462338281457406, |
| "azj_Latn": 0.019992976790920432, |
| "ben_Beng": -0.16789892750113863, |
| "bul_Cyrl": 0.2971403726411454, |
| "cat_Latn": 0.6147170553906413, |
| "ces_Latn": 0.39892159831065555, |
| "cmn_Hani": 0.33236056343520864, |
| "dan_Latn": 0.627986710030984, |
| "deu_Latn": 0.5421583888861886, |
| "ekk_Latn": 0.3412602251624262, |
| "ell_Grek": 0.2877695026495254, |
| "eus_Latn": 0.26273050172930285, |
| "fas_Arab": 0.24343030252432032, |
| "fil_Latn": 0.46762500797401985, |
| "fin_Latn": 0.5002472399939247, |
| "fra_Latn": 0.6700508212975463, |
| "glg_Latn": 0.5633083561358194, |
| "guj_Gujr": 0.14136444259530326, |
| "heb_Hebr": 0.04724066128936813, |
| "hin_Deva": 0.22134439322375482, |
| "hrv_Latn": 0.3966049006476282, |
| "hun_Latn": 0.45954093494157644, |
| "hye_Armn": 0.16936739322102382, |
| "ind_Latn": 0.5748003182538665, |
| "isl_Latn": 0.30026748669376563, |
| "ita_Latn": 0.6741036997984878, |
| "jpn_Jpan": 0.3174746581217568, |
| "kan_Knda": 0.023717881864377573, |
| "kat_Geor": 0.21079149629341118, |
| "kaz_Cyrl": 0.033985861656622506, |
| "khm_Khmr": 0.1334958353366269, |
| "kor_Hang": 0.3697988953836682, |
| "lit_Latn": 0.5016548732038005, |
| "lvs_Latn": 0.42824583647664066, |
| "mal_Mlym": 0.22060159988132563, |
| "mar_Deva": 0.012161465310395518, |
| "mkd_Cyrl": 0.2016088441163244, |
| "mya_Mymr": 0.04140935967384132, |
| "nld_Latn": 0.6911094076936222, |
| "nob_Latn": 0.6496771853598793, |
| "npi_Deva": 0.23923633067121167, |
| "pan_Guru": 0.31902012692687165, |
| "pol_Latn": 0.5516762448979281, |
| "por_Latn": 0.6629914217097714, |
| "ron_Latn": 0.5406291022067823, |
| "rus_Cyrl": 0.2241661254766134, |
| "sin_Sinh": 0.43543983599659747, |
| "slk_Latn": 0.4690493972306032, |
| "slv_Latn": 0.42703517488655535, |
| "spa_Latn": 0.6779816003990441, |
| "srp_Cyrl": 0.25743830692710934, |
| "swe_Latn": 0.6697583783089793, |
| "swh_Latn": 0.2949237410105869, |
| "tam_Taml": 0.2864941876005439, |
| "tel_Telu": 0.24945242629027456, |
| "tha_Thai": 0.23070699677653075, |
| "tur_Latn": 0.5232767110609888, |
| "ukr_Cyrl": 0.17294253430222284, |
| "urd_Arab": 0.09084790133296561, |
| "uzn_Latn": 0.36802535578818296, |
| "vie_Latn": 0.3066835352546123, |
| "yue_Hani": 0.2059018457197195, |
| "zsm_Latn": 0.6224283932948171 |
| }, |
| "lang_mean_spearman": 0.34039126385629354 |
| } |
| }, |
| "nvidia/nemocurator-fineweb-mixtral-edu-classifier": { |
| "en_llama3": { |
| "n": 6000, |
| "spearman": 0.7175490136043337, |
| "pearson": 0.7287184005488518, |
| "f1_ge3": 0.614963503649635, |
| "acc_round": 0.6028333333333333, |
| "mean_pred": 1.3429691162109374 |
| }, |
| "fw2_gemma": { |
| "n": 9750, |
| "spearman": 0.2819134647436298, |
| "pearson": 0.28562988667839534, |
| "f1_ge3": 0.002044989775051125, |
| "acc_round": 0.45374358974358975, |
| "mean_pred": 0.47195639272836537, |
| "by_lang": { |
| "afr_Latn": 0.47159183192913195, |
| "amh_Ethi": 0.019598498538088625, |
| "arb_Arab": 0.06822991834984035, |
| "azj_Latn": 0.06953770032467291, |
| "ben_Beng": -0.03358448985760162, |
| "bul_Cyrl": 0.27245858557578745, |
| "cat_Latn": 0.5538705546867907, |
| "ces_Latn": 0.35157906715040116, |
| "cmn_Hani": 0.3077231124197137, |
| "dan_Latn": 0.6222795229134043, |
| "deu_Latn": 0.6118166183216837, |
| "ekk_Latn": 0.40074372684756604, |
| "ell_Grek": 0.19232566498760245, |
| "eus_Latn": 0.3155052398487781, |
| "fas_Arab": 0.25391343265204913, |
| "fil_Latn": 0.4646692271862731, |
| "fin_Latn": 0.4759450597456664, |
| "fra_Latn": 0.663792113652442, |
| "glg_Latn": 0.5322872358954756, |
| "guj_Gujr": 0.20596184394454778, |
| "heb_Hebr": 0.10716405578859328, |
| "hin_Deva": 0.1838808075368145, |
| "hrv_Latn": 0.3139358872501065, |
| "hun_Latn": 0.5081555792186178, |
| "hye_Armn": 0.12716098990970462, |
| "ind_Latn": 0.5621676785379013, |
| "isl_Latn": 0.24641095943858662, |
| "ita_Latn": 0.658190727051437, |
| "jpn_Jpan": 0.2598441397658701, |
| "kan_Knda": 0.17806570191716214, |
| "kat_Geor": 0.0701215725725029, |
| "kaz_Cyrl": 0.10190875695164618, |
| "khm_Khmr": 0.1593273479867589, |
| "kor_Hang": 0.24812461710892444, |
| "lit_Latn": 0.44872328247117094, |
| "lvs_Latn": 0.45462607107025493, |
| "mal_Mlym": 0.24630664840221128, |
| "mar_Deva": -0.07545268867603293, |
| "mkd_Cyrl": 0.30128833130178045, |
| "mya_Mymr": 0.09942915864528967, |
| "nld_Latn": 0.6984457525965199, |
| "nob_Latn": 0.6136533102913947, |
| "npi_Deva": 0.1824949802899655, |
| "pan_Guru": 0.2549241058903939, |
| "pol_Latn": 0.5444900316379813, |
| "por_Latn": 0.6894638747639885, |
| "ron_Latn": 0.5337841174706431, |
| "rus_Cyrl": 0.28276913928432285, |
| "sin_Sinh": 0.36221079195463884, |
| "slk_Latn": 0.5251382460522451, |
| "slv_Latn": 0.4324206374112496, |
| "spa_Latn": 0.6301397510276678, |
| "srp_Cyrl": 0.2410437691136725, |
| "swe_Latn": 0.626292385430151, |
| "swh_Latn": 0.3046579614341224, |
| "tam_Taml": 0.1746132945264641, |
| "tel_Telu": 0.2719895557887898, |
| "tha_Thai": 0.18303644114058565, |
| "tur_Latn": 0.5260531985897241, |
| "ukr_Cyrl": 0.2415192086179947, |
| "urd_Arab": 0.09887985837154838, |
| "uzn_Latn": 0.43560234566036277, |
| "vie_Latn": 0.3548488051997094, |
| "yue_Hani": 0.1843154325260853, |
| "zsm_Latn": 0.63176129857042 |
| }, |
| "lang_mean_spearman": 0.3391411135846501 |
| } |
| }, |
| "lapa-llm/fineweb-nemotron-edu-score": { |
| "en_llama3": { |
| "n": 6000, |
| "spearman": 0.40435173649634193, |
| "pearson": 0.4554141865972126, |
| "f1_ge3": 0.0, |
| "acc_round": 0.12883333333333333, |
| "mean_pred": 0.1276990892092387 |
| }, |
| "fw2_gemma": { |
| "n": 9750, |
| "spearman": 0.5349173487614419, |
| "pearson": 0.562077935847803, |
| "f1_ge3": 0.0, |
| "acc_round": 0.4177435897435897, |
| "mean_pred": 0.19270957252306817, |
| "by_lang": { |
| "afr_Latn": 0.48389042201036964, |
| "amh_Ethi": 0.3822232362514379, |
| "arb_Arab": 0.6781892347769921, |
| "azj_Latn": 0.38443635733657094, |
| "ben_Beng": 0.5281859315473573, |
| "bul_Cyrl": 0.48416152951960584, |
| "cat_Latn": 0.6253514204117694, |
| "ces_Latn": 0.5902868972407341, |
| "cmn_Hani": 0.6736600224465833, |
| "dan_Latn": 0.6767123462816199, |
| "deu_Latn": 0.46482435883639295, |
| "ekk_Latn": 0.5720293642062684, |
| "ell_Grek": 0.5321063476860347, |
| "eus_Latn": 0.6167297891091834, |
| "fas_Arab": 0.6654732838260431, |
| "fil_Latn": 0.3046322173530426, |
| "fin_Latn": 0.4976454311956763, |
| "fra_Latn": 0.4741827081021615, |
| "glg_Latn": 0.4788506407702441, |
| "guj_Gujr": 0.5375404269294075, |
| "heb_Hebr": 0.6232003990837884, |
| "hin_Deva": 0.4683015882094025, |
| "hrv_Latn": 0.6515306767013056, |
| "hun_Latn": 0.5487119820727876, |
| "hye_Armn": 0.5343386111543995, |
| "ind_Latn": 0.6399088599480961, |
| "isl_Latn": 0.46881074934543976, |
| "ita_Latn": 0.49199121937179136, |
| "jpn_Jpan": 0.5072588635254016, |
| "kan_Knda": 0.5728007384491273, |
| "kat_Geor": 0.3114576309330108, |
| "kaz_Cyrl": 0.6849515903763737, |
| "khm_Khmr": 0.5585335415841745, |
| "kor_Hang": 0.518389228774535, |
| "lit_Latn": 0.48353769727687057, |
| "lvs_Latn": 0.46027063501806825, |
| "mal_Mlym": 0.5056591355491, |
| "mar_Deva": 0.6576983043650662, |
| "mkd_Cyrl": 0.6161256356836985, |
| "mya_Mymr": 0.439066691648072, |
| "nld_Latn": 0.5930061490179646, |
| "nob_Latn": 0.6455107316809844, |
| "npi_Deva": 0.342032804852713, |
| "pan_Guru": 0.7212425124428427, |
| "pol_Latn": 0.644092344107793, |
| "por_Latn": 0.4978996668679927, |
| "ron_Latn": 0.5688079965194102, |
| "rus_Cyrl": 0.5835212681138919, |
| "sin_Sinh": 0.5163239082658436, |
| "slk_Latn": 0.3903792269394212, |
| "slv_Latn": 0.5715113878665266, |
| "spa_Latn": 0.5381107050499219, |
| "srp_Cyrl": 0.5831102400119055, |
| "swe_Latn": 0.44876467976597684, |
| "swh_Latn": 0.4479382379994304, |
| "tam_Taml": 0.629789225785611, |
| "tel_Telu": 0.678583433741731, |
| "tha_Thai": 0.5951537073862054, |
| "tur_Latn": 0.745334937300227, |
| "ukr_Cyrl": 0.6728853453790729, |
| "urd_Arab": 0.40202281132559226, |
| "uzn_Latn": 0.6915804088792905, |
| "vie_Latn": 0.5662923896778848, |
| "yue_Hani": 0.4409938684986199, |
| "zsm_Latn": 0.6690450331407745 |
| }, |
| "lang_mean_spearman": 0.5473475194696251 |
| } |
| }, |
| "hotchpotch/fineweb-2-edu-japanese-classifier": { |
| "en_llama3": { |
| "n": 6000, |
| "spearman": 0.5921578182237801, |
| "pearson": 0.5077444923946944, |
| "f1_ge3": 0.17989417989417988, |
| "acc_round": 0.07183333333333333, |
| "mean_pred": 2.7901276041666665 |
| }, |
| "fw2_gemma": { |
| "n": 9750, |
| "spearman": 0.5499382217361815, |
| "pearson": 0.4203910799363762, |
| "f1_ge3": 0.21786941580756014, |
| "acc_round": 0.09753846153846153, |
| "mean_pred": 2.7127336738782053, |
| "by_lang": { |
| "afr_Latn": 0.5116149862577334, |
| "amh_Ethi": 0.5064297055669538, |
| "arb_Arab": 0.5099635720859134, |
| "azj_Latn": 0.47582135322997676, |
| "ben_Beng": 0.629888278412506, |
| "bul_Cyrl": 0.5763550433470112, |
| "cat_Latn": 0.5371870739031886, |
| "ces_Latn": 0.6359393476583343, |
| "cmn_Hani": 0.7934241923026797, |
| "dan_Latn": 0.5932684884828309, |
| "deu_Latn": 0.6200653239781251, |
| "ekk_Latn": 0.6441123351958554, |
| "ell_Grek": 0.5666255428792976, |
| "eus_Latn": 0.5705360935004703, |
| "fas_Arab": 0.6509585545132225, |
| "fil_Latn": 0.6806891768731289, |
| "fin_Latn": 0.6201138974162479, |
| "fra_Latn": 0.5822901397136012, |
| "glg_Latn": 0.4674084841492625, |
| "guj_Gujr": 0.47720265526183564, |
| "heb_Hebr": 0.47699062883410864, |
| "hin_Deva": 0.5098192382858678, |
| "hrv_Latn": 0.5136587261418535, |
| "hun_Latn": 0.6202557558487989, |
| "hye_Armn": 0.423231753717233, |
| "ind_Latn": 0.6207925246234596, |
| "isl_Latn": 0.4463491772204554, |
| "ita_Latn": 0.6264924596415915, |
| "jpn_Jpan": 0.7896990661983702, |
| "kan_Knda": 0.4984784498274562, |
| "kat_Geor": 0.48444127752189386, |
| "kaz_Cyrl": 0.3465341923924144, |
| "khm_Khmr": 0.6382914448173299, |
| "kor_Hang": 0.7989747457588643, |
| "lit_Latn": 0.560250793614278, |
| "lvs_Latn": 0.5238597331244348, |
| "mal_Mlym": 0.6142332586223039, |
| "mar_Deva": 0.5178096821445864, |
| "mkd_Cyrl": 0.6245294420022526, |
| "mya_Mymr": 0.6614666436063987, |
| "nld_Latn": 0.6589492341211373, |
| "nob_Latn": 0.5884598745905916, |
| "npi_Deva": 0.5057900274710645, |
| "pan_Guru": 0.48239238889222125, |
| "pol_Latn": 0.6431501842808603, |
| "por_Latn": 0.6774251610323179, |
| "ron_Latn": 0.5720068345901783, |
| "rus_Cyrl": 0.6598487846963305, |
| "sin_Sinh": 0.5681730728940162, |
| "slk_Latn": 0.5433020799375309, |
| "slv_Latn": 0.5532141047294149, |
| "spa_Latn": 0.6671276675142975, |
| "srp_Cyrl": 0.3244343644279143, |
| "swe_Latn": 0.6699832323601396, |
| "swh_Latn": 0.4209064301683013, |
| "tam_Taml": 0.5554827292258321, |
| "tel_Telu": 0.5806660381234184, |
| "tha_Thai": 0.7152391004088681, |
| "tur_Latn": 0.7131602490856848, |
| "ukr_Cyrl": 0.5066338460868683, |
| "urd_Arab": 0.49444220274143674, |
| "uzn_Latn": 0.42703751159186915, |
| "vie_Latn": 0.577798064164233, |
| "yue_Hani": 0.663907562897532, |
| "zsm_Latn": 0.683626270718332 |
| }, |
| "lang_mean_spearman": 0.5753724654680388 |
| } |
| } |
| } |
