eval/baselines.json

{
 "HuggingFaceFW/fineweb-edu-classifier": {
  "en_llama3": {
   "n": 6000,
   "spearman": 0.7811745959348265,
   "pearson": 0.8254685326164869,
   "f1_ge3": 0.7167381974248928,
   "acc_round": 0.7331666666666666,
   "mean_pred": 1.303708251953125
  },
  "fw2_gemma": {
   "n": 9750,
   "spearman": 0.31332828330222384,
   "pearson": 0.3201400964584369,
   "f1_ge3": 0.014141414141414142,
   "acc_round": 0.4175384615384615,
   "mean_pred": 0.7903328074919872,
   "by_lang": {
    "afr_Latn": 0.4783886151902705,
    "amh_Ethi": 0.0035800511195597565,
    "arb_Arab": 0.18433856457178072,
    "azj_Latn": 0.09337074285835445,
    "ben_Beng": -0.09760323618540227,
    "bul_Cyrl": 0.20503335722132798,
    "cat_Latn": 0.6595302771997552,
    "ces_Latn": 0.3599752431781002,
    "cmn_Hani": 0.36116549630361733,
    "dan_Latn": 0.6001897248450279,
    "deu_Latn": 0.6149203082616248,
    "ekk_Latn": 0.3708713726345026,
    "ell_Grek": 0.2641394141294525,
    "eus_Latn": 0.3617396078726179,
    "fas_Arab": 0.25725247538293716,
    "fil_Latn": 0.5192007594329661,
    "fin_Latn": 0.4594319524209811,
    "fra_Latn": 0.7030699852374831,
    "glg_Latn": 0.46222411627307064,
    "guj_Gujr": 0.17410758233215715,
    "heb_Hebr": 0.2552741558144449,
    "hin_Deva": 0.21684813093532157,
    "hrv_Latn": 0.3901958638082959,
    "hun_Latn": 0.4679083623541295,
    "hye_Armn": 0.17378206897576237,
    "ind_Latn": 0.5425439672915935,
    "isl_Latn": 0.36410277837656496,
    "ita_Latn": 0.7114280686997687,
    "jpn_Jpan": 0.35637333571825525,
    "kan_Knda": 0.2159835575450683,
    "kat_Geor": 0.20181702902943727,
    "kaz_Cyrl": 0.07028438701894076,
    "khm_Khmr": 0.19854930856895855,
    "kor_Hang": 0.36423478651136526,
    "lit_Latn": 0.44426372175382817,
    "lvs_Latn": 0.42898220014392424,
    "mal_Mlym": 0.254422574383917,
    "mar_Deva": -0.02052601043908013,
    "mkd_Cyrl": 0.17931674439965872,
    "mya_Mymr": 0.060353023381646796,
    "nld_Latn": 0.7176838890206896,
    "nob_Latn": 0.6039381224420028,
    "npi_Deva": 0.24364382637822735,
    "pan_Guru": 0.3141409775437518,
    "pol_Latn": 0.5030397252962449,
    "por_Latn": 0.6909333335761909,
    "ron_Latn": 0.5864636995970052,
    "rus_Cyrl": 0.2812129159232862,
    "sin_Sinh": 0.3592815174301788,
    "slk_Latn": 0.4598330329453881,
    "slv_Latn": 0.5057813338679621,
    "spa_Latn": 0.6448157099816524,
    "srp_Cyrl": 0.23893350817449613,
    "swe_Latn": 0.6506862369110045,
    "swh_Latn": 0.25299473872794553,
    "tam_Taml": 0.22364087822978565,
    "tel_Telu": 0.1593746895000516,
    "tha_Thai": 0.25103745018468,
    "tur_Latn": 0.5612733688350385,
    "ukr_Cyrl": 0.2528674145533893,
    "urd_Arab": 0.15879523016824826,
    "uzn_Latn": 0.5122183422461761,
    "vie_Latn": 0.23123533417387576,
    "yue_Hani": 0.13726957886753305,
    "zsm_Latn": 0.6246551025461468
   },
   "lang_mean_spearman": 0.3550279141795836
  }
 },
 "nvidia/nemocurator-fineweb-nemotron-4-edu-classifier": {
  "en_llama3": {
   "n": 6000,
   "spearman": 0.6900049364432077,
   "pearson": 0.6947466119935968,
   "f1_ge3": 0.5593952483801296,
   "acc_round": 0.5658333333333333,
   "mean_pred": 1.387032918294271
  },
  "fw2_gemma": {
   "n": 9750,
   "spearman": 0.2895303544363772,
   "pearson": 0.2906974159907195,
   "f1_ge3": 0.004081632653061225,
   "acc_round": 0.45066666666666666,
   "mean_pred": 0.640254885942508,
   "by_lang": {
    "afr_Latn": 0.4125513088566815,
    "amh_Ethi": 0.018373213098583384,
    "arb_Arab": 0.0462338281457406,
    "azj_Latn": 0.019992976790920432,
    "ben_Beng": -0.16789892750113863,
    "bul_Cyrl": 0.2971403726411454,
    "cat_Latn": 0.6147170553906413,
    "ces_Latn": 0.39892159831065555,
    "cmn_Hani": 0.33236056343520864,
    "dan_Latn": 0.627986710030984,
    "deu_Latn": 0.5421583888861886,
    "ekk_Latn": 0.3412602251624262,
    "ell_Grek": 0.2877695026495254,
    "eus_Latn": 0.26273050172930285,
    "fas_Arab": 0.24343030252432032,
    "fil_Latn": 0.46762500797401985,
    "fin_Latn": 0.5002472399939247,
    "fra_Latn": 0.6700508212975463,
    "glg_Latn": 0.5633083561358194,
    "guj_Gujr": 0.14136444259530326,
    "heb_Hebr": 0.04724066128936813,
    "hin_Deva": 0.22134439322375482,
    "hrv_Latn": 0.3966049006476282,
    "hun_Latn": 0.45954093494157644,
    "hye_Armn": 0.16936739322102382,
    "ind_Latn": 0.5748003182538665,
    "isl_Latn": 0.30026748669376563,
    "ita_Latn": 0.6741036997984878,
    "jpn_Jpan": 0.3174746581217568,
    "kan_Knda": 0.023717881864377573,
    "kat_Geor": 0.21079149629341118,
    "kaz_Cyrl": 0.033985861656622506,
    "khm_Khmr": 0.1334958353366269,
    "kor_Hang": 0.3697988953836682,
    "lit_Latn": 0.5016548732038005,
    "lvs_Latn": 0.42824583647664066,
    "mal_Mlym": 0.22060159988132563,
    "mar_Deva": 0.012161465310395518,
    "mkd_Cyrl": 0.2016088441163244,
    "mya_Mymr": 0.04140935967384132,
    "nld_Latn": 0.6911094076936222,
    "nob_Latn": 0.6496771853598793,
    "npi_Deva": 0.23923633067121167,
    "pan_Guru": 0.31902012692687165,
    "pol_Latn": 0.5516762448979281,
    "por_Latn": 0.6629914217097714,
    "ron_Latn": 0.5406291022067823,
    "rus_Cyrl": 0.2241661254766134,
    "sin_Sinh": 0.43543983599659747,
    "slk_Latn": 0.4690493972306032,
    "slv_Latn": 0.42703517488655535,
    "spa_Latn": 0.6779816003990441,
    "srp_Cyrl": 0.25743830692710934,
    "swe_Latn": 0.6697583783089793,
    "swh_Latn": 0.2949237410105869,
    "tam_Taml": 0.2864941876005439,
    "tel_Telu": 0.24945242629027456,
    "tha_Thai": 0.23070699677653075,
    "tur_Latn": 0.5232767110609888,
    "ukr_Cyrl": 0.17294253430222284,
    "urd_Arab": 0.09084790133296561,
    "uzn_Latn": 0.36802535578818296,
    "vie_Latn": 0.3066835352546123,
    "yue_Hani": 0.2059018457197195,
    "zsm_Latn": 0.6224283932948171
   },
   "lang_mean_spearman": 0.34039126385629354
  }
 },
 "nvidia/nemocurator-fineweb-mixtral-edu-classifier": {
  "en_llama3": {
   "n": 6000,
   "spearman": 0.7175490136043337,
   "pearson": 0.7287184005488518,
   "f1_ge3": 0.614963503649635,
   "acc_round": 0.6028333333333333,
   "mean_pred": 1.3429691162109374
  },
  "fw2_gemma": {
   "n": 9750,
   "spearman": 0.2819134647436298,
   "pearson": 0.28562988667839534,
   "f1_ge3": 0.002044989775051125,
   "acc_round": 0.45374358974358975,
   "mean_pred": 0.47195639272836537,
   "by_lang": {
    "afr_Latn": 0.47159183192913195,
    "amh_Ethi": 0.019598498538088625,
    "arb_Arab": 0.06822991834984035,
    "azj_Latn": 0.06953770032467291,
    "ben_Beng": -0.03358448985760162,
    "bul_Cyrl": 0.27245858557578745,
    "cat_Latn": 0.5538705546867907,
    "ces_Latn": 0.35157906715040116,
    "cmn_Hani": 0.3077231124197137,
    "dan_Latn": 0.6222795229134043,
    "deu_Latn": 0.6118166183216837,
    "ekk_Latn": 0.40074372684756604,
    "ell_Grek": 0.19232566498760245,
    "eus_Latn": 0.3155052398487781,
    "fas_Arab": 0.25391343265204913,
    "fil_Latn": 0.4646692271862731,
    "fin_Latn": 0.4759450597456664,
    "fra_Latn": 0.663792113652442,
    "glg_Latn": 0.5322872358954756,
    "guj_Gujr": 0.20596184394454778,
    "heb_Hebr": 0.10716405578859328,
    "hin_Deva": 0.1838808075368145,
    "hrv_Latn": 0.3139358872501065,
    "hun_Latn": 0.5081555792186178,
    "hye_Armn": 0.12716098990970462,
    "ind_Latn": 0.5621676785379013,
    "isl_Latn": 0.24641095943858662,
    "ita_Latn": 0.658190727051437,
    "jpn_Jpan": 0.2598441397658701,
    "kan_Knda": 0.17806570191716214,
    "kat_Geor": 0.0701215725725029,
    "kaz_Cyrl": 0.10190875695164618,
    "khm_Khmr": 0.1593273479867589,
    "kor_Hang": 0.24812461710892444,
    "lit_Latn": 0.44872328247117094,
    "lvs_Latn": 0.45462607107025493,
    "mal_Mlym": 0.24630664840221128,
    "mar_Deva": -0.07545268867603293,
    "mkd_Cyrl": 0.30128833130178045,
    "mya_Mymr": 0.09942915864528967,
    "nld_Latn": 0.6984457525965199,
    "nob_Latn": 0.6136533102913947,
    "npi_Deva": 0.1824949802899655,
    "pan_Guru": 0.2549241058903939,
    "pol_Latn": 0.5444900316379813,
    "por_Latn": 0.6894638747639885,
    "ron_Latn": 0.5337841174706431,
    "rus_Cyrl": 0.28276913928432285,
    "sin_Sinh": 0.36221079195463884,
    "slk_Latn": 0.5251382460522451,
    "slv_Latn": 0.4324206374112496,
    "spa_Latn": 0.6301397510276678,
    "srp_Cyrl": 0.2410437691136725,
    "swe_Latn": 0.626292385430151,
    "swh_Latn": 0.3046579614341224,
    "tam_Taml": 0.1746132945264641,
    "tel_Telu": 0.2719895557887898,
    "tha_Thai": 0.18303644114058565,
    "tur_Latn": 0.5260531985897241,
    "ukr_Cyrl": 0.2415192086179947,
    "urd_Arab": 0.09887985837154838,
    "uzn_Latn": 0.43560234566036277,
    "vie_Latn": 0.3548488051997094,
    "yue_Hani": 0.1843154325260853,
    "zsm_Latn": 0.63176129857042
   },
   "lang_mean_spearman": 0.3391411135846501
  }
 },
 "lapa-llm/fineweb-nemotron-edu-score": {
  "en_llama3": {
   "n": 6000,
   "spearman": 0.40435173649634193,
   "pearson": 0.4554141865972126,
   "f1_ge3": 0.0,
   "acc_round": 0.12883333333333333,
   "mean_pred": 0.1276990892092387
  },
  "fw2_gemma": {
   "n": 9750,
   "spearman": 0.5349173487614419,
   "pearson": 0.562077935847803,
   "f1_ge3": 0.0,
   "acc_round": 0.4177435897435897,
   "mean_pred": 0.19270957252306817,
   "by_lang": {
    "afr_Latn": 0.48389042201036964,
    "amh_Ethi": 0.3822232362514379,
    "arb_Arab": 0.6781892347769921,
    "azj_Latn": 0.38443635733657094,
    "ben_Beng": 0.5281859315473573,
    "bul_Cyrl": 0.48416152951960584,
    "cat_Latn": 0.6253514204117694,
    "ces_Latn": 0.5902868972407341,
    "cmn_Hani": 0.6736600224465833,
    "dan_Latn": 0.6767123462816199,
    "deu_Latn": 0.46482435883639295,
    "ekk_Latn": 0.5720293642062684,
    "ell_Grek": 0.5321063476860347,
    "eus_Latn": 0.6167297891091834,
    "fas_Arab": 0.6654732838260431,
    "fil_Latn": 0.3046322173530426,
    "fin_Latn": 0.4976454311956763,
    "fra_Latn": 0.4741827081021615,
    "glg_Latn": 0.4788506407702441,
    "guj_Gujr": 0.5375404269294075,
    "heb_Hebr": 0.6232003990837884,
    "hin_Deva": 0.4683015882094025,
    "hrv_Latn": 0.6515306767013056,
    "hun_Latn": 0.5487119820727876,
    "hye_Armn": 0.5343386111543995,
    "ind_Latn": 0.6399088599480961,
    "isl_Latn": 0.46881074934543976,
    "ita_Latn": 0.49199121937179136,
    "jpn_Jpan": 0.5072588635254016,
    "kan_Knda": 0.5728007384491273,
    "kat_Geor": 0.3114576309330108,
    "kaz_Cyrl": 0.6849515903763737,
    "khm_Khmr": 0.5585335415841745,
    "kor_Hang": 0.518389228774535,
    "lit_Latn": 0.48353769727687057,
    "lvs_Latn": 0.46027063501806825,
    "mal_Mlym": 0.5056591355491,
    "mar_Deva": 0.6576983043650662,
    "mkd_Cyrl": 0.6161256356836985,
    "mya_Mymr": 0.439066691648072,
    "nld_Latn": 0.5930061490179646,
    "nob_Latn": 0.6455107316809844,
    "npi_Deva": 0.342032804852713,
    "pan_Guru": 0.7212425124428427,
    "pol_Latn": 0.644092344107793,
    "por_Latn": 0.4978996668679927,
    "ron_Latn": 0.5688079965194102,
    "rus_Cyrl": 0.5835212681138919,
    "sin_Sinh": 0.5163239082658436,
    "slk_Latn": 0.3903792269394212,
    "slv_Latn": 0.5715113878665266,
    "spa_Latn": 0.5381107050499219,
    "srp_Cyrl": 0.5831102400119055,
    "swe_Latn": 0.44876467976597684,
    "swh_Latn": 0.4479382379994304,
    "tam_Taml": 0.629789225785611,
    "tel_Telu": 0.678583433741731,
    "tha_Thai": 0.5951537073862054,
    "tur_Latn": 0.745334937300227,
    "ukr_Cyrl": 0.6728853453790729,
    "urd_Arab": 0.40202281132559226,
    "uzn_Latn": 0.6915804088792905,
    "vie_Latn": 0.5662923896778848,
    "yue_Hani": 0.4409938684986199,
    "zsm_Latn": 0.6690450331407745
   },
   "lang_mean_spearman": 0.5473475194696251
  }
 },
 "hotchpotch/fineweb-2-edu-japanese-classifier": {
  "en_llama3": {
   "n": 6000,
   "spearman": 0.5921578182237801,
   "pearson": 0.5077444923946944,
   "f1_ge3": 0.17989417989417988,
   "acc_round": 0.07183333333333333,
   "mean_pred": 2.7901276041666665
  },
  "fw2_gemma": {
   "n": 9750,
   "spearman": 0.5499382217361815,
   "pearson": 0.4203910799363762,
   "f1_ge3": 0.21786941580756014,
   "acc_round": 0.09753846153846153,
   "mean_pred": 2.7127336738782053,
   "by_lang": {
    "afr_Latn": 0.5116149862577334,
    "amh_Ethi": 0.5064297055669538,
    "arb_Arab": 0.5099635720859134,
    "azj_Latn": 0.47582135322997676,
    "ben_Beng": 0.629888278412506,
    "bul_Cyrl": 0.5763550433470112,
    "cat_Latn": 0.5371870739031886,
    "ces_Latn": 0.6359393476583343,
    "cmn_Hani": 0.7934241923026797,
    "dan_Latn": 0.5932684884828309,
    "deu_Latn": 0.6200653239781251,
    "ekk_Latn": 0.6441123351958554,
    "ell_Grek": 0.5666255428792976,
    "eus_Latn": 0.5705360935004703,
    "fas_Arab": 0.6509585545132225,
    "fil_Latn": 0.6806891768731289,
    "fin_Latn": 0.6201138974162479,
    "fra_Latn": 0.5822901397136012,
    "glg_Latn": 0.4674084841492625,
    "guj_Gujr": 0.47720265526183564,
    "heb_Hebr": 0.47699062883410864,
    "hin_Deva": 0.5098192382858678,
    "hrv_Latn": 0.5136587261418535,
    "hun_Latn": 0.6202557558487989,
    "hye_Armn": 0.423231753717233,
    "ind_Latn": 0.6207925246234596,
    "isl_Latn": 0.4463491772204554,
    "ita_Latn": 0.6264924596415915,
    "jpn_Jpan": 0.7896990661983702,
    "kan_Knda": 0.4984784498274562,
    "kat_Geor": 0.48444127752189386,
    "kaz_Cyrl": 0.3465341923924144,
    "khm_Khmr": 0.6382914448173299,
    "kor_Hang": 0.7989747457588643,
    "lit_Latn": 0.560250793614278,
    "lvs_Latn": 0.5238597331244348,
    "mal_Mlym": 0.6142332586223039,
    "mar_Deva": 0.5178096821445864,
    "mkd_Cyrl": 0.6245294420022526,
    "mya_Mymr": 0.6614666436063987,
    "nld_Latn": 0.6589492341211373,
    "nob_Latn": 0.5884598745905916,
    "npi_Deva": 0.5057900274710645,
    "pan_Guru": 0.48239238889222125,
    "pol_Latn": 0.6431501842808603,
    "por_Latn": 0.6774251610323179,
    "ron_Latn": 0.5720068345901783,
    "rus_Cyrl": 0.6598487846963305,
    "sin_Sinh": 0.5681730728940162,
    "slk_Latn": 0.5433020799375309,
    "slv_Latn": 0.5532141047294149,
    "spa_Latn": 0.6671276675142975,
    "srp_Cyrl": 0.3244343644279143,
    "swe_Latn": 0.6699832323601396,
    "swh_Latn": 0.4209064301683013,
    "tam_Taml": 0.5554827292258321,
    "tel_Telu": 0.5806660381234184,
    "tha_Thai": 0.7152391004088681,
    "tur_Latn": 0.7131602490856848,
    "ukr_Cyrl": 0.5066338460868683,
    "urd_Arab": 0.49444220274143674,
    "uzn_Latn": 0.42703751159186915,
    "vie_Latn": 0.577798064164233,
    "yue_Hani": 0.663907562897532,
    "zsm_Latn": 0.683626270718332
   },
   "lang_mean_spearman": 0.5753724654680388
  }
 }
}