Upload full run contents (scripts, checkpoints, and results) (part 12)

.gitattributes

@@ -37,3 +37,6 @@ babylm_eval_repo/multilingual/meco/data/meco_l1/wave1_joint_l1_data_trimmed_vers
 babylm_eval_repo/multilingual/meco/data/meco_l1/wave2_joint_data_trimmed_wave2_version2.1.rda filter=lfs diff=lfs merge=lfs -text
 babylm_eval_repo/multilingual/meco/data/meco_l2/joint_data_l2_trimmed_version2.2.rda filter=lfs diff=lfs merge=lfs -text
 babylm_eval_repo/multilingual/meco/data/meco_l2/joint_data_trimmed_L2_wave2_version2.2.rda filter=lfs diff=lfs merge=lfs -text
+babylm_eval_repo/strict/assets/babylm.png filter=lfs diff=lfs merge=lfs -text
+babylm_eval_repo/strict/evaluation_data/full_eval/comps/comps_base.jsonl filter=lfs diff=lfs merge=lfs -text
+babylm_eval_repo/strict/evaluation_data/full_eval/glue_filtered/multirc.train.jsonl filter=lfs diff=lfs merge=lfs -text

babylm_eval_repo/multilingual/tasks/sib200/sib200_slv_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_slv_Latn
+dataset_name: slv_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_smo_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_smo_Latn
+dataset_name: smo_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_sna_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_sna_Latn
+dataset_name: sna_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_snd_Arab.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_snd_Arab
+dataset_name: snd_Arab

babylm_eval_repo/multilingual/tasks/sib200/sib200_som_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_som_Latn
+dataset_name: som_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_sot_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_sot_Latn
+dataset_name: sot_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_spa_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_spa_Latn
+dataset_name: spa_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_srd_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_srd_Latn
+dataset_name: srd_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_srp_Cyrl.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_srp_Cyrl
+dataset_name: srp_Cyrl

babylm_eval_repo/multilingual/tasks/sib200/sib200_ssw_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_ssw_Latn
+dataset_name: ssw_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_sun_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_sun_Latn
+dataset_name: sun_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_swe_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_swe_Latn
+dataset_name: swe_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_swh_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_swh_Latn
+dataset_name: swh_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_szl_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_szl_Latn
+dataset_name: szl_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_tam_Taml.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_tam_Taml
+dataset_name: tam_Taml

babylm_eval_repo/multilingual/tasks/sib200/sib200_taq_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_taq_Latn
+dataset_name: taq_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_taq_Tfng.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_taq_Tfng
+dataset_name: taq_Tfng

babylm_eval_repo/multilingual/tasks/sib200/sib200_tat_Cyrl.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_tat_Cyrl
+dataset_name: tat_Cyrl

babylm_eval_repo/multilingual/tasks/sib200/sib200_tel_Telu.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_tel_Telu
+dataset_name: tel_Telu

babylm_eval_repo/multilingual/tasks/sib200/sib200_tgk_Cyrl.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_tgk_Cyrl
+dataset_name: tgk_Cyrl

babylm_eval_repo/multilingual/tasks/sib200/sib200_tgl_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_tgl_Latn
+dataset_name: tgl_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_tha_Thai.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_tha_Thai
+dataset_name: tha_Thai

babylm_eval_repo/multilingual/tasks/sib200/sib200_tir_Ethi.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_tir_Ethi
+dataset_name: tir_Ethi

babylm_eval_repo/multilingual/tasks/sib200/sib200_tpi_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_tpi_Latn
+dataset_name: tpi_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_tsn_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_tsn_Latn
+dataset_name: tsn_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_tso_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_tso_Latn
+dataset_name: tso_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_tuk_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_tuk_Latn
+dataset_name: tuk_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_tum_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_tum_Latn
+dataset_name: tum_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_tur_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_tur_Latn
+dataset_name: tur_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_twi_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_twi_Latn
+dataset_name: twi_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_tzm_Tfng.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_tzm_Tfng
+dataset_name: tzm_Tfng

babylm_eval_repo/multilingual/tasks/sib200/sib200_uig_Arab.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_uig_Arab
+dataset_name: uig_Arab

babylm_eval_repo/multilingual/tasks/sib200/sib200_ukr_Cyrl.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_ukr_Cyrl
+dataset_name: ukr_Cyrl

babylm_eval_repo/multilingual/tasks/sib200/sib200_umb_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_umb_Latn
+dataset_name: umb_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_urd_Arab.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_urd_Arab
+dataset_name: urd_Arab

babylm_eval_repo/multilingual/tasks/sib200/sib200_uzn_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_uzn_Latn
+dataset_name: uzn_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_vec_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_vec_Latn
+dataset_name: vec_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_vie_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_vie_Latn
+dataset_name: vie_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_war_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_war_Latn
+dataset_name: war_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_wol_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_wol_Latn
+dataset_name: wol_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_xho_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_xho_Latn
+dataset_name: xho_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_ydd_Hebr.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_ydd_Hebr
+dataset_name: ydd_Hebr

babylm_eval_repo/multilingual/tasks/sib200/sib200_yor_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_yor_Latn
+dataset_name: yor_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_yue_Hant.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_yue_Hant
+dataset_name: yue_Hant

babylm_eval_repo/multilingual/tasks/sib200/sib200_zho_Hans.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_zho_Hans
+dataset_name: zho_Hans

babylm_eval_repo/multilingual/tasks/sib200/sib200_zho_Hant.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_zho_Hant
+dataset_name: zho_Hant

babylm_eval_repo/multilingual/tasks/sib200/sib200_zsm_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_zsm_Latn
+dataset_name: zsm_Latn

babylm_eval_repo/multilingual/tasks/sib200/sib200_zul_Latn.yaml

@@ -0,0 +1,3 @@
+include: sib200_eng_Latn.yaml
+task: sib200_zul_Latn
+dataset_name: zul_Latn

babylm_eval_repo/multilingual/tasks/snli_mubench/README.md

@@ -0,0 +1,108 @@
+# Snli
+
+### Paper
+
+Title: `MuBench: Assessment of Multilingual Capabilities of Large Language Models Across 61 Languages`
+
+Abstract: Multilingual large language models (LLMs) are advancing rapidly, with new models frequently claiming support for an increasing number of languages. However, existing evaluation datasets are limited and lack cross-lingual alignment, leaving assessments of multilingual capabilities fragmented in both language and skill coverage. To address this, we introduce MuBench, a benchmark covering 61 languages and evaluating a broad range of capabilities. We evaluate several state-of-the-art multilingual LLMs and find notable gaps between claimed and actual language coverage, particularly a persistent performance disparity between English and low-resource languages. Leveraging MuBench's alignment, we propose Multilingual Consistency (MLC) as a complementary metric to accuracy for analyzing performance bottlenecks and guiding model improvement. Finally, we pretrain a suite of 1.2B-parameter models on English and Chinese with 500B tokens, varying language ratios and parallel data proportions to investigate cross-lingual transfer dynamics.
+
+Homepage: https://huggingface.co/datasets/aialt/MuBench
+
+
+### Citation
+
+```
+@misc{han2025mubench,
+      title={MuBench: Assessment of Multilingual Capabilities of Large Language Models Across 61 Languages}, 
+      author={Wenhan Han and Yifan Zhang and Zhixun Chen and Binbin Liu and Haobin Lin and Bingni Zhang and Taifeng Wang and Mykola Pechenizkiy and Meng Fang and Yin Zheng},
+      year={2025},
+      eprint={2506.19468},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2506.19468}, 
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+* `snli_mubench`
+
+#### Tasks
+* `snli_af_mubench`: Afrikaans
+* `snli_ar_mubench`: Arabic
+* `snli_az_mubench`: Azerbaijani
+* `snli_bg_mubench`: Bulgarian
+* `snli_bn_mubench`: Bengali
+* `snli_ca_mubench`: Catalan
+* `snli_ceb_mubench`: Cebuano
+* `snli_cs_mubench`: Czech
+* `snli_da_mubench`: Danish
+* `snli_de_mubench`: German
+* `snli_el_mubench`: Greek
+* `snli_en_mubench`: English
+* `snli_es_mubench`: Spanish
+* `snli_et_mubench`: Estonian
+* `snli_fa_mubench`: Persian
+* `snli_fi_mubench`: Finnish
+* `snli_fr_mubench`: French
+* `snli_ga_mubench`: Irish
+* `snli_gu_mubench`: Gujarati
+* `snli_he_mubench`: Hebrew
+* `snli_hi_mubench`: Hindi
+* `snli_hr_mubench`: Croatian
+* `snli_hu_mubench`: Hungarian
+* `snli_id_mubench`: Indonesian
+* `snli_is_mubench`: Icelandic
+* `snli_it_mubench`: Italian
+* `snli_ja_mubench`: Japanese
+* `snli_jv_mubench`: Javanese
+* `snli_kk_mubench`: Kazakh
+* `snli_km_mubench`: Khmer
+* `snli_kn_mubench`: Kannada
+* `snli_ko_mubench`: Korean
+* `snli_lt_mubench`: Lithuanian
+* `snli_lv_mubench`: Latvian
+* `snli_ml_mubench`: Malayalam
+* `snli_mr_mubench`: Marathi
+* `snli_ms_mubench`: Malay
+* `snli_my_mubench`: Myanmar
+* `snli_nl_mubench`: Dutch
+* `snli_no_mubench`: Norwegian
+* `snli_pa_mubench`: Punjabi
+* `snli_pl_mubench`: Polish
+* `snli_pt_mubench`: Portuguese
+* `snli_ro_mubench`: Romanian
+* `snli_ru_mubench`: Russian
+* `snli_sk_mubench`: Slovak
+* `snli_sl_mubench`: Slovenian
+* `snli_sq_mubench`: Albanian
+* `snli_sr_mubench`: Serbian
+* `snli_sv_mubench`: Swedish
+* `snli_sw_mubench`: Swahili
+* `snli_ta_mubench`: Tamil
+* `snli_te_mubench`: Telugu
+* `snli_th_mubench`: Thai
+* `snli_tl_mubench`: Filipino
+* `snli_tr_mubench`: Turkish
+* `snli_uk_mubench`: Ukrainian
+* `snli_ur_mubench`: Urdu
+* `snli_uz_mubench`: Uzbek
+* `snli_vi_mubench`: Vietnamese
+* `snli_zh_mubench`: Chinese
+
+
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [ ] Is the task an existing benchmark in the literature?
+  * [ ] Have you referenced the original paper that introduced the task?
+  * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?