| { |
| "generated": "2026-09-15 06:44 UTC", |
| "letter_position": { |
| "model": "Qwen3.5-2B", |
| "wrong_milu_answers": 4822, |
| "most_chosen_letter": "C", |
| "times": 4163, |
| "mmlu_chat_before": 0.2525, |
| "mmlu_raw": 0.49, |
| "milu_chat_before": 0.2433, |
| "mmlu_chat_after": 0.26, |
| "milu_chat_after": 0.242 |
| }, |
| "sdpa": { |
| "model": "Gemma-3-1B", |
| "tasks": { |
| "in22gen_ta_en": { |
| "single": 39.0717, |
| "batched_sdpa": 36.4533, |
| "batched_eager": 39.529 |
| }, |
| "flores_ta_en": { |
| "single": 40.3854, |
| "batched_sdpa": 38.4472, |
| "batched_eager": 40.8063 |
| } |
| } |
| }, |
| "start_token": { |
| "material": [ |
| "Gemma-3-1B-it: dev mmlu_en 21.0 to 41.0, test milu_ta 26.8 to 31.4, test mmlu_en 23.5 to 39.0", |
| "Llama-3.2-1B-Instruct: dev milu_ta 26.0 to 28.0, dev mmlu_en 39.0 to 35.0, test mmlu_en 39.0 to 35.5", |
| "Sarvam-1: dev milu_ta 34.0 to 36.7, dev mmlu_en 43.0 to 45.0" |
| ], |
| "pending": [], |
| "gemma4_e4b_raw_dev_before": { |
| "flores_en_ta_chrf": 0.0346, |
| "flores_ta_en_chrf": 1.025, |
| "tamil_bpc": 5.1868, |
| "mmlu": 0.31 |
| } |
| }, |
| "preamble": { |
| "local_chat": [ |
| { |
| "model": "tamil-lm-2b-instruct-r4", |
| "share": 0.0, |
| "largest_gap": 0.0, |
| "task": "in22gen_ta_en", |
| "first": 48.8408, |
| "extracted": 48.8408 |
| }, |
| { |
| "model": "Qwen3.5-2B", |
| "share": 7.8, |
| "largest_gap": 0.4, |
| "task": "flores_en_ta", |
| "first": 27.9656, |
| "extracted": 28.4019 |
| }, |
| { |
| "model": "Qwen3.5-4B", |
| "share": 1.6, |
| "largest_gap": 0.2, |
| "task": "in22gen_en_ta", |
| "first": 35.8439, |
| "extracted": 36.0683 |
| }, |
| { |
| "model": "Gemma-3-1B-it", |
| "share": 55.0, |
| "largest_gap": 21.1, |
| "task": "flores_en_ta", |
| "first": 7.0102, |
| "extracted": 28.082 |
| }, |
| { |
| "model": "Llama-3.2-1B-Instruct", |
| "share": 0.3, |
| "largest_gap": 0.2, |
| "task": "in22gen_ta_en", |
| "first": 33.1526, |
| "extracted": 33.3336 |
| }, |
| { |
| "model": "Gemma-3-4B-it", |
| "share": 89.1, |
| "largest_gap": 44.8, |
| "task": "flores_en_ta", |
| "first": 1.3475, |
| "extracted": 46.1687 |
| }, |
| { |
| "model": "Sarvam-1", |
| "share": 0.6, |
| "largest_gap": 0.1, |
| "task": "in22gen_en_ta", |
| "first": 30.4741, |
| "extracted": 30.5594 |
| }, |
| { |
| "model": "Qwen3-1.7B-tamil-Instruct", |
| "share": 18.7, |
| "largest_gap": 1.2, |
| "task": "in22gen_en_ta", |
| "first": 15.264, |
| "extracted": 16.4391 |
| } |
| ], |
| "gemini": { |
| "share": 56.3, |
| "in22gen_ta_en_first": 24.1883, |
| "in22gen_ta_en_extracted": 59.448 |
| } |
| }, |
| "caps": { |
| "llama_flores_refs_over_160": 725, |
| "flores_items": 1012, |
| "llama3b_chat_translations_cut_mid_character": 358, |
| "captured": 1012, |
| "llama1b_dev_indicqa_contains_before": 0.1569, |
| "llama1b_dev_indicqa_contains_after": 0.2471, |
| "gemini_gsm8k_cut_at_256": 72, |
| "gemini_gsm8k_before": 0.5125, |
| "gemini_gsm8k_after": 0.8313 |
| }, |
| "gsm8k_numeric": { |
| "correct_answers_marked_wrong": 45 |
| }, |
| "bpc_first_token": { |
| "our_model_share_of_characters_counted_before": { |
| "tamil_heldout": 98.0, |
| "tanglish_heldout": 96.4 |
| } |
| } |
| } |
