{
 "generated": "2026-09-15 06:44 UTC",
 "letter_position": {
  "model": "Qwen3.5-2B",
  "wrong_milu_answers": 4822,
  "most_chosen_letter": "C",
  "times": 4163,
  "mmlu_chat_before": 0.2525,
  "mmlu_raw": 0.49,
  "milu_chat_before": 0.2433,
  "mmlu_chat_after": 0.26,
  "milu_chat_after": 0.242
 },
 "sdpa": {
  "model": "Gemma-3-1B",
  "tasks": {
   "in22gen_ta_en": {
    "single": 39.0717,
    "batched_sdpa": 36.4533,
    "batched_eager": 39.529
   },
   "flores_ta_en": {
    "single": 40.3854,
    "batched_sdpa": 38.4472,
    "batched_eager": 40.8063
   }
  }
 },
 "start_token": {
  "material": [
   "Gemma-3-1B-it: dev mmlu_en 21.0 to 41.0, test milu_ta 26.8 to 31.4, test mmlu_en 23.5 to 39.0",
   "Llama-3.2-1B-Instruct: dev milu_ta 26.0 to 28.0, dev mmlu_en 39.0 to 35.0, test mmlu_en 39.0 to 35.5",
   "Sarvam-1: dev milu_ta 34.0 to 36.7, dev mmlu_en 43.0 to 45.0"
  ],
  "pending": [],
  "gemma4_e4b_raw_dev_before": {
   "flores_en_ta_chrf": 0.0346,
   "flores_ta_en_chrf": 1.025,
   "tamil_bpc": 5.1868,
   "mmlu": 0.31
  }
 },
 "preamble": {
  "local_chat": [
   {
    "model": "tamil-lm-2b-instruct-r4",
    "share": 0.0,
    "largest_gap": 0.0,
    "task": "in22gen_ta_en",
    "first": 48.8408,
    "extracted": 48.8408
   },
   {
    "model": "Qwen3.5-2B",
    "share": 7.8,
    "largest_gap": 0.4,
    "task": "flores_en_ta",
    "first": 27.9656,
    "extracted": 28.4019
   },
   {
    "model": "Qwen3.5-4B",
    "share": 1.6,
    "largest_gap": 0.2,
    "task": "in22gen_en_ta",
    "first": 35.8439,
    "extracted": 36.0683
   },
   {
    "model": "Gemma-3-1B-it",
    "share": 55.0,
    "largest_gap": 21.1,
    "task": "flores_en_ta",
    "first": 7.0102,
    "extracted": 28.082
   },
   {
    "model": "Llama-3.2-1B-Instruct",
    "share": 0.3,
    "largest_gap": 0.2,
    "task": "in22gen_ta_en",
    "first": 33.1526,
    "extracted": 33.3336
   },
   {
    "model": "Gemma-3-4B-it",
    "share": 89.1,
    "largest_gap": 44.8,
    "task": "flores_en_ta",
    "first": 1.3475,
    "extracted": 46.1687
   },
   {
    "model": "Sarvam-1",
    "share": 0.6,
    "largest_gap": 0.1,
    "task": "in22gen_en_ta",
    "first": 30.4741,
    "extracted": 30.5594
   },
   {
    "model": "Qwen3-1.7B-tamil-Instruct",
    "share": 18.7,
    "largest_gap": 1.2,
    "task": "in22gen_en_ta",
    "first": 15.264,
    "extracted": 16.4391
   }
  ],
  "gemini": {
   "share": 56.3,
   "in22gen_ta_en_first": 24.1883,
   "in22gen_ta_en_extracted": 59.448
  }
 },
 "caps": {
  "llama_flores_refs_over_160": 725,
  "flores_items": 1012,
  "llama3b_chat_translations_cut_mid_character": 358,
  "captured": 1012,
  "llama1b_dev_indicqa_contains_before": 0.1569,
  "llama1b_dev_indicqa_contains_after": 0.2471,
  "gemini_gsm8k_cut_at_256": 72,
  "gemini_gsm8k_before": 0.5125,
  "gemini_gsm8k_after": 0.8313
 },
 "gsm8k_numeric": {
  "correct_answers_marked_wrong": 45
 },
 "bpc_first_token": {
  "our_model_share_of_characters_counted_before": {
   "tamil_heldout": 98.0,
   "tanglish_heldout": 96.4
  }
 }
}