{
 "model": "BrainboxAI/nitzotz",
 "eval_set": "8 frozen suites, 3990 questions (checksums in the repo's eval/manifest.json)",
 "models": {
  "nz": "BrainboxAI/nitzotz",
  "roeig": "RoeiG/laya-hebrew",
  "laya_ml": "convaiinnovations/laya (multilingual)"
 },
 "licences": {
  "nz": "Apache-2.0",
  "roeig": "CC-BY-NC-SA-4.0",
  "laya_ml": "Apache-2.0"
 },
 "suites": {
  "spam_scam": {
   "title": "Scam or not? (298 messages)",
   "n": 298,
   "chance": 0.5,
   "acc": {
    "nz": 0.9195,
    "roeig": 0.3221,
    "laya_ml": 0.5034
   },
   "brier": {
    "nz": 0.1321,
    "roeig": 0.7604,
    "laya_ml": 0.8042
   },
   "ece": {
    "nz": 0.0714,
    "roeig": 0.3802,
    "laya_ml": 0.38
   },
   "empty_acc": {
    "nz": 0.7047,
    "roeig": 0.2953,
    "laya_ml": 0.2953
   },
   "vs_nitzotz": {
    "roeig": {
     "nz_only": 186,
     "other_only": 8,
     "p": 3.5766488810898616e-45
    },
    "laya_ml": {
     "nz_only": 138,
     "other_only": 14,
     "p": 8.462475337125398e-27
    }
   }
  },
  "spam_scam_hard": {
   "title": "same, hard cases only (65)",
   "n": 65,
   "chance": 0.5,
   "acc": {
    "nz": 0.8308,
    "roeig": 0.3231,
    "laya_ml": 0.3538
   },
   "brier": {
    "nz": 0.2842,
    "roeig": 0.7076,
    "laya_ml": 1.1057
   },
   "ece": {
    "nz": 0.1132,
    "roeig": 0.3391,
    "laya_ml": 0.5564
   },
   "empty_acc": {
    "nz": 0.7077,
    "roeig": 0.2923,
    "laya_ml": 0.2923
   },
   "vs_nitzotz": {
    "roeig": {
     "nz_only": 34,
     "other_only": 1,
     "p": 2.0954757928848267e-09
    },
    "laya_ml": {
     "nz_only": 35,
     "other_only": 4,
     "p": 3.353161446284503e-07
    }
   }
  },
  "spam_type": {
   "title": "Message type, 6 options (298)",
   "n": 298,
   "chance": 0.1667,
   "acc": {
    "nz": 0.7785,
    "roeig": 0.5268,
    "laya_ml": 0.2081
   },
   "brier": {
    "nz": 0.3312,
    "roeig": 0.6489,
    "laya_ml": 1.0746
   },
   "ece": {
    "nz": 0.0683,
    "roeig": 0.1043,
    "laya_ml": 0.4131
   },
   "empty_acc": {
    "nz": 0.0772,
    "roeig": 0.2013,
    "laya_ml": 0.0772
   },
   "vs_nitzotz": {
    "roeig": {
     "nz_only": 108,
     "other_only": 33,
     "p": 1.6862794099851315e-10
    },
    "laya_ml": {
     "nz_only": 177,
     "other_only": 7,
     "p": 1.0714267651676255e-43
    }
   }
  },
  "spam_type_hard": {
   "title": "same, hard cases only (65)",
   "n": 65,
   "chance": 0.1667,
   "acc": {
    "nz": 0.6769,
    "roeig": 0.4462,
    "laya_ml": 0.2462
   },
   "brier": {
    "nz": 0.4865,
    "roeig": 0.7315,
    "laya_ml": 1.1094
   },
   "ece": {
    "nz": 0.1027,
    "roeig": 0.1877,
    "laya_ml": 0.4109
   },
   "empty_acc": {
    "nz": 0.1385,
    "roeig": 0.1538,
    "laya_ml": 0.1385
   },
   "vs_nitzotz": {
    "roeig": {
     "nz_only": 23,
     "other_only": 8,
     "p": 0.010673840530216694
    },
    "laya_ml": {
     "nz_only": 31,
     "other_only": 3,
     "p": 7.660128176212311e-07
    }
   }
  },
  "triage_type": {
   "title": "Support ticket type, 5 options (30)",
   "n": 30,
   "chance": 0.2,
   "acc": {
    "nz": 0.9,
    "roeig": 0.8,
    "laya_ml": 0.5667
   },
   "brier": {
    "nz": 0.1838,
    "roeig": 0.3191,
    "laya_ml": 0.6667
   },
   "ece": {
    "nz": 0.1292,
    "roeig": 0.1321,
    "laya_ml": 0.2772
   },
   "empty_acc": {
    "nz": 0.1667,
    "roeig": 0.1667,
    "laya_ml": 0.3333
   },
   "vs_nitzotz": {
    "roeig": {
     "nz_only": 5,
     "other_only": 2,
     "p": 0.453125
    },
    "laya_ml": {
     "nz_only": 12,
     "other_only": 2,
     "p": 0.012939453125
    }
   }
  },
  "triage_urgency": {
   "title": "Ticket urgency, 5 levels (30)",
   "n": 30,
   "chance": 0.2,
   "acc": {
    "nz": 0.7,
    "roeig": 0.3667,
    "laya_ml": 0.3333
   },
   "brier": {
    "nz": 0.4868,
    "roeig": 0.708,
    "laya_ml": 0.7501
   },
   "ece": {
    "nz": 0.2041,
    "roeig": 0.0322,
    "laya_ml": 0.2167
   },
   "empty_acc": {
    "nz": 0.4,
    "roeig": 0.2333,
    "laya_ml": 0.0667
   },
   "vs_nitzotz": {
    "roeig": {
     "nz_only": 14,
     "other_only": 4,
     "p": 0.0308837890625
    },
    "laya_ml": {
     "nz_only": 15,
     "other_only": 4,
     "p": 0.0192108154296875
    }
   }
  },
  "triage_paying": {
   "title": "Paying customer? yes/no (30)",
   "n": 30,
   "chance": 0.5,
   "acc": {
    "nz": 0.8,
    "roeig": 0.5667,
    "laya_ml": 0.5
   },
   "brier": {
    "nz": 0.2841,
    "roeig": 0.4338,
    "laya_ml": 0.8183
   },
   "ece": {
    "nz": 0.1466,
    "roeig": 0.2362,
    "laya_ml": 0.4284
   },
   "empty_acc": {
    "nz": 0.5333,
    "roeig": 0.5333,
    "laya_ml": 0.4667
   },
   "vs_nitzotz": {
    "roeig": {
     "nz_only": 7,
     "other_only": 0,
     "p": 0.015625
    },
    "laya_ml": {
     "nz_only": 14,
     "other_only": 5,
     "p": 0.063568115234375
    }
   }
  },
  "massive_20": {
   "title": "Voice command intent, 20 options (500)",
   "n": 500,
   "chance": 0.05,
   "acc": {
    "nz": 0.9,
    "roeig": 0.726,
    "laya_ml": 0.474
   },
   "brier": {
    "nz": 0.1671,
    "roeig": 0.3801,
    "laya_ml": 0.7338
   },
   "ece": {
    "nz": 0.0583,
    "roeig": 0.0395,
    "laya_ml": 0.1661
   },
   "empty_acc": {
    "nz": 0.064,
    "roeig": 0.052,
    "laya_ml": 0.044
   },
   "vs_nitzotz": {
    "roeig": {
     "nz_only": 98,
     "other_only": 11,
     "p": 1.3281749873159828e-18
    },
    "laya_ml": {
     "nz_only": 222,
     "other_only": 9,
     "p": 2.661469004156119e-54
    }
   }
  },
  "massive_4": {
   "title": "Voice command intent, 4 options (500)",
   "n": 500,
   "chance": 0.25,
   "acc": {
    "nz": 0.972,
    "roeig": 0.908,
    "laya_ml": 0.694
   },
   "brier": {
    "nz": 0.0462,
    "roeig": 0.1369,
    "laya_ml": 0.3987
   },
   "ece": {
    "nz": 0.0501,
    "roeig": 0.0393,
    "laya_ml": 0.0468
   },
   "empty_acc": {
    "nz": 0.254,
    "roeig": 0.242,
    "laya_ml": 0.256
   },
   "vs_nitzotz": {
    "roeig": {
     "nz_only": 35,
     "other_only": 3,
     "p": 6.677873898297548e-08
    },
    "laya_ml": {
     "nz_only": 141,
     "other_only": 2,
     "p": 1.846933796755538e-39
    }
   }
  },
  "sib200": {
   "title": "News topic, 7 options (204)",
   "n": 204,
   "chance": 0.1429,
   "acc": {
    "nz": 0.7941,
    "roeig": 0.8235,
    "laya_ml": 0.6618
   },
   "brier": {
    "nz": 0.3027,
    "roeig": 0.285,
    "laya_ml": 0.4918
   },
   "ece": {
    "nz": 0.0708,
    "roeig": 0.0751,
    "laya_ml": 0.1784
   },
   "empty_acc": {
    "nz": 0.1471,
    "roeig": 0.1961,
    "laya_ml": 0.25
   },
   "vs_nitzotz": {
    "roeig": {
     "nz_only": 12,
     "other_only": 18,
     "p": 0.361594608053565
    },
    "laya_ml": {
     "nz_only": 47,
     "other_only": 20,
     "p": 0.0013071686857636005
    }
   }
  },
  "heq_verify": {
   "title": "Does the passage support this answer? (600)",
   "n": 600,
   "chance": 0.5,
   "acc": {
    "nz": 0.9483,
    "roeig": 0.9417,
    "laya_ml": 0.4883
   },
   "brier": {
    "nz": 0.0938,
    "roeig": 0.0926,
    "laya_ml": 0.8274
   },
   "ece": {
    "nz": 0.0365,
    "roeig": 0.0178,
    "laya_ml": 0.3946
   },
   "empty_acc": {
    "nz": 0.51,
    "roeig": 0.8,
    "laya_ml": 0.59
   },
   "vs_nitzotz": {
    "roeig": {
     "nz_only": 30,
     "other_only": 26,
     "p": 0.6888797607233396
    },
    "laya_ml": {
     "nz_only": 296,
     "other_only": 20,
     "p": 3.520918733970021e-64
    }
   }
  },
  "heq_unans": {
   "title": "Plausible answer the passage does not give (600)",
   "n": 600,
   "chance": 0.5,
   "acc": {
    "nz": 0.8917,
    "roeig": 0.53,
    "laya_ml": 0.5367
   },
   "brier": {
    "nz": 0.1739,
    "roeig": 0.856,
    "laya_ml": 0.7348
   },
   "ece": {
    "nz": 0.0484,
    "roeig": 0.4336,
    "laya_ml": 0.3421
   },
   "empty_acc": {
    "nz": 0.5033,
    "roeig": 0.5033,
    "laya_ml": 0.5167
   },
   "vs_nitzotz": {
    "roeig": {
     "nz_only": 243,
     "other_only": 26,
     "p": 2.502982180208191e-45
    },
    "laya_ml": {
     "nz_only": 240,
     "other_only": 27,
     "p": 7.334224519429334e-44
    }
   }
  },
  "belebele": {
   "title": "Reading comprehension, 4 options (900)",
   "n": 900,
   "chance": 0.25,
   "acc": {
    "nz": 0.6322,
    "roeig": 0.7544,
    "laya_ml": 0.3144
   },
   "brier": {
    "nz": 0.5672,
    "roeig": 0.3442,
    "laya_ml": 0.8341
   },
   "ece": {
    "nz": 0.1919,
    "roeig": 0.0324,
    "laya_ml": 0.2307
   },
   "empty_acc": {
    "nz": 0.3011,
    "roeig": 0.3189,
    "laya_ml": 0.2844
   },
   "vs_nitzotz": {
    "roeig": {
     "nz_only": 72,
     "other_only": 182,
     "p": 3.66511528586527e-12
    },
    "laya_ml": {
     "nz_only": 366,
     "other_only": 80,
     "p": 9.158467645861244e-45
    }
   }
  }
 },
 "significance": "paired exact McNemar against Nitzotz on the same questions",
 "latency_ms": {
  "q8": {
   "median": 47.7,
   "p90": 75.6,
   "n": 50
  },
  "f16": {
   "median": 50.5,
   "p90": 77.0,
   "n": 50
  },
  "py": {
   "median": 60.0,
   "p90": 98.2,
   "n": 50
  },
  "q8_cpu": {
   "median": 309.1,
   "p90": 711.0,
   "n": 50
  },
  "f16_cpu": {
   "median": 425.9,
   "p90": 854.0,
   "n": 50
  },
  "py_cpu": {
   "median": 223.9,
   "p90": 452.7,
   "n": 50
  }
 },
 "latency_points_ms": {
  "q8": {
   "short": {
    "item": "massive_he_4-0075",
    "input_tokens": 38,
    "median_ms": 36.9,
    "p90_ms": 38.4,
    "min_ms": 35.6
   },
   "long": {
    "item": "heq_unanswerable-0522",
    "input_tokens": 430,
    "median_ms": 127.4,
    "p90_ms": 130.0,
    "min_ms": 108.0
   }
  },
  "f16": {
   "short": {
    "item": "massive_he_4-0075",
    "input_tokens": 38,
    "median_ms": 43.2,
    "p90_ms": 44.3,
    "min_ms": 41.2
   },
   "long": {
    "item": "heq_unanswerable-0522",
    "input_tokens": 430,
    "median_ms": 111.2,
    "p90_ms": 118.2,
    "min_ms": 108.7
   }
  },
  "py": {
   "short": {
    "item": "massive_he_4-0075",
    "input_tokens": 38,
    "median_ms": 43.6,
    "p90_ms": 44.7,
    "min_ms": 42.4
   },
   "long": {
    "item": "heq_unanswerable-0522",
    "input_tokens": 430,
    "median_ms": 139.3,
    "p90_ms": 157.0,
    "min_ms": 137.1
   }
  },
  "q8_cpu": {
   "short": {
    "item": "massive_he_4-0075",
    "input_tokens": 38,
    "median_ms": 159.5,
    "p90_ms": 185.0,
    "min_ms": 117.5
   },
   "long": {
    "item": "heq_unanswerable-0522",
    "input_tokens": 430,
    "median_ms": 1137.4,
    "p90_ms": 1243.6,
    "min_ms": 1072.4
   }
  },
  "py_cpu": {
   "short": {
    "item": "massive_he_4-0075",
    "input_tokens": 38,
    "median_ms": 112.0,
    "p90_ms": 131.3,
    "min_ms": 97.1
   },
   "long": {
    "item": "heq_unanswerable-0522",
    "input_tokens": 430,
    "median_ms": 1021.8,
    "p90_ms": 1100.7,
    "min_ms": 918.8
   }
  }
 },
 "latency_machine": "Intel Core Ultra 9 285H laptop, built-in Arc 140T GPU, Windows 11",
 "gguf_parity_50": {
  "q8": {
   "max_abs_diff": 0.0375,
   "mean_abs_diff": 0.001814,
   "argmax_agree": "50/50"
  },
  "f16": {
   "max_abs_diff": 0.0017,
   "mean_abs_diff": 0.000288,
   "argmax_agree": "50/50"
  },
  "q8_cpu": {
   "max_abs_diff": 0.0549,
   "mean_abs_diff": 0.003498,
   "argmax_agree": "50/50"
  },
  "f16_cpu": {
   "max_abs_diff": 0.0032,
   "mean_abs_diff": 0.000278,
   "argmax_agree": "50/50"
  }
 },
 "gguf_parity_spam_596": {
  "q8": {
   "max_abs_diff": 0.0166,
   "mean_abs_diff": 0.001297,
   "argmax_agree": "596/596",
   "accuracy_vs_gold": 0.8473
  },
  "f16": {
   "max_abs_diff": 0.0015,
   "mean_abs_diff": 0.000203,
   "argmax_agree": "596/596",
   "accuracy_vs_gold": 0.8473
  },
  "py": {
   "max_abs_diff": 0.0,
   "mean_abs_diff": 0.0,
   "argmax_agree": "596/596",
   "accuracy_vs_gold": 0.8473
  },
  "q8_cpu": {
   "max_abs_diff": 0.0207,
   "mean_abs_diff": 0.001918,
   "argmax_agree": "595/596",
   "accuracy_vs_gold": 0.8456
  },
  "f16_cpu": {
   "max_abs_diff": 0.0012,
   "mean_abs_diff": 0.000185,
   "argmax_agree": "596/596",
   "accuracy_vs_gold": 0.8473
  },
  "py_cpu": {
   "max_abs_diff": 0.0,
   "mean_abs_diff": 0.0,
   "argmax_agree": "596/596",
   "accuracy_vs_gold": 0.8473
  }
 },
 "belebele_split": {
  "verbatim_n": 252,
  "reworded_n": 648,
  "nz": {
   "verbatim": 0.7658730158730159,
   "reworded": 0.5802469135802469
  },
  "roeig": {
   "verbatim": 0.8412698412698413,
   "reworded": 0.720679012345679
  },
  "laya_ml": {
   "verbatim": 0.44047619047619047,
   "reworded": 0.2654320987654321
  }
 },
 "calibration_pooled": {
  "n": 3990,
  "ece_15bin": 0.0309
 },
 "wording_robustness": {
  "suite": "wording_he (frozen): 1786 questions, each asked in 6 wordings with the same meaning",
  "flip_rate": "share of questions whose top answer is not the same in all 6 wordings (lower is better)",
  "models": {
   "BrainboxAI/nitzotz": {
    "all_item_weighted": {
     "flip_rate": 0.0504,
     "flip_vs_orig": 0.018,
     "mean_acc": 0.9118,
     "worst_acc": 0.9048,
     "flip_rate_spam_type_without_form_swap": 0.0481
    },
    "parts": {
     "heq_verify": {
      "n_items": 600,
      "mean_acc": 0.9478,
      "worst_acc": 0.945,
      "flip_rate": 0.0283
     },
     "massive_he_4": {
      "n_items": 500,
      "mean_acc": 0.97,
      "worst_acc": 0.962,
      "flip_rate": 0.036
     },
     "spam_business_he:category": {
      "n_items": 298,
      "mean_acc": 0.7752,
      "worst_acc": 0.7685,
      "flip_rate": 0.0805
     },
     "spam_business_he:category:hard": {
      "n_items": 65,
      "mean_acc": 0.6692,
      "worst_acc": 0.6462,
      "flip_rate": 0.1077
     },
     "spam_business_he:scam": {
      "n_items": 298,
      "mean_acc": 0.915,
      "worst_acc": 0.9128,
      "flip_rate": 0.0201
     },
     "spam_business_he:scam:hard": {
      "n_items": 65,
      "mean_acc": 0.8154,
      "worst_acc": 0.8,
      "flip_rate": 0.0615
     },
     "triage30:category": {
      "n_items": 30,
      "mean_acc": 0.9166,
      "worst_acc": 0.9,
      "flip_rate": 0.0333
     },
     "triage30:paying": {
      "n_items": 30,
      "mean_acc": 0.7944,
      "worst_acc": 0.7333,
      "flip_rate": 0.1667
     },
     "triage30:urgency": {
      "n_items": 30,
      "mean_acc": 0.6611,
      "worst_acc": 0.6,
      "flip_rate": 0.6333
     }
    },
    "single_answer_wordings": {}
   }
  }
 },
 "belebele_negation": {
  "n": 158,
  "acc": 0.5886
 },
 "seeds": {
  "heldout_this": 0.9505,
  "others": [
   {
    "name": "seed 2",
    "heldout": 0.9519,
    "acc": {
     "spam_scam": 0.9195,
     "spam_scam_hard": 0.8308,
     "spam_type": 0.802,
     "spam_type_hard": 0.7077,
     "triage_type": 0.9,
     "triage_urgency": 0.7333,
     "triage_paying": 0.8,
     "massive_20": 0.894,
     "massive_4": 0.96,
     "sib200": 0.8284,
     "heq_verify": 0.9467,
     "heq_unans": 0.8983,
     "belebele": 0.6089
    }
   },
   {
    "name": "seed 3",
    "heldout": 0.9521,
    "acc": {
     "spam_scam": 0.906,
     "spam_scam_hard": 0.7846,
     "spam_type": 0.7785,
     "spam_type_hard": 0.6615,
     "triage_type": 0.8333,
     "triage_urgency": 0.7667,
     "triage_paying": 0.8333,
     "massive_20": 0.902,
     "massive_4": 0.968,
     "sib200": 0.8039,
     "heq_verify": 0.95,
     "heq_unans": 0.8983,
     "belebele": 0.6311
    }
   }
  ],
  "agreement_min": 0.9008,
  "agreement_max": 0.9063,
  "picked_on": {
   "this": {
    "n": 188,
    "acc": 0.984
   },
   "others": [
    {
     "n": 188,
     "acc": 0.9734
    },
    {
     "n": 188,
     "acc": 0.9734
    }
   ]
  }
 },
 "none_of_the_options": {
  "suite": "notinlist_he (frozen): 400 MASSIVE and Belebele test questions with a 'none of the options' choice added; in half of them 'none' is right, in the other half the right answer is still in the list",
  "model": "BrainboxAI/nitzotz",
  "parts": {
   "all": {
    "n": 400,
    "acc": 0.685,
    "picks_none": 0.535
   },
   "belebele_he/control": {
    "n": 100,
    "acc": 0.45,
    "picks_none": 0.43
   },
   "belebele_he/none": {
    "n": 100,
    "acc": 0.76,
    "picks_none": 0.76
   },
   "control": {
    "n": 200,
    "acc": 0.61,
    "picks_none": 0.31
   },
   "massive_he_20/control": {
    "n": 100,
    "acc": 0.77,
    "picks_none": 0.19
   },
   "massive_he_20/none": {
    "n": 100,
    "acc": 0.76,
    "picks_none": 0.76
   },
   "none": {
    "n": 200,
    "acc": 0.76,
    "picks_none": 0.76
   }
  },
  "text_removed": {
   "all": {
    "n": 400,
    "acc": 0.49,
    "picks_none": 0.9825
   },
   "belebele_he/control": {
    "n": 100,
    "acc": 0.01,
    "picks_none": 0.99
   },
   "belebele_he/none": {
    "n": 100,
    "acc": 0.95,
    "picks_none": 0.95
   },
   "control": {
    "n": 200,
    "acc": 0.005,
    "picks_none": 0.99
   },
   "massive_he_20/control": {
    "n": 100,
    "acc": 0.0,
    "picks_none": 0.99
   },
   "massive_he_20/none": {
    "n": 100,
    "acc": 1.0,
    "picks_none": 1.0
   },
   "none": {
    "n": 200,
    "acc": 0.975,
    "picks_none": 0.975
   }
  }
 }
}