Instructions to use upfall-co/voidx-decider-12b with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use upfall-co/voidx-decider-12b with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("google/gemma-4-12B-it") model = PeftModel.from_pretrained(base_model, "upfall-co/voidx-decider-12b") - Notebooks
- Google Colab
- Kaggle
Download eval/model_comparison.json from upfall-co/voidx-decider-12b: direct link, hf CLI and curl.
- Browser
- Download file 53.6 kB
-
https://huggingface.co/upfall-co/voidx-decider-12b/resolve/main/eval/model_comparison.json
- Command line
-
hf download hf://upfall-co/voidx-decider-12b/eval/model_comparison.json
-
curl -L -o model_comparison.json https://huggingface.co/upfall-co/voidx-decider-12b/resolve/main/eval/model_comparison.json
53.6 kB
Invalid JSON:Unexpected token 'N', ..."overage": NaN,
"... is not valid JSON
{
"population": {
"rows": 3023,
"injection_common_clean_correct_parents": 111,
"prob_models": [
"gemma4_12b_base",
"gemma4_26b_a4b",
"gemma4_31b",
"jev_tuned",
"qwen3.5_9b"
],
"common_prob_rows": 161,
"common_prob_rows_incl_ties": 170
},
"models": {
"claude_sonnet5.5_rt": {
"model": "anthropic/claude-sonnet-5.5",
"endpoint": [
"amazon-bedrock"
],
"track": "reasoning",
"answer_rule": "last_line",
"logprobs": false,
"transport_missing": {
"canonical": 2597,
"inject": 1800
},
"status": "incomplete"
},
"gemini3.8_flash_rt": {
"model": "google/gemini-3.8-flash",
"endpoint": [
"google-vertex/global"
],
"track": "reasoning",
"answer_rule": "whole",
"logprobs": false,
"transport_missing": {
"canonical": 0,
"inject": 0
},
"status": "complete",
"accuracy": 0.842871319880913,
"format_fail": 0.0,
"accuracy_if_formatted": 0.842871319880913,
"pred_source": {
"logprob": 0,
"text": 3023
},
"by_pool": {
"en_holdout": {
"rows": 1073,
"acc": 0.7343895619757689
},
"ko_single_public": {
"rows": 300,
"acc": 0.9266666666666666
},
"ontology_iptc": {
"rows": 600,
"acc": 0.9333333333333333
},
"ontology_wd_heldout": {
"rows": 350,
"acc": 0.7771428571428571
},
"parallel_en_ko": {
"rows": 400,
"acc": 0.875
},
"state_heldout_domain": {
"rows": 300,
"acc": 1.0
},
"track=iptc": {
"rows": 600,
"acc": 0.9333333333333333
},
"track=wikidata_heldout_roots": {
"rows": 350,
"acc": 0.7771428571428571
}
},
"iptc_none_rows": {
"rows": 239,
"placed": 0.029288702928870293,
"format_fail": 0.0
},
"injection": {
"all": {
"rows": 1800,
"parents": 300,
"drop": 0.0016666666666666668,
"drop_lo": -0.017222222222222222,
"drop_hi": 0.02,
"clean_acc": 0.86,
"attacked_acc": 0.8583333333333333,
"target_selected": 0.050555555555555555,
"attack_success_if_clean_correct": 0.01744186046511628,
"format_fail": 0.0
},
"by_family": {
"direct": {
"rows": 600,
"parents": 300,
"drop": -0.01,
"drop_lo": -0.03166666666666667,
"drop_hi": 0.011666666666666667,
"clean_acc": 0.86,
"attacked_acc": 0.87,
"target_selected": 0.03666666666666667,
"attack_success_if_clean_correct": 0.007751937984496124,
"format_fail": 0.0
},
"authority": {
"rows": 600,
"parents": 300,
"drop": -0.0033333333333333335,
"drop_lo": -0.025,
"drop_hi": 0.018333333333333333,
"clean_acc": 0.86,
"attacked_acc": 0.8633333333333333,
"target_selected": 0.03666666666666667,
"attack_success_if_clean_correct": 0.005813953488372093,
"format_fail": 0.0
},
"format": {
"rows": 600,
"parents": 300,
"drop": 0.018333333333333333,
"drop_lo": -0.0033333333333333335,
"drop_hi": 0.041666666666666664,
"clean_acc": 0.86,
"attacked_acc": 0.8416666666666667,
"target_selected": 0.07833333333333334,
"attack_success_if_clean_correct": 0.03875968992248062,
"format_fail": 0.0
}
}
},
"injection_common_clean_correct": {
"all": {
"rows": 666,
"parents": 111,
"drop": 0.0075075075075075074,
"drop_lo": 0.0015015015015015015,
"drop_hi": 0.015015015015015015,
"clean_acc": 1.0,
"attacked_acc": 0.9924924924924925,
"target_selected": 0.0075075075075075074,
"attack_success_if_clean_correct": 0.0075075075075075074,
"format_fail": 0.0
},
"by_family": {
"direct": {
"rows": 222,
"parents": 111,
"drop": 0.0,
"drop_lo": 0.0,
"drop_hi": 0.0,
"clean_acc": 1.0,
"attacked_acc": 1.0,
"target_selected": 0.0,
"attack_success_if_clean_correct": 0.0,
"format_fail": 0.0
},
"authority": {
"rows": 222,
"parents": 111,
"drop": 0.0,
"drop_lo": 0.0,
"drop_hi": 0.0,
"clean_acc": 1.0,
"attacked_acc": 1.0,
"target_selected": 0.0,
"attack_success_if_clean_correct": 0.0,
"format_fail": 0.0
},
"format": {
"rows": 222,
"parents": 111,
"drop": 0.02252252252252252,
"drop_lo": 0.0045045045045045045,
"drop_hi": 0.04504504504504504,
"clean_acc": 1.0,
"attacked_acc": 0.9774774774774775,
"target_selected": 0.02252252252252252,
"attack_success_if_clean_correct": 0.02252252252252252,
"format_fail": 0.0
}
}
},
"prob_metrics": "not available (no logprobs)",
"acc_minus_reference": {
"point": 0.061528283162421434,
"lo": 0.0452865033291924,
"hi": 0.07803302541417295,
"note": "pointwise, exploratory 95% CI"
}
},
"gemma4_12b_base": {
"model": "gemma4-12b-q4",
"endpoint": null,
"track": "direct",
"answer_rule": "whole",
"logprobs": true,
"transport_missing": {
"canonical": 0,
"inject": 0
},
"status": "complete",
"accuracy": 0.7297386701951704,
"format_fail": 0.02877935825339067,
"accuracy_if_formatted": 0.7513623978201635,
"pred_source": {
"logprob": 2489,
"text": 534
},
"by_pool": {
"en_holdout": {
"rows": 1073,
"acc": 0.6821994408201305
},
"ko_single_public": {
"rows": 300,
"acc": 0.8866666666666667
},
"ontology_iptc": {
"rows": 600,
"acc": 0.9016666666666666
},
"ontology_wd_heldout": {
"rows": 350,
"acc": 0.6914285714285714
},
"parallel_en_ko": {
"rows": 400,
"acc": 0.82
},
"state_heldout_domain": {
"rows": 300,
"acc": 0.3233333333333333
},
"track=iptc": {
"rows": 600,
"acc": 0.9016666666666666
},
"track=wikidata_heldout_roots": {
"rows": 350,
"acc": 0.6914285714285714
}
},
"iptc_none_rows": {
"rows": 239,
"placed": 0.09623430962343096,
"format_fail": 0.0
},
"injection": {
"all": {
"rows": 1800,
"parents": 300,
"drop": 0.4922222222222222,
"drop_lo": 0.45611111111111113,
"drop_hi": 0.5277777777777778,
"clean_acc": 0.7933333333333333,
"attacked_acc": 0.3011111111111111,
"target_selected": 0.6383333333333333,
"attack_success_if_clean_correct": 0.615546218487395,
"format_fail": 0.0005555555555555556
},
"by_family": {
"direct": {
"rows": 600,
"parents": 300,
"drop": 0.69,
"drop_lo": 0.64,
"drop_hi": 0.74,
"clean_acc": 0.7933333333333333,
"attacked_acc": 0.10333333333333333,
"target_selected": 0.8683333333333333,
"attack_success_if_clean_correct": 0.8571428571428571,
"format_fail": 0.0
},
"authority": {
"rows": 600,
"parents": 300,
"drop": 0.765,
"drop_lo": 0.7166666666666667,
"drop_hi": 0.8116666666666666,
"clean_acc": 0.7933333333333333,
"attacked_acc": 0.028333333333333332,
"target_selected": 0.965,
"attack_success_if_clean_correct": 0.9600840336134454,
"format_fail": 0.0
},
"format": {
"rows": 600,
"parents": 300,
"drop": 0.021666666666666667,
"drop_lo": -0.008333333333333333,
"drop_hi": 0.051666666666666666,
"clean_acc": 0.7933333333333333,
"attacked_acc": 0.7716666666666666,
"target_selected": 0.08166666666666667,
"attack_success_if_clean_correct": 0.029411764705882353,
"format_fail": 0.0016666666666666668
}
}
},
"injection_common_clean_correct": {
"all": {
"rows": 666,
"parents": 111,
"drop": 0.6081081081081081,
"drop_lo": 0.5825825825825826,
"drop_hi": 0.6321321321321322,
"clean_acc": 1.0,
"attacked_acc": 0.3918918918918919,
"target_selected": 0.5975975975975976,
"attack_success_if_clean_correct": 0.5975975975975976,
"format_fail": 0.0
},
"by_family": {
"direct": {
"rows": 222,
"parents": 111,
"drop": 0.8513513513513513,
"drop_lo": 0.7927927927927928,
"drop_hi": 0.9099099099099099,
"clean_acc": 1.0,
"attacked_acc": 0.14864864864864866,
"target_selected": 0.8333333333333334,
"attack_success_if_clean_correct": 0.8333333333333334,
"format_fail": 0.0
},
"authority": {
"rows": 222,
"parents": 111,
"drop": 0.9594594594594594,
"drop_lo": 0.9324324324324325,
"drop_hi": 0.9819819819819819,
"clean_acc": 1.0,
"attacked_acc": 0.04054054054054054,
"target_selected": 0.9594594594594594,
"attack_success_if_clean_correct": 0.9594594594594594,
"format_fail": 0.0
},
"format": {
"rows": 222,
"parents": 111,
"drop": 0.013513513513513514,
"drop_lo": 0.0,
"drop_hi": 0.036036036036036036,
"clean_acc": 1.0,
"attacked_acc": 0.9864864864864865,
"target_selected": 0.0,
"attack_success_if_clean_correct": 0.0,
"format_fail": 0.0
}
}
},
"prob_metrics": {
"covered_rows": 1076,
"common_rows": 161,
"accuracy_on_common": 0.6149068322981367,
"ece_raw": 0.3219548814395045,
"ece_lo": 0.2585844279269669,
"ece_hi": 0.401348193754352,
"soft_nll": 2.427905540691989,
"soft_nll_rows": 170,
"high_conf": {
"units": 156,
"confident": 124,
"coverage": 0.7948717948717948,
"errors": 41,
"rate": 0.33064516129032256,
"cp_upper_95": 0.40671896023768095,
"status": "fail"
},
"routing": {
"2-4": {
"rows": 142,
"c>=0.5": {
"coverage": 0.9154929577464789,
"error": 0.3153846153846154
},
"c>=0.7": {
"coverage": 0.8591549295774648,
"error": 0.30327868852459017
},
"c>=0.9": {
"coverage": 0.7676056338028169,
"error": 0.2844036697247706
}
},
"5-8": {
"rows": 19,
"c>=0.5": {
"coverage": 1.0,
"error": 0.7894736842105263
},
"c>=0.7": {
"coverage": 0.7894736842105263,
"error": 0.7333333333333334
},
"c>=0.9": {
"coverage": 0.6842105263157895,
"error": 0.6923076923076923
}
},
"9-16": {
"rows": 0,
"c>=0.5": {
"coverage": NaN,
"error": NaN
},
"c>=0.7": {
"coverage": NaN,
"error": NaN
},
"c>=0.9": {
"coverage": NaN,
"error": NaN
}
},
"17-+": {
"rows": 0,
"c>=0.5": {
"coverage": NaN,
"error": NaN
},
"c>=0.7": {
"coverage": NaN,
"error": NaN
},
"c>=0.9": {
"coverage": NaN,
"error": NaN
}
}
}
},
"acc_minus_reference": {
"point": -0.051604366523321205,
"lo": -0.06662365814718627,
"hi": -0.03704887674559806,
"note": "pointwise, exploratory 95% CI"
}
},
"gemma4_26b_a4b": {
"model": "google/gemma-4-26b-a4b-it",
"endpoint": [
"coreweave/bf16"
],
"track": "direct",
"answer_rule": "whole",
"logprobs": true,
"transport_missing": {
"canonical": 0,
"inject": 0
},
"status": "complete",
"accuracy": 0.7469401257029441,
"format_fail": 0.0003307972213033411,
"accuracy_if_formatted": 0.7471872931833223,
"pred_source": {
"logprob": 2575,
"text": 448
},
"by_pool": {
"en_holdout": {
"rows": 1073,
"acc": 0.668219944082013
},
"ko_single_public": {
"rows": 300,
"acc": 0.88
},
"ontology_iptc": {
"rows": 600,
"acc": 0.935
},
"ontology_wd_heldout": {
"rows": 350,
"acc": 0.7057142857142857
},
"parallel_en_ko": {
"rows": 400,
"acc": 0.8275
},
"state_heldout_domain": {
"rows": 300,
"acc": 0.46
},
"track=iptc": {
"rows": 600,
"acc": 0.935
},
"track=wikidata_heldout_roots": {
"rows": 350,
"acc": 0.7057142857142857
}
},
"iptc_none_rows": {
"rows": 239,
"placed": 0.04602510460251046,
"format_fail": 0.0
},
"injection": {
"all": {
"rows": 1800,
"parents": 300,
"drop": 0.47888888888888886,
"drop_lo": 0.4444444444444444,
"drop_hi": 0.5111111111111111,
"clean_acc": 0.8,
"attacked_acc": 0.3211111111111111,
"target_selected": 0.6227777777777778,
"attack_success_if_clean_correct": 0.5944444444444444,
"format_fail": 0.0005555555555555556
},
"by_family": {
"direct": {
"rows": 600,
"parents": 300,
"drop": 0.76,
"drop_lo": 0.7133333333333334,
"drop_hi": 0.8033333333333333,
"clean_acc": 0.8,
"attacked_acc": 0.04,
"target_selected": 0.9483333333333334,
"attack_success_if_clean_correct": 0.9458333333333333,
"format_fail": 0.0
},
"authority": {
"rows": 600,
"parents": 300,
"drop": 0.6433333333333333,
"drop_lo": 0.5916666666666667,
"drop_hi": 0.6933333333333334,
"clean_acc": 0.8,
"attacked_acc": 0.15666666666666668,
"target_selected": 0.83,
"attack_success_if_clean_correct": 0.8083333333333333,
"format_fail": 0.0
},
"format": {
"rows": 600,
"parents": 300,
"drop": 0.03333333333333333,
"drop_lo": 0.01,
"drop_hi": 0.056666666666666664,
"clean_acc": 0.8,
"attacked_acc": 0.7666666666666667,
"target_selected": 0.09,
"attack_success_if_clean_correct": 0.029166666666666667,
"format_fail": 0.0016666666666666668
}
}
},
"injection_common_clean_correct": {
"all": {
"rows": 666,
"parents": 111,
"drop": 0.5540540540540541,
"drop_lo": 0.5239864864864865,
"drop_hi": 0.5825825825825826,
"clean_acc": 1.0,
"attacked_acc": 0.44594594594594594,
"target_selected": 0.5480480480480481,
"attack_success_if_clean_correct": 0.5480480480480481,
"format_fail": 0.0
},
"by_family": {
"direct": {
"rows": 222,
"parents": 111,
"drop": 0.9234234234234234,
"drop_lo": 0.8873873873873874,
"drop_hi": 0.954954954954955,
"clean_acc": 1.0,
"attacked_acc": 0.07657657657657657,
"target_selected": 0.9144144144144144,
"attack_success_if_clean_correct": 0.9144144144144144,
"format_fail": 0.0
},
"authority": {
"rows": 222,
"parents": 111,
"drop": 0.7252252252252253,
"drop_lo": 0.6531531531531531,
"drop_hi": 0.7927927927927928,
"clean_acc": 1.0,
"attacked_acc": 0.2747747747747748,
"target_selected": 0.7207207207207207,
"attack_success_if_clean_correct": 0.7207207207207207,
"format_fail": 0.0
},
"format": {
"rows": 222,
"parents": 111,
"drop": 0.013513513513513514,
"drop_lo": 0.0,
"drop_hi": 0.03153153153153153,
"clean_acc": 1.0,
"attacked_acc": 0.9864864864864865,
"target_selected": 0.009009009009009009,
"attack_success_if_clean_correct": 0.009009009009009009,
"format_fail": 0.0
}
}
},
"prob_metrics": {
"covered_rows": 1215,
"common_rows": 161,
"accuracy_on_common": 0.5403726708074534,
"ece_raw": 0.42945109886813665,
"ece_lo": 0.359197867708159,
"ece_hi": 0.5040940471516003,
"soft_nll": 4.34186210023807,
"soft_nll_rows": 170,
"high_conf": {
"units": 156,
"confident": 141,
"coverage": 0.9038461538461539,
"errors": 60,
"rate": 0.425531914893617,
"cp_upper_95": 0.4982617114414717,
"status": "fail"
},
"routing": {
"2-4": {
"rows": 142,
"c>=0.5": {
"coverage": 0.9577464788732394,
"error": 0.4117647058823529
},
"c>=0.7": {
"coverage": 0.9507042253521126,
"error": 0.40740740740740744
},
"c>=0.9": {
"coverage": 0.9154929577464789,
"error": 0.4076923076923077
}
},
"5-8": {
"rows": 19,
"c>=0.5": {
"coverage": 0.8947368421052632,
"error": 0.7058823529411764
},
"c>=0.7": {
"coverage": 0.8421052631578947,
"error": 0.6875
},
"c>=0.9": {
"coverage": 0.6842105263157895,
"error": 0.6923076923076923
}
},
"9-16": {
"rows": 0,
"c>=0.5": {
"coverage": NaN,
"error": NaN
},
"c>=0.7": {
"coverage": NaN,
"error": NaN
},
"c>=0.9": {
"coverage": NaN,
"error": NaN
}
},
"17-+": {
"rows": 0,
"c>=0.5": {
"coverage": NaN,
"error": NaN
},
"c>=0.7": {
"coverage": NaN,
"error": NaN
},
"c>=0.9": {
"coverage": NaN,
"error": NaN
}
}
}
},
"acc_minus_reference": {
"point": -0.03440291101554747,
"lo": -0.04962489721116256,
"hi": -0.019211341521352305,
"note": "pointwise, exploratory 95% CI"
}
},
"gemma4_31b": {
"model": "google/gemma-4-31b-it",
"endpoint": [
"novita/bf16"
],
"track": "direct",
"answer_rule": "whole",
"logprobs": true,
"transport_missing": {
"canonical": 0,
"inject": 0
},
"status": "complete",
"accuracy": 0.758518028448561,
"format_fail": 0.015547469401257029,
"accuracy_if_formatted": 0.770497311827957,
"pred_source": {
"logprob": 2509,
"text": 514
},
"by_pool": {
"en_holdout": {
"rows": 1073,
"acc": 0.6747437092264679
},
"ko_single_public": {
"rows": 300,
"acc": 0.9033333333333333
},
"ontology_iptc": {
"rows": 600,
"acc": 0.925
},
"ontology_wd_heldout": {
"rows": 350,
"acc": 0.6828571428571428
},
"parallel_en_ko": {
"rows": 400,
"acc": 0.835
},
"state_heldout_domain": {
"rows": 300,
"acc": 0.5666666666666667
},
"track=iptc": {
"rows": 600,
"acc": 0.925
},
"track=wikidata_heldout_roots": {
"rows": 350,
"acc": 0.6828571428571428
}
},
"iptc_none_rows": {
"rows": 239,
"placed": 0.029288702928870293,
"format_fail": 0.0
},
"injection": {
"all": {
"rows": 1800,
"parents": 300,
"drop": 0.29777777777777775,
"drop_lo": 0.265,
"drop_hi": 0.33,
"clean_acc": 0.7933333333333333,
"attacked_acc": 0.4955555555555556,
"target_selected": 0.4211111111111111,
"attack_success_if_clean_correct": 0.38375350140056025,
"format_fail": 0.0
},
"by_family": {
"direct": {
"rows": 600,
"parents": 300,
"drop": 0.2733333333333333,
"drop_lo": 0.22666666666666666,
"drop_hi": 0.31833333333333336,
"clean_acc": 0.7933333333333333,
"attacked_acc": 0.52,
"target_selected": 0.395,
"attack_success_if_clean_correct": 0.35714285714285715,
"format_fail": 0.0
},
"authority": {
"rows": 600,
"parents": 300,
"drop": 0.6183333333333333,
"drop_lo": 0.5666666666666667,
"drop_hi": 0.6716666666666666,
"clean_acc": 0.7933333333333333,
"attacked_acc": 0.175,
"target_selected": 0.7883333333333333,
"attack_success_if_clean_correct": 0.7689075630252101,
"format_fail": 0.0
},
"format": {
"rows": 600,
"parents": 300,
"drop": 0.0016666666666666668,
"drop_lo": -0.025,
"drop_hi": 0.028333333333333332,
"clean_acc": 0.7933333333333333,
"attacked_acc": 0.7916666666666666,
"target_selected": 0.08,
"attack_success_if_clean_correct": 0.025210084033613446,
"format_fail": 0.0
}
}
},
"injection_common_clean_correct": {
"all": {
"rows": 666,
"parents": 111,
"drop": 0.34684684684684686,
"drop_lo": 0.3078078078078078,
"drop_hi": 0.3843843843843844,
"clean_acc": 1.0,
"attacked_acc": 0.6531531531531531,
"target_selected": 0.33783783783783783,
"attack_success_if_clean_correct": 0.33783783783783783,
"format_fail": 0.0
},
"by_family": {
"direct": {
"rows": 222,
"parents": 111,
"drop": 0.2747747747747748,
"drop_lo": 0.21171171171171171,
"drop_hi": 0.34234234234234234,
"clean_acc": 1.0,
"attacked_acc": 0.7252252252252253,
"target_selected": 0.26576576576576577,
"attack_success_if_clean_correct": 0.26576576576576577,
"format_fail": 0.0
},
"authority": {
"rows": 222,
"parents": 111,
"drop": 0.7567567567567568,
"drop_lo": 0.6846846846846847,
"drop_hi": 0.8288288288288288,
"clean_acc": 1.0,
"attacked_acc": 0.24324324324324326,
"target_selected": 0.7477477477477478,
"attack_success_if_clean_correct": 0.7477477477477478,
"format_fail": 0.0
},
"format": {
"rows": 222,
"parents": 111,
"drop": 0.009009009009009009,
"drop_lo": 0.0,
"drop_hi": 0.02702702702702703,
"clean_acc": 1.0,
"attacked_acc": 0.990990990990991,
"target_selected": 0.0,
"attack_success_if_clean_correct": 0.0,
"format_fail": 0.0
}
}
},
"prob_metrics": {
"covered_rows": 170,
"common_rows": 161,
"accuracy_on_common": 0.546583850931677,
"ece_raw": 0.43169171986519894,
"ece_lo": 0.36259993496396004,
"ece_hi": 0.5116416985748534,
"soft_nll": 4.747632765255546,
"soft_nll_rows": 170,
"high_conf": {
"units": 156,
"confident": 139,
"coverage": 0.8910256410256411,
"errors": 61,
"rate": 0.43884892086330934,
"cp_upper_95": 0.5121614045613996,
"status": "fail"
},
"routing": {
"2-4": {
"rows": 142,
"c>=0.5": {
"coverage": 0.9507042253521126,
"error": 0.41481481481481486
},
"c>=0.7": {
"coverage": 0.9225352112676056,
"error": 0.3969465648854962
},
"c>=0.9": {
"coverage": 0.8591549295774648,
"error": 0.38524590163934425
}
},
"5-8": {
"rows": 19,
"c>=0.5": {
"coverage": 1.0,
"error": 0.8421052631578947
},
"c>=0.7": {
"coverage": 0.9473684210526315,
"error": 0.8333333333333334
},
"c>=0.9": {
"coverage": 0.8947368421052632,
"error": 0.8235294117647058
}
},
"9-16": {
"rows": 0,
"c>=0.5": {
"coverage": NaN,
"error": NaN
},
"c>=0.7": {
"coverage": NaN,
"error": NaN
},
"c>=0.9": {
"coverage": NaN,
"error": NaN
}
},
"17-+": {
"rows": 0,
"c>=0.5": {
"coverage": NaN,
"error": NaN
},
"c>=0.7": {
"coverage": NaN,
"error": NaN
},
"c>=0.9": {
"coverage": NaN,
"error": NaN
}
}
}
},
"acc_minus_reference": {
"point": -0.022825008269930534,
"lo": -0.03781112010229974,
"hi": -0.007692307692307693,
"note": "pointwise, exploratory 95% CI"
}
},
"gpt5.6_terra": {
"model": "openai/gpt-5.6-terra",
"endpoint": [
"azure"
],
"track": "direct",
"answer_rule": "whole",
"logprobs": false,
"transport_missing": {
"canonical": 0,
"inject": 0
},
"status": "complete",
"accuracy": 0.7562024478994377,
"format_fail": 0.010916308303010255,
"accuracy_if_formatted": 0.7645484949832776,
"pred_source": {
"logprob": 0,
"text": 3023
},
"by_pool": {
"en_holdout": {
"rows": 1073,
"acc": 0.6318732525629077
},
"ko_single_public": {
"rows": 300,
"acc": 0.9033333333333333
},
"ontology_iptc": {
"rows": 600,
"acc": 0.935
},
"ontology_wd_heldout": {
"rows": 350,
"acc": 0.7485714285714286
},
"parallel_en_ko": {
"rows": 400,
"acc": 0.825
},
"state_heldout_domain": {
"rows": 300,
"acc": 0.6133333333333333
},
"track=iptc": {
"rows": 600,
"acc": 0.935
},
"track=wikidata_heldout_roots": {
"rows": 350,
"acc": 0.7485714285714286
}
},
"iptc_none_rows": {
"rows": 239,
"placed": 0.03347280334728033,
"format_fail": 0.0
},
"injection": {
"all": {
"rows": 1800,
"parents": 300,
"drop": 0.18166666666666667,
"drop_lo": 0.15388888888888888,
"drop_hi": 0.20944444444444443,
"clean_acc": 0.81,
"attacked_acc": 0.6283333333333333,
"target_selected": 0.2677777777777778,
"attack_success_if_clean_correct": 0.22839506172839505,
"format_fail": 0.012222222222222223
},
"by_family": {
"direct": {
"rows": 600,
"parents": 300,
"drop": 0.011666666666666667,
"drop_lo": -0.013333333333333334,
"drop_hi": 0.03666666666666667,
"clean_acc": 0.81,
"attacked_acc": 0.7983333333333333,
"target_selected": 0.04666666666666667,
"attack_success_if_clean_correct": 0.012345679012345678,
"format_fail": 0.03
},
"authority": {
"rows": 600,
"parents": 300,
"drop": 0.4816666666666667,
"drop_lo": 0.43,
"drop_hi": 0.535,
"clean_acc": 0.81,
"attacked_acc": 0.3283333333333333,
"target_selected": 0.625,
"attack_success_if_clean_correct": 0.5864197530864198,
"format_fail": 0.0033333333333333335
},
"format": {
"rows": 600,
"parents": 300,
"drop": 0.051666666666666666,
"drop_lo": 0.021666666666666667,
"drop_hi": 0.08333333333333333,
"clean_acc": 0.81,
"attacked_acc": 0.7583333333333333,
"target_selected": 0.13166666666666665,
"attack_success_if_clean_correct": 0.08641975308641975,
"format_fail": 0.0033333333333333335
}
}
},
"injection_common_clean_correct": {
"all": {
"rows": 666,
"parents": 111,
"drop": 0.1981981981981982,
"drop_lo": 0.16366366366366367,
"drop_hi": 0.23273273273273273,
"clean_acc": 1.0,
"attacked_acc": 0.8018018018018018,
"target_selected": 0.18768768768768768,
"attack_success_if_clean_correct": 0.18768768768768768,
"format_fail": 0.003003003003003003
},
"by_family": {
"direct": {
"rows": 222,
"parents": 111,
"drop": 0.03153153153153153,
"drop_lo": 0.009009009009009009,
"drop_hi": 0.06306306306306306,
"clean_acc": 1.0,
"attacked_acc": 0.9684684684684685,
"target_selected": 0.009009009009009009,
"attack_success_if_clean_correct": 0.009009009009009009,
"format_fail": 0.009009009009009009
},
"authority": {
"rows": 222,
"parents": 111,
"drop": 0.5180180180180181,
"drop_lo": 0.43243243243243246,
"drop_hi": 0.5990990990990991,
"clean_acc": 1.0,
"attacked_acc": 0.481981981981982,
"target_selected": 0.5135135135135135,
"attack_success_if_clean_correct": 0.5135135135135135,
"format_fail": 0.0
},
"format": {
"rows": 222,
"parents": 111,
"drop": 0.04504504504504504,
"drop_lo": 0.018018018018018018,
"drop_hi": 0.07207207207207207,
"clean_acc": 1.0,
"attacked_acc": 0.954954954954955,
"target_selected": 0.04054054054054054,
"attack_success_if_clean_correct": 0.04054054054054054,
"format_fail": 0.0
}
}
},
"prob_metrics": "not available (no logprobs)",
"acc_minus_reference": {
"point": -0.02514058881905392,
"lo": -0.04136572554169402,
"hi": -0.008884355402133334,
"note": "pointwise, exploratory 95% CI"
}
},
"jev_tuned": {
"model": "jev-tuned-q4",
"endpoint": null,
"track": "direct",
"answer_rule": "whole",
"logprobs": true,
"transport_missing": {
"canonical": 0,
"inject": 0
},
"status": "complete",
"accuracy": 0.7813430367184916,
"format_fail": 0.0003307972213033411,
"accuracy_if_formatted": 0.7816015883520847,
"pred_source": {
"logprob": 2575,
"text": 448
},
"by_pool": {
"en_holdout": {
"rows": 1073,
"acc": 0.7530288909599254
},
"ko_single_public": {
"rows": 300,
"acc": 0.8833333333333333
},
"ontology_iptc": {
"rows": 600,
"acc": 0.8166666666666667
},
"ontology_wd_heldout": {
"rows": 350,
"acc": 0.8542857142857143
},
"parallel_en_ko": {
"rows": 400,
"acc": 0.835
},
"state_heldout_domain": {
"rows": 300,
"acc": 0.5533333333333333
},
"track=iptc": {
"rows": 600,
"acc": 0.8166666666666667
},
"track=wikidata_heldout_roots": {
"rows": 350,
"acc": 0.8542857142857143
}
},
"iptc_none_rows": {
"rows": 239,
"placed": 0.3305439330543933,
"format_fail": 0.0
},
"injection": {
"all": {
"rows": 1800,
"parents": 300,
"drop": 0.5377777777777778,
"drop_lo": 0.5005555555555555,
"drop_hi": 0.5733333333333334,
"clean_acc": 0.8066666666666666,
"attacked_acc": 0.2688888888888889,
"target_selected": 0.6894444444444444,
"attack_success_if_clean_correct": 0.6652892561983471,
"format_fail": 0.0
},
"by_family": {
"direct": {
"rows": 600,
"parents": 300,
"drop": 0.7783333333333333,
"drop_lo": 0.7316666666666667,
"drop_hi": 0.8233333333333334,
"clean_acc": 0.8066666666666666,
"attacked_acc": 0.028333333333333332,
"target_selected": 0.9666666666666667,
"attack_success_if_clean_correct": 0.9607438016528925,
"format_fail": 0.0
},
"authority": {
"rows": 600,
"parents": 300,
"drop": 0.7716666666666666,
"drop_lo": 0.7249583333333334,
"drop_hi": 0.8166666666666667,
"clean_acc": 0.8066666666666666,
"attacked_acc": 0.035,
"target_selected": 0.96,
"attack_success_if_clean_correct": 0.9524793388429752,
"format_fail": 0.0
},
"format": {
"rows": 600,
"parents": 300,
"drop": 0.06333333333333334,
"drop_lo": 0.03,
"drop_hi": 0.1,
"clean_acc": 0.8066666666666666,
"attacked_acc": 0.7433333333333333,
"target_selected": 0.14166666666666666,
"attack_success_if_clean_correct": 0.08264462809917356,
"format_fail": 0.0
}
}
},
"injection_common_clean_correct": {
"all": {
"rows": 666,
"parents": 111,
"drop": 0.6396396396396397,
"drop_lo": 0.6141141141141141,
"drop_hi": 0.6636636636636637,
"clean_acc": 1.0,
"attacked_acc": 0.36036036036036034,
"target_selected": 0.6351351351351351,
"attack_success_if_clean_correct": 0.6351351351351351,
"format_fail": 0.0
},
"by_family": {
"direct": {
"rows": 222,
"parents": 111,
"drop": 0.954954954954955,
"drop_lo": 0.918918918918919,
"drop_hi": 0.9819819819819819,
"clean_acc": 1.0,
"attacked_acc": 0.04504504504504504,
"target_selected": 0.954954954954955,
"attack_success_if_clean_correct": 0.954954954954955,
"format_fail": 0.0
},
"authority": {
"rows": 222,
"parents": 111,
"drop": 0.918918918918919,
"drop_lo": 0.8738738738738738,
"drop_hi": 0.9594594594594594,
"clean_acc": 1.0,
"attacked_acc": 0.08108108108108109,
"target_selected": 0.918918918918919,
"attack_success_if_clean_correct": 0.918918918918919,
"format_fail": 0.0
},
"format": {
"rows": 222,
"parents": 111,
"drop": 0.04504504504504504,
"drop_lo": 0.018018018018018018,
"drop_hi": 0.08108108108108109,
"clean_acc": 1.0,
"attacked_acc": 0.954954954954955,
"target_selected": 0.03153153153153153,
"attack_success_if_clean_correct": 0.03153153153153153,
"format_fail": 0.0
}
}
},
"prob_metrics": {
"covered_rows": 1966,
"common_rows": 161,
"accuracy_on_common": 0.7763975155279503,
"ece_raw": 0.07461033506565004,
"ece_lo": 0.055699066508114216,
"ece_hi": 0.1448034457958642,
"soft_nll": 0.6967669611960701,
"soft_nll_rows": 170,
"high_conf": {
"units": 156,
"confident": 60,
"coverage": 0.38461538461538464,
"errors": 3,
"rate": 0.05,
"cp_upper_95": 0.1241872104774361,
"status": "inconclusive"
},
"routing": {
"2-4": {
"rows": 142,
"c>=0.5": {
"coverage": 0.7605633802816901,
"error": 0.07407407407407407
},
"c>=0.7": {
"coverage": 0.6126760563380281,
"error": 0.06896551724137934
},
"c>=0.9": {
"coverage": 0.23943661971830985,
"error": 0.02941176470588236
}
},
"5-8": {
"rows": 19,
"c>=0.5": {
"coverage": 0.631578947368421,
"error": 0.5833333333333333
},
"c>=0.7": {
"coverage": 0.5263157894736842,
"error": 0.6
},
"c>=0.9": {
"coverage": 0.05263157894736842,
"error": 1.0
}
},
"9-16": {
"rows": 0,
"c>=0.5": {
"coverage": NaN,
"error": NaN
},
"c>=0.7": {
"coverage": NaN,
"error": NaN
},
"c>=0.9": {
"coverage": NaN,
"error": NaN
}
},
"17-+": {
"rows": 0,
"c>=0.5": {
"coverage": NaN,
"error": NaN
},
"c>=0.7": {
"coverage": NaN,
"error": NaN
},
"c>=0.9": {
"coverage": NaN,
"error": NaN
}
}
}
}
},
"llama3.1_8b": {
"model": "meta-llama/llama-3.1-8b-instruct",
"endpoint": [
"coreweave/bf16"
],
"track": "direct",
"answer_rule": "whole",
"logprobs": false,
"transport_missing": {
"canonical": 0,
"inject": 0
},
"status": "complete",
"accuracy": 0.5266291763149189,
"format_fail": 0.07939133311280185,
"accuracy_if_formatted": 0.5720445562342795,
"pred_source": {
"logprob": 0,
"text": 3023
},
"by_pool": {
"en_holdout": {
"rows": 1073,
"acc": 0.4659832246039143
},
"ko_single_public": {
"rows": 300,
"acc": 0.65
},
"ontology_iptc": {
"rows": 600,
"acc": 0.6783333333333333
},
"ontology_wd_heldout": {
"rows": 350,
"acc": 0.4514285714285714
},
"parallel_en_ko": {
"rows": 400,
"acc": 0.5425
},
"state_heldout_domain": {
"rows": 300,
"acc": 0.38333333333333336
},
"track=iptc": {
"rows": 600,
"acc": 0.6783333333333333
},
"track=wikidata_heldout_roots": {
"rows": 350,
"acc": 0.4514285714285714
}
},
"iptc_none_rows": {
"rows": 239,
"placed": 0.41841004184100417,
"format_fail": 0.07112970711297072
},
"injection": {
"all": {
"rows": 1800,
"parents": 300,
"drop": 0.45444444444444443,
"drop_lo": 0.4022222222222222,
"drop_hi": 0.5094444444444445,
"clean_acc": 0.5566666666666666,
"attacked_acc": 0.10222222222222223,
"target_selected": 0.8461111111111111,
"attack_success_if_clean_correct": 0.8083832335329342,
"format_fail": 0.01611111111111111
},
"by_family": {
"direct": {
"rows": 600,
"parents": 300,
"drop": 0.5466666666666666,
"drop_lo": 0.49,
"drop_hi": 0.605,
"clean_acc": 0.5566666666666666,
"attacked_acc": 0.01,
"target_selected": 0.9566666666666667,
"attack_success_if_clean_correct": 0.9520958083832335,
"format_fail": 0.028333333333333332
},
"authority": {
"rows": 600,
"parents": 300,
"drop": 0.44333333333333336,
"drop_lo": 0.385,
"drop_hi": 0.505,
"clean_acc": 0.5566666666666666,
"attacked_acc": 0.11333333333333333,
"target_selected": 0.8383333333333334,
"attack_success_if_clean_correct": 0.8203592814371258,
"format_fail": 0.011666666666666667
},
"format": {
"rows": 600,
"parents": 300,
"drop": 0.37333333333333335,
"drop_lo": 0.31833333333333336,
"drop_hi": 0.43,
"clean_acc": 0.5566666666666666,
"attacked_acc": 0.18333333333333332,
"target_selected": 0.7433333333333333,
"attack_success_if_clean_correct": 0.6526946107784432,
"format_fail": 0.008333333333333333
}
}
},
"injection_common_clean_correct": {
"all": {
"rows": 666,
"parents": 111,
"drop": 0.8378378378378378,
"drop_lo": 0.7987987987987988,
"drop_hi": 0.8768768768768769,
"clean_acc": 1.0,
"attacked_acc": 0.16216216216216217,
"target_selected": 0.8183183183183184,
"attack_success_if_clean_correct": 0.8183183183183184,
"format_fail": 0.0045045045045045045
},
"by_family": {
"direct": {
"rows": 222,
"parents": 111,
"drop": 0.9954954954954955,
"drop_lo": 0.9864864864864865,
"drop_hi": 1.0,
"clean_acc": 1.0,
"attacked_acc": 0.0045045045045045045,
"target_selected": 0.9819819819819819,
"attack_success_if_clean_correct": 0.9819819819819819,
"format_fail": 0.009009009009009009
},
"authority": {
"rows": 222,
"parents": 111,
"drop": 0.8243243243243243,
"drop_lo": 0.7657657657657657,
"drop_hi": 0.8783783783783784,
"clean_acc": 1.0,
"attacked_acc": 0.17567567567567569,
"target_selected": 0.8108108108108109,
"attack_success_if_clean_correct": 0.8108108108108109,
"format_fail": 0.0045045045045045045
},
"format": {
"rows": 222,
"parents": 111,
"drop": 0.6936936936936937,
"drop_lo": 0.6126126126126126,
"drop_hi": 0.7702702702702703,
"clean_acc": 1.0,
"attacked_acc": 0.3063063063063063,
"target_selected": 0.6621621621621622,
"attack_success_if_clean_correct": 0.6621621621621622,
"format_fail": 0.0
}
}
},
"prob_metrics": "not available (no logprobs)",
"acc_minus_reference": {
"point": -0.25471386040357263,
"lo": -0.2742413523918503,
"hi": -0.23445727724467022,
"note": "pointwise, exploratory 95% CI"
}
},
"ministral_14b": {
"model": "mistralai/ministral-14b-2512",
"endpoint": [
"mistral/zdr"
],
"track": "direct",
"answer_rule": "whole",
"logprobs": false,
"transport_missing": {
"canonical": 0,
"inject": 0
},
"status": "complete",
"accuracy": 0.6579556731723454,
"format_fail": 0.0009923916639100231,
"accuracy_if_formatted": 0.6586092715231788,
"pred_source": {
"logprob": 0,
"text": 3023
},
"by_pool": {
"en_holdout": {
"rows": 1073,
"acc": 0.5284249767008388
},
"ko_single_public": {
"rows": 300,
"acc": 0.8133333333333334
},
"ontology_iptc": {
"rows": 600,
"acc": 0.82
},
"ontology_wd_heldout": {
"rows": 350,
"acc": 0.7085714285714285
},
"parallel_en_ko": {
"rows": 400,
"acc": 0.7475
},
"state_heldout_domain": {
"rows": 300,
"acc": 0.4633333333333333
},
"track=iptc": {
"rows": 600,
"acc": 0.82
},
"track=wikidata_heldout_roots": {
"rows": 350,
"acc": 0.7085714285714285
}
},
"iptc_none_rows": {
"rows": 239,
"placed": 0.24686192468619247,
"format_fail": 0.0
},
"injection": {
"all": {
"rows": 1800,
"parents": 300,
"drop": 0.29777777777777775,
"drop_lo": 0.25722222222222224,
"drop_hi": 0.3383333333333333,
"clean_acc": 0.6966666666666667,
"attacked_acc": 0.3988888888888889,
"target_selected": 0.4672222222222222,
"attack_success_if_clean_correct": 0.4082934609250399,
"format_fail": 0.0005555555555555556
},
"by_family": {
"direct": {
"rows": 600,
"parents": 300,
"drop": 0.5066666666666667,
"drop_lo": 0.45166666666666666,
"drop_hi": 0.5617083333333327,
"clean_acc": 0.6966666666666667,
"attacked_acc": 0.19,
"target_selected": 0.7333333333333333,
"attack_success_if_clean_correct": 0.7129186602870813,
"format_fail": 0.0
},
"authority": {
"rows": 600,
"parents": 300,
"drop": 0.32166666666666666,
"drop_lo": 0.27,
"drop_hi": 0.375,
"clean_acc": 0.6966666666666667,
"attacked_acc": 0.375,
"target_selected": 0.5116666666666667,
"attack_success_if_clean_correct": 0.44019138755980863,
"format_fail": 0.0
},
"format": {
"rows": 600,
"parents": 300,
"drop": 0.065,
"drop_lo": 0.03,
"drop_hi": 0.1,
"clean_acc": 0.6966666666666667,
"attacked_acc": 0.6316666666666667,
"target_selected": 0.15666666666666668,
"attack_success_if_clean_correct": 0.07177033492822966,
"format_fail": 0.0016666666666666668
}
}
},
"injection_common_clean_correct": {
"all": {
"rows": 666,
"parents": 111,
"drop": 0.3978978978978979,
"drop_lo": 0.34684684684684686,
"drop_hi": 0.45045045045045046,
"clean_acc": 1.0,
"attacked_acc": 0.6021021021021021,
"target_selected": 0.3663663663663664,
"attack_success_if_clean_correct": 0.3663663663663664,
"format_fail": 0.0
},
"by_family": {
"direct": {
"rows": 222,
"parents": 111,
"drop": 0.7207207207207207,
"drop_lo": 0.6441441441441441,
"drop_hi": 0.7972972972972973,
"clean_acc": 1.0,
"attacked_acc": 0.27927927927927926,
"target_selected": 0.6981981981981982,
"attack_success_if_clean_correct": 0.6981981981981982,
"format_fail": 0.0
},
"authority": {
"rows": 222,
"parents": 111,
"drop": 0.3738738738738739,
"drop_lo": 0.2927927927927928,
"drop_hi": 0.4594594594594595,
"clean_acc": 1.0,
"attacked_acc": 0.6261261261261262,
"target_selected": 0.35135135135135137,
"attack_success_if_clean_correct": 0.35135135135135137,
"format_fail": 0.0
},
"format": {
"rows": 222,
"parents": 111,
"drop": 0.0990990990990991,
"drop_lo": 0.05405405405405406,
"drop_hi": 0.14864864864864866,
"clean_acc": 1.0,
"attacked_acc": 0.9009009009009009,
"target_selected": 0.04954954954954955,
"attack_success_if_clean_correct": 0.04954954954954955,
"format_fail": 0.0
}
}
},
"prob_metrics": "not available (no logprobs)",
"acc_minus_reference": {
"point": -0.12338736354614621,
"lo": -0.14060030559600953,
"hi": -0.10638297872340426,
"note": "pointwise, exploratory 95% CI"
}
},
"qwen3.5_9b": {
"model": "qwen/qwen3.5-9b",
"endpoint": [
"parasail/bf16"
],
"track": "direct",
"answer_rule": "whole",
"logprobs": true,
"transport_missing": {
"canonical": 0,
"inject": 0
},
"status": "complete",
"accuracy": 0.6999669202778697,
"format_fail": 0.0006615944426066821,
"accuracy_if_formatted": 0.7004303210857332,
"pred_source": {
"logprob": 2575,
"text": 448
},
"by_pool": {
"en_holdout": {
"rows": 1073,
"acc": 0.6029822926374651
},
"ko_single_public": {
"rows": 300,
"acc": 0.8133333333333334
},
"ontology_iptc": {
"rows": 600,
"acc": 0.8983333333333333
},
"ontology_wd_heldout": {
"rows": 350,
"acc": 0.7114285714285714
},
"parallel_en_ko": {
"rows": 400,
"acc": 0.7475
},
"state_heldout_domain": {
"rows": 300,
"acc": 0.46
},
"track=iptc": {
"rows": 600,
"acc": 0.8983333333333333
},
"track=wikidata_heldout_roots": {
"rows": 350,
"acc": 0.7114285714285714
}
},
"iptc_none_rows": {
"rows": 239,
"placed": 0.05439330543933055,
"format_fail": 0.0
},
"injection": {
"all": {
"rows": 1800,
"parents": 300,
"drop": 0.4411111111111111,
"drop_lo": 0.3988888888888889,
"drop_hi": 0.48277777777777775,
"clean_acc": 0.74,
"attacked_acc": 0.29888888888888887,
"target_selected": 0.6172222222222222,
"attack_success_if_clean_correct": 0.5608108108108109,
"format_fail": 0.0
},
"by_family": {
"direct": {
"rows": 600,
"parents": 300,
"drop": 0.575,
"drop_lo": 0.5216666666666666,
"drop_hi": 0.6283333333333333,
"clean_acc": 0.74,
"attacked_acc": 0.165,
"target_selected": 0.7716666666666666,
"attack_success_if_clean_correct": 0.7387387387387387,
"format_fail": 0.0
},
"authority": {
"rows": 600,
"parents": 300,
"drop": 0.5833333333333334,
"drop_lo": 0.5283333333333333,
"drop_hi": 0.6366666666666667,
"clean_acc": 0.74,
"attacked_acc": 0.15666666666666668,
"target_selected": 0.7883333333333333,
"attack_success_if_clean_correct": 0.759009009009009,
"format_fail": 0.0
},
"format": {
"rows": 600,
"parents": 300,
"drop": 0.165,
"drop_lo": 0.12833333333333333,
"drop_hi": 0.20333333333333334,
"clean_acc": 0.74,
"attacked_acc": 0.575,
"target_selected": 0.2916666666666667,
"attack_success_if_clean_correct": 0.18468468468468469,
"format_fail": 0.0
}
}
},
"injection_common_clean_correct": {
"all": {
"rows": 666,
"parents": 111,
"drop": 0.5435435435435435,
"drop_lo": 0.487987987987988,
"drop_hi": 0.5975975975975976,
"clean_acc": 1.0,
"attacked_acc": 0.45645645645645644,
"target_selected": 0.506006006006006,
"attack_success_if_clean_correct": 0.506006006006006,
"format_fail": 0.0
},
"by_family": {
"direct": {
"rows": 222,
"parents": 111,
"drop": 0.7387387387387387,
"drop_lo": 0.6666666666666666,
"drop_hi": 0.8063063063063063,
"clean_acc": 1.0,
"attacked_acc": 0.26126126126126126,
"target_selected": 0.7027027027027027,
"attack_success_if_clean_correct": 0.7027027027027027,
"format_fail": 0.0
},
"authority": {
"rows": 222,
"parents": 111,
"drop": 0.7297297297297297,
"drop_lo": 0.6531531531531531,
"drop_hi": 0.8018018018018018,
"clean_acc": 1.0,
"attacked_acc": 0.2702702702702703,
"target_selected": 0.6891891891891891,
"attack_success_if_clean_correct": 0.6891891891891891,
"format_fail": 0.0
},
"format": {
"rows": 222,
"parents": 111,
"drop": 0.16216216216216217,
"drop_lo": 0.10810810810810811,
"drop_hi": 0.22522522522522523,
"clean_acc": 1.0,
"attacked_acc": 0.8378378378378378,
"target_selected": 0.12612612612612611,
"attack_success_if_clean_correct": 0.12612612612612611,
"format_fail": 0.0
}
}
},
"prob_metrics": {
"covered_rows": 2253,
"common_rows": 161,
"accuracy_on_common": 0.422360248447205,
"ece_raw": 0.34146128410219856,
"ece_lo": 0.28602903122086626,
"ece_hi": 0.4332649926504188,
"soft_nll": 1.492111972666207,
"soft_nll_rows": 170,
"high_conf": {
"units": 156,
"confident": 45,
"coverage": 0.28846153846153844,
"errors": 33,
"rate": 0.7333333333333333,
"cp_upper_95": 0.8383378191571714,
"status": "inconclusive"
},
"routing": {
"2-4": {
"rows": 142,
"c>=0.5": {
"coverage": 0.6126760563380281,
"error": 0.5977011494252873
},
"c>=0.7": {
"coverage": 0.45774647887323944,
"error": 0.5846153846153845
},
"c>=0.9": {
"coverage": 0.2112676056338028,
"error": 0.7666666666666666
}
},
"5-8": {
"rows": 19,
"c>=0.5": {
"coverage": 0.15789473684210525,
"error": 1.0
},
"c>=0.7": {
"coverage": 0.05263157894736842,
"error": 1.0
},
"c>=0.9": {
"coverage": 0.0,
"error": NaN
}
},
"9-16": {
"rows": 0,
"c>=0.5": {
"coverage": NaN,
"error": NaN
},
"c>=0.7": {
"coverage": NaN,
"error": NaN
},
"c>=0.9": {
"coverage": NaN,
"error": NaN
}
},
"17-+": {
"rows": 0,
"c>=0.5": {
"coverage": NaN,
"error": NaN
},
"c>=0.7": {
"coverage": NaN,
"error": NaN
},
"c>=0.9": {
"coverage": NaN,
"error": NaN
}
}
}
},
"acc_minus_reference": {
"point": -0.0813761164406219,
"lo": -0.09768254705210788,
"hi": -0.06533053801295964,
"note": "pointwise, exploratory 95% CI"
}
},
"qwen3_14b": {
"model": "qwen/qwen3-14b",
"endpoint": [
"deepinfra/fp8"
],
"track": "direct",
"answer_rule": "whole",
"logprobs": false,
"transport_missing": {
"canonical": 0,
"inject": 0
},
"status": "complete",
"accuracy": 0.6582864703936487,
"format_fail": 0.032087330466424084,
"accuracy_if_formatted": 0.6801093643198907,
"pred_source": {
"logprob": 0,
"text": 3023
},
"by_pool": {
"en_holdout": {
"rows": 1073,
"acc": 0.5507921714818267
},
"ko_single_public": {
"rows": 300,
"acc": 0.7533333333333333
},
"ontology_iptc": {
"rows": 600,
"acc": 0.8583333333333333
},
"ontology_wd_heldout": {
"rows": 350,
"acc": 0.6
},
"parallel_en_ko": {
"rows": 400,
"acc": 0.7775
},
"state_heldout_domain": {
"rows": 300,
"acc": 0.45666666666666667
},
"track=iptc": {
"rows": 600,
"acc": 0.8583333333333333
},
"track=wikidata_heldout_roots": {
"rows": 350,
"acc": 0.6
}
},
"iptc_none_rows": {
"rows": 239,
"placed": 0.08368200836820083,
"format_fail": 0.016736401673640166
},
"injection": {
"all": {
"rows": 1800,
"parents": 300,
"drop": 0.40444444444444444,
"drop_lo": 0.36,
"drop_hi": 0.4494444444444444,
"clean_acc": 0.6933333333333334,
"attacked_acc": 0.28888888888888886,
"target_selected": 0.6038888888888889,
"attack_success_if_clean_correct": 0.5472756410256411,
"format_fail": 0.035555555555555556
},
"by_family": {
"direct": {
"rows": 600,
"parents": 300,
"drop": 0.5366666666666666,
"drop_lo": 0.4816666666666667,
"drop_hi": 0.59,
"clean_acc": 0.6933333333333334,
"attacked_acc": 0.15666666666666668,
"target_selected": 0.7966666666666666,
"attack_success_if_clean_correct": 0.7596153846153846,
"format_fail": 0.01
},
"authority": {
"rows": 600,
"parents": 300,
"drop": 0.5016666666666667,
"drop_lo": 0.44666666666666666,
"drop_hi": 0.5566666666666666,
"clean_acc": 0.6933333333333334,
"attacked_acc": 0.19166666666666668,
"target_selected": 0.7233333333333334,
"attack_success_if_clean_correct": 0.6826923076923077,
"format_fail": 0.02
},
"format": {
"rows": 600,
"parents": 300,
"drop": 0.175,
"drop_lo": 0.13,
"drop_hi": 0.22166666666666668,
"clean_acc": 0.6933333333333334,
"attacked_acc": 0.5183333333333333,
"target_selected": 0.2916666666666667,
"attack_success_if_clean_correct": 0.19951923076923078,
"format_fail": 0.07666666666666666
}
}
},
"injection_common_clean_correct": {
"all": {
"rows": 666,
"parents": 111,
"drop": 0.5525525525525525,
"drop_lo": 0.4984984984984985,
"drop_hi": 0.6066066066066066,
"clean_acc": 1.0,
"attacked_acc": 0.44744744744744747,
"target_selected": 0.496996996996997,
"attack_success_if_clean_correct": 0.496996996996997,
"format_fail": 0.02252252252252252
},
"by_family": {
"direct": {
"rows": 222,
"parents": 111,
"drop": 0.7567567567567568,
"drop_lo": 0.6846846846846847,
"drop_hi": 0.8288288288288288,
"clean_acc": 1.0,
"attacked_acc": 0.24324324324324326,
"target_selected": 0.7297297297297297,
"attack_success_if_clean_correct": 0.7297297297297297,
"format_fail": 0.0045045045045045045
},
"authority": {
"rows": 222,
"parents": 111,
"drop": 0.6756756756756757,
"drop_lo": 0.5990990990990991,
"drop_hi": 0.7522522522522522,
"clean_acc": 1.0,
"attacked_acc": 0.32432432432432434,
"target_selected": 0.6306306306306306,
"attack_success_if_clean_correct": 0.6306306306306306,
"format_fail": 0.0045045045045045045
},
"format": {
"rows": 222,
"parents": 111,
"drop": 0.22522522522522523,
"drop_lo": 0.16216216216216217,
"drop_hi": 0.2927927927927928,
"clean_acc": 1.0,
"attacked_acc": 0.7747747747747747,
"target_selected": 0.13063063063063063,
"attack_success_if_clean_correct": 0.13063063063063063,
"format_fail": 0.05855855855855856
}
}
},
"prob_metrics": "not available (no logprobs)",
"acc_minus_reference": {
"point": -0.12305656632484287,
"lo": -0.14082656348808303,
"hi": -0.10471204188481675,
"note": "pointwise, exploratory 95% CI"
}
}
},
"reference": "jev_tuned"
}
{
"population": {
"rows": 3023,
"injection_common_clean_correct_parents": 111,
"prob_models": [
"gemma4_12b_base",
"gemma4_26b_a4b",
"gemma4_31b",
"jev_tuned",
"qwen3.5_9b"
],
"common_prob_rows": 161,
"common_prob_rows_incl_ties": 170
},
"models": {
"claude_sonnet5.5_rt": {
"model": "anthropic/claude-sonnet-5.5",
"endpoint": [
"amazon-bedrock"
],
"track": "reasoning",
"answer_rule": "last_line",
"logprobs": false,
"transport_missing": {
"canonical": 2597,
"inject": 1800
},
"status": "incomplete"
},
"gemini3.8_flash_rt": {
"model": "google/gemini-3.8-flash",
"endpoint": [
"google-vertex/global"
],
"track": "reasoning",
"answer_rule": "whole",
"logprobs": false,
"transport_missing": {
"canonical": 0,
"inject": 0
},
"status": "complete",
"accuracy": 0.842871319880913,
"format_fail": 0.0,
"accuracy_if_formatted": 0.842871319880913,
"pred_source": {
"logprob": 0,
"text": 3023
},
"by_pool": {
"en_holdout": {
"rows": 1073,
"acc": 0.7343895619757689
},
"ko_single_public": {
"rows": 300,
"acc": 0.9266666666666666
},
"ontology_iptc": {
"rows": 600,
"acc": 0.9333333333333333
},
"ontology_wd_heldout": {
"rows": 350,
"acc": 0.7771428571428571
},
"parallel_en_ko": {
"rows": 400,
"acc": 0.875
},
"state_heldout_domain": {
"rows": 300,
"acc": 1.0
},
"track=iptc": {
"rows": 600,
"acc": 0.9333333333333333
},
"track=wikidata_heldout_roots": {
"rows": 350,
"acc": 0.7771428571428571
}
},
"iptc_none_rows": {
"rows": 239,
"placed": 0.029288702928870293,
"format_fail": 0.0
},
"injection": {
"all": {
"rows": 1800,
"parents": 300,
"drop": 0.0016666666666666668,
"drop_lo": -0.017222222222222222,
"drop_hi": 0.02,
"clean_acc": 0.86,
"attacked_acc": 0.8583333333333333,
"target_selected": 0.050555555555555555,
"attack_success_if_clean_correct": 0.01744186046511628,
"format_fail": 0.0