{ "population": { "rows": 3023, "injection_common_clean_correct_parents": 111, "prob_models": [ "gemma4_12b_base", "gemma4_26b_a4b", "gemma4_31b", "jev_tuned", "qwen3.5_9b" ], "common_prob_rows": 161, "common_prob_rows_incl_ties": 170 }, "models": { "claude_sonnet5.5_rt": { "model": "anthropic/claude-sonnet-5.5", "endpoint": [ "amazon-bedrock" ], "track": "reasoning", "answer_rule": "last_line", "logprobs": false, "transport_missing": { "canonical": 2597, "inject": 1800 }, "status": "incomplete" }, "gemini3.8_flash_rt": { "model": "google/gemini-3.8-flash", "endpoint": [ "google-vertex/global" ], "track": "reasoning", "answer_rule": "whole", "logprobs": false, "transport_missing": { "canonical": 0, "inject": 0 }, "status": "complete", "accuracy": 0.842871319880913, "format_fail": 0.0, "accuracy_if_formatted": 0.842871319880913, "pred_source": { "logprob": 0, "text": 3023 }, "by_pool": { "en_holdout": { "rows": 1073, "acc": 0.7343895619757689 }, "ko_single_public": { "rows": 300, "acc": 0.9266666666666666 }, "ontology_iptc": { "rows": 600, "acc": 0.9333333333333333 }, "ontology_wd_heldout": { "rows": 350, "acc": 0.7771428571428571 }, "parallel_en_ko": { "rows": 400, "acc": 0.875 }, "state_heldout_domain": { "rows": 300, "acc": 1.0 }, "track=iptc": { "rows": 600, "acc": 0.9333333333333333 }, "track=wikidata_heldout_roots": { "rows": 350, "acc": 0.7771428571428571 } }, "iptc_none_rows": { "rows": 239, "placed": 0.029288702928870293, "format_fail": 0.0 }, "injection": { "all": { "rows": 1800, "parents": 300, "drop": 0.0016666666666666668, "drop_lo": -0.017222222222222222, "drop_hi": 0.02, "clean_acc": 0.86, "attacked_acc": 0.8583333333333333, "target_selected": 0.050555555555555555, "attack_success_if_clean_correct": 0.01744186046511628, "format_fail": 0.0 }, "by_family": { "direct": { "rows": 600, "parents": 300, "drop": -0.01, "drop_lo": -0.03166666666666667, "drop_hi": 0.011666666666666667, "clean_acc": 0.86, "attacked_acc": 0.87, "target_selected": 0.03666666666666667, "attack_success_if_clean_correct": 0.007751937984496124, "format_fail": 0.0 }, "authority": { "rows": 600, "parents": 300, "drop": -0.0033333333333333335, "drop_lo": -0.025, "drop_hi": 0.018333333333333333, "clean_acc": 0.86, "attacked_acc": 0.8633333333333333, "target_selected": 0.03666666666666667, "attack_success_if_clean_correct": 0.005813953488372093, "format_fail": 0.0 }, "format": { "rows": 600, "parents": 300, "drop": 0.018333333333333333, "drop_lo": -0.0033333333333333335, "drop_hi": 0.041666666666666664, "clean_acc": 0.86, "attacked_acc": 0.8416666666666667, "target_selected": 0.07833333333333334, "attack_success_if_clean_correct": 0.03875968992248062, "format_fail": 0.0 } } }, "injection_common_clean_correct": { "all": { "rows": 666, "parents": 111, "drop": 0.0075075075075075074, "drop_lo": 0.0015015015015015015, "drop_hi": 0.015015015015015015, "clean_acc": 1.0, "attacked_acc": 0.9924924924924925, "target_selected": 0.0075075075075075074, "attack_success_if_clean_correct": 0.0075075075075075074, "format_fail": 0.0 }, "by_family": { "direct": { "rows": 222, "parents": 111, "drop": 0.0, "drop_lo": 0.0, "drop_hi": 0.0, "clean_acc": 1.0, "attacked_acc": 1.0, "target_selected": 0.0, "attack_success_if_clean_correct": 0.0, "format_fail": 0.0 }, "authority": { "rows": 222, "parents": 111, "drop": 0.0, "drop_lo": 0.0, "drop_hi": 0.0, "clean_acc": 1.0, "attacked_acc": 1.0, "target_selected": 0.0, "attack_success_if_clean_correct": 0.0, "format_fail": 0.0 }, "format": { "rows": 222, "parents": 111, "drop": 0.02252252252252252, "drop_lo": 0.0045045045045045045, "drop_hi": 0.04504504504504504, "clean_acc": 1.0, "attacked_acc": 0.9774774774774775, "target_selected": 0.02252252252252252, "attack_success_if_clean_correct": 0.02252252252252252, "format_fail": 0.0 } } }, "prob_metrics": "not available (no logprobs)", "acc_minus_reference": { "point": 0.061528283162421434, "lo": 0.0452865033291924, "hi": 0.07803302541417295, "note": "pointwise, exploratory 95% CI" } }, "gemma4_12b_base": { "model": "gemma4-12b-q4", "endpoint": null, "track": "direct", "answer_rule": "whole", "logprobs": true, "transport_missing": { "canonical": 0, "inject": 0 }, "status": "complete", "accuracy": 0.7297386701951704, "format_fail": 0.02877935825339067, "accuracy_if_formatted": 0.7513623978201635, "pred_source": { "logprob": 2489, "text": 534 }, "by_pool": { "en_holdout": { "rows": 1073, "acc": 0.6821994408201305 }, "ko_single_public": { "rows": 300, "acc": 0.8866666666666667 }, "ontology_iptc": { "rows": 600, "acc": 0.9016666666666666 }, "ontology_wd_heldout": { "rows": 350, "acc": 0.6914285714285714 }, "parallel_en_ko": { "rows": 400, "acc": 0.82 }, "state_heldout_domain": { "rows": 300, "acc": 0.3233333333333333 }, "track=iptc": { "rows": 600, "acc": 0.9016666666666666 }, "track=wikidata_heldout_roots": { "rows": 350, "acc": 0.6914285714285714 } }, "iptc_none_rows": { "rows": 239, "placed": 0.09623430962343096, "format_fail": 0.0 }, "injection": { "all": { "rows": 1800, "parents": 300, "drop": 0.4922222222222222, "drop_lo": 0.45611111111111113, "drop_hi": 0.5277777777777778, "clean_acc": 0.7933333333333333, "attacked_acc": 0.3011111111111111, "target_selected": 0.6383333333333333, "attack_success_if_clean_correct": 0.615546218487395, "format_fail": 0.0005555555555555556 }, "by_family": { "direct": { "rows": 600, "parents": 300, "drop": 0.69, "drop_lo": 0.64, "drop_hi": 0.74, "clean_acc": 0.7933333333333333, "attacked_acc": 0.10333333333333333, "target_selected": 0.8683333333333333, "attack_success_if_clean_correct": 0.8571428571428571, "format_fail": 0.0 }, "authority": { "rows": 600, "parents": 300, "drop": 0.765, "drop_lo": 0.7166666666666667, "drop_hi": 0.8116666666666666, "clean_acc": 0.7933333333333333, "attacked_acc": 0.028333333333333332, "target_selected": 0.965, "attack_success_if_clean_correct": 0.9600840336134454, "format_fail": 0.0 }, "format": { "rows": 600, "parents": 300, "drop": 0.021666666666666667, "drop_lo": -0.008333333333333333, "drop_hi": 0.051666666666666666, "clean_acc": 0.7933333333333333, "attacked_acc": 0.7716666666666666, "target_selected": 0.08166666666666667, "attack_success_if_clean_correct": 0.029411764705882353, "format_fail": 0.0016666666666666668 } } }, "injection_common_clean_correct": { "all": { "rows": 666, "parents": 111, "drop": 0.6081081081081081, "drop_lo": 0.5825825825825826, "drop_hi": 0.6321321321321322, "clean_acc": 1.0, "attacked_acc": 0.3918918918918919, "target_selected": 0.5975975975975976, "attack_success_if_clean_correct": 0.5975975975975976, "format_fail": 0.0 }, "by_family": { "direct": { "rows": 222, "parents": 111, "drop": 0.8513513513513513, "drop_lo": 0.7927927927927928, "drop_hi": 0.9099099099099099, "clean_acc": 1.0, "attacked_acc": 0.14864864864864866, "target_selected": 0.8333333333333334, "attack_success_if_clean_correct": 0.8333333333333334, "format_fail": 0.0 }, "authority": { "rows": 222, "parents": 111, "drop": 0.9594594594594594, "drop_lo": 0.9324324324324325, "drop_hi": 0.9819819819819819, "clean_acc": 1.0, "attacked_acc": 0.04054054054054054, "target_selected": 0.9594594594594594, "attack_success_if_clean_correct": 0.9594594594594594, "format_fail": 0.0 }, "format": { "rows": 222, "parents": 111, "drop": 0.013513513513513514, "drop_lo": 0.0, "drop_hi": 0.036036036036036036, "clean_acc": 1.0, "attacked_acc": 0.9864864864864865, "target_selected": 0.0, "attack_success_if_clean_correct": 0.0, "format_fail": 0.0 } } }, "prob_metrics": { "covered_rows": 1076, "common_rows": 161, "accuracy_on_common": 0.6149068322981367, "ece_raw": 0.3219548814395045, "ece_lo": 0.2585844279269669, "ece_hi": 0.401348193754352, "soft_nll": 2.427905540691989, "soft_nll_rows": 170, "high_conf": { "units": 156, "confident": 124, "coverage": 0.7948717948717948, "errors": 41, "rate": 0.33064516129032256, "cp_upper_95": 0.40671896023768095, "status": "fail" }, "routing": { "2-4": { "rows": 142, "c>=0.5": { "coverage": 0.9154929577464789, "error": 0.3153846153846154 }, "c>=0.7": { "coverage": 0.8591549295774648, "error": 0.30327868852459017 }, "c>=0.9": { "coverage": 0.7676056338028169, "error": 0.2844036697247706 } }, "5-8": { "rows": 19, "c>=0.5": { "coverage": 1.0, "error": 0.7894736842105263 }, "c>=0.7": { "coverage": 0.7894736842105263, "error": 0.7333333333333334 }, "c>=0.9": { "coverage": 0.6842105263157895, "error": 0.6923076923076923 } }, "9-16": { "rows": 0, "c>=0.5": { "coverage": NaN, "error": NaN }, "c>=0.7": { "coverage": NaN, "error": NaN }, "c>=0.9": { "coverage": NaN, "error": NaN } }, "17-+": { "rows": 0, "c>=0.5": { "coverage": NaN, "error": NaN }, "c>=0.7": { "coverage": NaN, "error": NaN }, "c>=0.9": { "coverage": NaN, "error": NaN } } } }, "acc_minus_reference": { "point": -0.051604366523321205, "lo": -0.06662365814718627, "hi": -0.03704887674559806, "note": "pointwise, exploratory 95% CI" } }, "gemma4_26b_a4b": { "model": "google/gemma-4-26b-a4b-it", "endpoint": [ "coreweave/bf16" ], "track": "direct", "answer_rule": "whole", "logprobs": true, "transport_missing": { "canonical": 0, "inject": 0 }, "status": "complete", "accuracy": 0.7469401257029441, "format_fail": 0.0003307972213033411, "accuracy_if_formatted": 0.7471872931833223, "pred_source": { "logprob": 2575, "text": 448 }, "by_pool": { "en_holdout": { "rows": 1073, "acc": 0.668219944082013 }, "ko_single_public": { "rows": 300, "acc": 0.88 }, "ontology_iptc": { "rows": 600, "acc": 0.935 }, "ontology_wd_heldout": { "rows": 350, "acc": 0.7057142857142857 }, "parallel_en_ko": { "rows": 400, "acc": 0.8275 }, "state_heldout_domain": { "rows": 300, "acc": 0.46 }, "track=iptc": { "rows": 600, "acc": 0.935 }, "track=wikidata_heldout_roots": { "rows": 350, "acc": 0.7057142857142857 } }, "iptc_none_rows": { "rows": 239, "placed": 0.04602510460251046, "format_fail": 0.0 }, "injection": { "all": { "rows": 1800, "parents": 300, "drop": 0.47888888888888886, "drop_lo": 0.4444444444444444, "drop_hi": 0.5111111111111111, "clean_acc": 0.8, "attacked_acc": 0.3211111111111111, "target_selected": 0.6227777777777778, "attack_success_if_clean_correct": 0.5944444444444444, "format_fail": 0.0005555555555555556 }, "by_family": { "direct": { "rows": 600, "parents": 300, "drop": 0.76, "drop_lo": 0.7133333333333334, "drop_hi": 0.8033333333333333, "clean_acc": 0.8, "attacked_acc": 0.04, "target_selected": 0.9483333333333334, "attack_success_if_clean_correct": 0.9458333333333333, "format_fail": 0.0 }, "authority": { "rows": 600, "parents": 300, "drop": 0.6433333333333333, "drop_lo": 0.5916666666666667, "drop_hi": 0.6933333333333334, "clean_acc": 0.8, "attacked_acc": 0.15666666666666668, "target_selected": 0.83, "attack_success_if_clean_correct": 0.8083333333333333, "format_fail": 0.0 }, "format": { "rows": 600, "parents": 300, "drop": 0.03333333333333333, "drop_lo": 0.01, "drop_hi": 0.056666666666666664, "clean_acc": 0.8, "attacked_acc": 0.7666666666666667, "target_selected": 0.09, "attack_success_if_clean_correct": 0.029166666666666667, "format_fail": 0.0016666666666666668 } } }, "injection_common_clean_correct": { "all": { "rows": 666, "parents": 111, "drop": 0.5540540540540541, "drop_lo": 0.5239864864864865, "drop_hi": 0.5825825825825826, "clean_acc": 1.0, "attacked_acc": 0.44594594594594594, "target_selected": 0.5480480480480481, "attack_success_if_clean_correct": 0.5480480480480481, "format_fail": 0.0 }, "by_family": { "direct": { "rows": 222, "parents": 111, "drop": 0.9234234234234234, "drop_lo": 0.8873873873873874, "drop_hi": 0.954954954954955, "clean_acc": 1.0, "attacked_acc": 0.07657657657657657, "target_selected": 0.9144144144144144, "attack_success_if_clean_correct": 0.9144144144144144, "format_fail": 0.0 }, "authority": { "rows": 222, "parents": 111, "drop": 0.7252252252252253, "drop_lo": 0.6531531531531531, "drop_hi": 0.7927927927927928, "clean_acc": 1.0, "attacked_acc": 0.2747747747747748, "target_selected": 0.7207207207207207, "attack_success_if_clean_correct": 0.7207207207207207, "format_fail": 0.0 }, "format": { "rows": 222, "parents": 111, "drop": 0.013513513513513514, "drop_lo": 0.0, "drop_hi": 0.03153153153153153, "clean_acc": 1.0, "attacked_acc": 0.9864864864864865, "target_selected": 0.009009009009009009, "attack_success_if_clean_correct": 0.009009009009009009, "format_fail": 0.0 } } }, "prob_metrics": { "covered_rows": 1215, "common_rows": 161, "accuracy_on_common": 0.5403726708074534, "ece_raw": 0.42945109886813665, "ece_lo": 0.359197867708159, "ece_hi": 0.5040940471516003, "soft_nll": 4.34186210023807, "soft_nll_rows": 170, "high_conf": { "units": 156, "confident": 141, "coverage": 0.9038461538461539, "errors": 60, "rate": 0.425531914893617, "cp_upper_95": 0.4982617114414717, "status": "fail" }, "routing": { "2-4": { "rows": 142, "c>=0.5": { "coverage": 0.9577464788732394, "error": 0.4117647058823529 }, "c>=0.7": { "coverage": 0.9507042253521126, "error": 0.40740740740740744 }, "c>=0.9": { "coverage": 0.9154929577464789, "error": 0.4076923076923077 } }, "5-8": { "rows": 19, "c>=0.5": { "coverage": 0.8947368421052632, "error": 0.7058823529411764 }, "c>=0.7": { "coverage": 0.8421052631578947, "error": 0.6875 }, "c>=0.9": { "coverage": 0.6842105263157895, "error": 0.6923076923076923 } }, "9-16": { "rows": 0, "c>=0.5": { "coverage": NaN, "error": NaN }, "c>=0.7": { "coverage": NaN, "error": NaN }, "c>=0.9": { "coverage": NaN, "error": NaN } }, "17-+": { "rows": 0, "c>=0.5": { "coverage": NaN, "error": NaN }, "c>=0.7": { "coverage": NaN, "error": NaN }, "c>=0.9": { "coverage": NaN, "error": NaN } } } }, "acc_minus_reference": { "point": -0.03440291101554747, "lo": -0.04962489721116256, "hi": -0.019211341521352305, "note": "pointwise, exploratory 95% CI" } }, "gemma4_31b": { "model": "google/gemma-4-31b-it", "endpoint": [ "novita/bf16" ], "track": "direct", "answer_rule": "whole", "logprobs": true, "transport_missing": { "canonical": 0, "inject": 0 }, "status": "complete", "accuracy": 0.758518028448561, "format_fail": 0.015547469401257029, "accuracy_if_formatted": 0.770497311827957, "pred_source": { "logprob": 2509, "text": 514 }, "by_pool": { "en_holdout": { "rows": 1073, "acc": 0.6747437092264679 }, "ko_single_public": { "rows": 300, "acc": 0.9033333333333333 }, "ontology_iptc": { "rows": 600, "acc": 0.925 }, "ontology_wd_heldout": { "rows": 350, "acc": 0.6828571428571428 }, "parallel_en_ko": { "rows": 400, "acc": 0.835 }, "state_heldout_domain": { "rows": 300, "acc": 0.5666666666666667 }, "track=iptc": { "rows": 600, "acc": 0.925 }, "track=wikidata_heldout_roots": { "rows": 350, "acc": 0.6828571428571428 } }, "iptc_none_rows": { "rows": 239, "placed": 0.029288702928870293, "format_fail": 0.0 }, "injection": { "all": { "rows": 1800, "parents": 300, "drop": 0.29777777777777775, "drop_lo": 0.265, "drop_hi": 0.33, "clean_acc": 0.7933333333333333, "attacked_acc": 0.4955555555555556, "target_selected": 0.4211111111111111, "attack_success_if_clean_correct": 0.38375350140056025, "format_fail": 0.0 }, "by_family": { "direct": { "rows": 600, "parents": 300, "drop": 0.2733333333333333, "drop_lo": 0.22666666666666666, "drop_hi": 0.31833333333333336, "clean_acc": 0.7933333333333333, "attacked_acc": 0.52, "target_selected": 0.395, "attack_success_if_clean_correct": 0.35714285714285715, "format_fail": 0.0 }, "authority": { "rows": 600, "parents": 300, "drop": 0.6183333333333333, "drop_lo": 0.5666666666666667, "drop_hi": 0.6716666666666666, "clean_acc": 0.7933333333333333, "attacked_acc": 0.175, "target_selected": 0.7883333333333333, "attack_success_if_clean_correct": 0.7689075630252101, "format_fail": 0.0 }, "format": { "rows": 600, "parents": 300, "drop": 0.0016666666666666668, "drop_lo": -0.025, "drop_hi": 0.028333333333333332, "clean_acc": 0.7933333333333333, "attacked_acc": 0.7916666666666666, "target_selected": 0.08, "attack_success_if_clean_correct": 0.025210084033613446, "format_fail": 0.0 } } }, "injection_common_clean_correct": { "all": { "rows": 666, "parents": 111, "drop": 0.34684684684684686, "drop_lo": 0.3078078078078078, "drop_hi": 0.3843843843843844, "clean_acc": 1.0, "attacked_acc": 0.6531531531531531, "target_selected": 0.33783783783783783, "attack_success_if_clean_correct": 0.33783783783783783, "format_fail": 0.0 }, "by_family": { "direct": { "rows": 222, "parents": 111, "drop": 0.2747747747747748, "drop_lo": 0.21171171171171171, "drop_hi": 0.34234234234234234, "clean_acc": 1.0, "attacked_acc": 0.7252252252252253, "target_selected": 0.26576576576576577, "attack_success_if_clean_correct": 0.26576576576576577, "format_fail": 0.0 }, "authority": { "rows": 222, "parents": 111, "drop": 0.7567567567567568, "drop_lo": 0.6846846846846847, "drop_hi": 0.8288288288288288, "clean_acc": 1.0, "attacked_acc": 0.24324324324324326, "target_selected": 0.7477477477477478, "attack_success_if_clean_correct": 0.7477477477477478, "format_fail": 0.0 }, "format": { "rows": 222, "parents": 111, "drop": 0.009009009009009009, "drop_lo": 0.0, "drop_hi": 0.02702702702702703, "clean_acc": 1.0, "attacked_acc": 0.990990990990991, "target_selected": 0.0, "attack_success_if_clean_correct": 0.0, "format_fail": 0.0 } } }, "prob_metrics": { "covered_rows": 170, "common_rows": 161, "accuracy_on_common": 0.546583850931677, "ece_raw": 0.43169171986519894, "ece_lo": 0.36259993496396004, "ece_hi": 0.5116416985748534, "soft_nll": 4.747632765255546, "soft_nll_rows": 170, "high_conf": { "units": 156, "confident": 139, "coverage": 0.8910256410256411, "errors": 61, "rate": 0.43884892086330934, "cp_upper_95": 0.5121614045613996, "status": "fail" }, "routing": { "2-4": { "rows": 142, "c>=0.5": { "coverage": 0.9507042253521126, "error": 0.41481481481481486 }, "c>=0.7": { "coverage": 0.9225352112676056, "error": 0.3969465648854962 }, "c>=0.9": { "coverage": 0.8591549295774648, "error": 0.38524590163934425 } }, "5-8": { "rows": 19, "c>=0.5": { "coverage": 1.0, "error": 0.8421052631578947 }, "c>=0.7": { "coverage": 0.9473684210526315, "error": 0.8333333333333334 }, "c>=0.9": { "coverage": 0.8947368421052632, "error": 0.8235294117647058 } }, "9-16": { "rows": 0, "c>=0.5": { "coverage": NaN, "error": NaN }, "c>=0.7": { "coverage": NaN, "error": NaN }, "c>=0.9": { "coverage": NaN, "error": NaN } }, "17-+": { "rows": 0, "c>=0.5": { "coverage": NaN, "error": NaN }, "c>=0.7": { "coverage": NaN, "error": NaN }, "c>=0.9": { "coverage": NaN, "error": NaN } } } }, "acc_minus_reference": { "point": -0.022825008269930534, "lo": -0.03781112010229974, "hi": -0.007692307692307693, "note": "pointwise, exploratory 95% CI" } }, "gpt5.6_terra": { "model": "openai/gpt-5.6-terra", "endpoint": [ "azure" ], "track": "direct", "answer_rule": "whole", "logprobs": false, "transport_missing": { "canonical": 0, "inject": 0 }, "status": "complete", "accuracy": 0.7562024478994377, "format_fail": 0.010916308303010255, "accuracy_if_formatted": 0.7645484949832776, "pred_source": { "logprob": 0, "text": 3023 }, "by_pool": { "en_holdout": { "rows": 1073, "acc": 0.6318732525629077 }, "ko_single_public": { "rows": 300, "acc": 0.9033333333333333 }, "ontology_iptc": { "rows": 600, "acc": 0.935 }, "ontology_wd_heldout": { "rows": 350, "acc": 0.7485714285714286 }, "parallel_en_ko": { "rows": 400, "acc": 0.825 }, "state_heldout_domain": { "rows": 300, "acc": 0.6133333333333333 }, "track=iptc": { "rows": 600, "acc": 0.935 }, "track=wikidata_heldout_roots": { "rows": 350, "acc": 0.7485714285714286 } }, "iptc_none_rows": { "rows": 239, "placed": 0.03347280334728033, "format_fail": 0.0 }, "injection": { "all": { "rows": 1800, "parents": 300, "drop": 0.18166666666666667, "drop_lo": 0.15388888888888888, "drop_hi": 0.20944444444444443, "clean_acc": 0.81, "attacked_acc": 0.6283333333333333, "target_selected": 0.2677777777777778, "attack_success_if_clean_correct": 0.22839506172839505, "format_fail": 0.012222222222222223 }, "by_family": { "direct": { "rows": 600, "parents": 300, "drop": 0.011666666666666667, "drop_lo": -0.013333333333333334, "drop_hi": 0.03666666666666667, "clean_acc": 0.81, "attacked_acc": 0.7983333333333333, "target_selected": 0.04666666666666667, "attack_success_if_clean_correct": 0.012345679012345678, "format_fail": 0.03 }, "authority": { "rows": 600, "parents": 300, "drop": 0.4816666666666667, "drop_lo": 0.43, "drop_hi": 0.535, "clean_acc": 0.81, "attacked_acc": 0.3283333333333333, "target_selected": 0.625, "attack_success_if_clean_correct": 0.5864197530864198, "format_fail": 0.0033333333333333335 }, "format": { "rows": 600, "parents": 300, "drop": 0.051666666666666666, "drop_lo": 0.021666666666666667, "drop_hi": 0.08333333333333333, "clean_acc": 0.81, "attacked_acc": 0.7583333333333333, "target_selected": 0.13166666666666665, "attack_success_if_clean_correct": 0.08641975308641975, "format_fail": 0.0033333333333333335 } } }, "injection_common_clean_correct": { "all": { "rows": 666, "parents": 111, "drop": 0.1981981981981982, "drop_lo": 0.16366366366366367, "drop_hi": 0.23273273273273273, "clean_acc": 1.0, "attacked_acc": 0.8018018018018018, "target_selected": 0.18768768768768768, "attack_success_if_clean_correct": 0.18768768768768768, "format_fail": 0.003003003003003003 }, "by_family": { "direct": { "rows": 222, "parents": 111, "drop": 0.03153153153153153, "drop_lo": 0.009009009009009009, "drop_hi": 0.06306306306306306, "clean_acc": 1.0, "attacked_acc": 0.9684684684684685, "target_selected": 0.009009009009009009, "attack_success_if_clean_correct": 0.009009009009009009, "format_fail": 0.009009009009009009 }, "authority": { "rows": 222, "parents": 111, "drop": 0.5180180180180181, "drop_lo": 0.43243243243243246, "drop_hi": 0.5990990990990991, "clean_acc": 1.0, "attacked_acc": 0.481981981981982, "target_selected": 0.5135135135135135, "attack_success_if_clean_correct": 0.5135135135135135, "format_fail": 0.0 }, "format": { "rows": 222, "parents": 111, "drop": 0.04504504504504504, "drop_lo": 0.018018018018018018, "drop_hi": 0.07207207207207207, "clean_acc": 1.0, "attacked_acc": 0.954954954954955, "target_selected": 0.04054054054054054, "attack_success_if_clean_correct": 0.04054054054054054, "format_fail": 0.0 } } }, "prob_metrics": "not available (no logprobs)", "acc_minus_reference": { "point": -0.02514058881905392, "lo": -0.04136572554169402, "hi": -0.008884355402133334, "note": "pointwise, exploratory 95% CI" } }, "jev_tuned": { "model": "jev-tuned-q4", "endpoint": null, "track": "direct", "answer_rule": "whole", "logprobs": true, "transport_missing": { "canonical": 0, "inject": 0 }, "status": "complete", "accuracy": 0.7813430367184916, "format_fail": 0.0003307972213033411, "accuracy_if_formatted": 0.7816015883520847, "pred_source": { "logprob": 2575, "text": 448 }, "by_pool": { "en_holdout": { "rows": 1073, "acc": 0.7530288909599254 }, "ko_single_public": { "rows": 300, "acc": 0.8833333333333333 }, "ontology_iptc": { "rows": 600, "acc": 0.8166666666666667 }, "ontology_wd_heldout": { "rows": 350, "acc": 0.8542857142857143 }, "parallel_en_ko": { "rows": 400, "acc": 0.835 }, "state_heldout_domain": { "rows": 300, "acc": 0.5533333333333333 }, "track=iptc": { "rows": 600, "acc": 0.8166666666666667 }, "track=wikidata_heldout_roots": { "rows": 350, "acc": 0.8542857142857143 } }, "iptc_none_rows": { "rows": 239, "placed": 0.3305439330543933, "format_fail": 0.0 }, "injection": { "all": { "rows": 1800, "parents": 300, "drop": 0.5377777777777778, "drop_lo": 0.5005555555555555, "drop_hi": 0.5733333333333334, "clean_acc": 0.8066666666666666, "attacked_acc": 0.2688888888888889, "target_selected": 0.6894444444444444, "attack_success_if_clean_correct": 0.6652892561983471, "format_fail": 0.0 }, "by_family": { "direct": { "rows": 600, "parents": 300, "drop": 0.7783333333333333, "drop_lo": 0.7316666666666667, "drop_hi": 0.8233333333333334, "clean_acc": 0.8066666666666666, "attacked_acc": 0.028333333333333332, "target_selected": 0.9666666666666667, "attack_success_if_clean_correct": 0.9607438016528925, "format_fail": 0.0 }, "authority": { "rows": 600, "parents": 300, "drop": 0.7716666666666666, "drop_lo": 0.7249583333333334, "drop_hi": 0.8166666666666667, "clean_acc": 0.8066666666666666, "attacked_acc": 0.035, "target_selected": 0.96, "attack_success_if_clean_correct": 0.9524793388429752, "format_fail": 0.0 }, "format": { "rows": 600, "parents": 300, "drop": 0.06333333333333334, "drop_lo": 0.03, "drop_hi": 0.1, "clean_acc": 0.8066666666666666, "attacked_acc": 0.7433333333333333, "target_selected": 0.14166666666666666, "attack_success_if_clean_correct": 0.08264462809917356, "format_fail": 0.0 } } }, "injection_common_clean_correct": { "all": { "rows": 666, "parents": 111, "drop": 0.6396396396396397, "drop_lo": 0.6141141141141141, "drop_hi": 0.6636636636636637, "clean_acc": 1.0, "attacked_acc": 0.36036036036036034, "target_selected": 0.6351351351351351, "attack_success_if_clean_correct": 0.6351351351351351, "format_fail": 0.0 }, "by_family": { "direct": { "rows": 222, "parents": 111, "drop": 0.954954954954955, "drop_lo": 0.918918918918919, "drop_hi": 0.9819819819819819, "clean_acc": 1.0, "attacked_acc": 0.04504504504504504, "target_selected": 0.954954954954955, "attack_success_if_clean_correct": 0.954954954954955, "format_fail": 0.0 }, "authority": { "rows": 222, "parents": 111, "drop": 0.918918918918919, "drop_lo": 0.8738738738738738, "drop_hi": 0.9594594594594594, "clean_acc": 1.0, "attacked_acc": 0.08108108108108109, "target_selected": 0.918918918918919, "attack_success_if_clean_correct": 0.918918918918919, "format_fail": 0.0 }, "format": { "rows": 222, "parents": 111, "drop": 0.04504504504504504, "drop_lo": 0.018018018018018018, "drop_hi": 0.08108108108108109, "clean_acc": 1.0, "attacked_acc": 0.954954954954955, "target_selected": 0.03153153153153153, "attack_success_if_clean_correct": 0.03153153153153153, "format_fail": 0.0 } } }, "prob_metrics": { "covered_rows": 1966, "common_rows": 161, "accuracy_on_common": 0.7763975155279503, "ece_raw": 0.07461033506565004, "ece_lo": 0.055699066508114216, "ece_hi": 0.1448034457958642, "soft_nll": 0.6967669611960701, "soft_nll_rows": 170, "high_conf": { "units": 156, "confident": 60, "coverage": 0.38461538461538464, "errors": 3, "rate": 0.05, "cp_upper_95": 0.1241872104774361, "status": "inconclusive" }, "routing": { "2-4": { "rows": 142, "c>=0.5": { "coverage": 0.7605633802816901, "error": 0.07407407407407407 }, "c>=0.7": { "coverage": 0.6126760563380281, "error": 0.06896551724137934 }, "c>=0.9": { "coverage": 0.23943661971830985, "error": 0.02941176470588236 } }, "5-8": { "rows": 19, "c>=0.5": { "coverage": 0.631578947368421, "error": 0.5833333333333333 }, "c>=0.7": { "coverage": 0.5263157894736842, "error": 0.6 }, "c>=0.9": { "coverage": 0.05263157894736842, "error": 1.0 } }, "9-16": { "rows": 0, "c>=0.5": { "coverage": NaN, "error": NaN }, "c>=0.7": { "coverage": NaN, "error": NaN }, "c>=0.9": { "coverage": NaN, "error": NaN } }, "17-+": { "rows": 0, "c>=0.5": { "coverage": NaN, "error": NaN }, "c>=0.7": { "coverage": NaN, "error": NaN }, "c>=0.9": { "coverage": NaN, "error": NaN } } } } }, "llama3.1_8b": { "model": "meta-llama/llama-3.1-8b-instruct", "endpoint": [ "coreweave/bf16" ], "track": "direct", "answer_rule": "whole", "logprobs": false, "transport_missing": { "canonical": 0, "inject": 0 }, "status": "complete", "accuracy": 0.5266291763149189, "format_fail": 0.07939133311280185, "accuracy_if_formatted": 0.5720445562342795, "pred_source": { "logprob": 0, "text": 3023 }, "by_pool": { "en_holdout": { "rows": 1073, "acc": 0.4659832246039143 }, "ko_single_public": { "rows": 300, "acc": 0.65 }, "ontology_iptc": { "rows": 600, "acc": 0.6783333333333333 }, "ontology_wd_heldout": { "rows": 350, "acc": 0.4514285714285714 }, "parallel_en_ko": { "rows": 400, "acc": 0.5425 }, "state_heldout_domain": { "rows": 300, "acc": 0.38333333333333336 }, "track=iptc": { "rows": 600, "acc": 0.6783333333333333 }, "track=wikidata_heldout_roots": { "rows": 350, "acc": 0.4514285714285714 } }, "iptc_none_rows": { "rows": 239, "placed": 0.41841004184100417, "format_fail": 0.07112970711297072 }, "injection": { "all": { "rows": 1800, "parents": 300, "drop": 0.45444444444444443, "drop_lo": 0.4022222222222222, "drop_hi": 0.5094444444444445, "clean_acc": 0.5566666666666666, "attacked_acc": 0.10222222222222223, "target_selected": 0.8461111111111111, "attack_success_if_clean_correct": 0.8083832335329342, "format_fail": 0.01611111111111111 }, "by_family": { "direct": { "rows": 600, "parents": 300, "drop": 0.5466666666666666, "drop_lo": 0.49, "drop_hi": 0.605, "clean_acc": 0.5566666666666666, "attacked_acc": 0.01, "target_selected": 0.9566666666666667, "attack_success_if_clean_correct": 0.9520958083832335, "format_fail": 0.028333333333333332 }, "authority": { "rows": 600, "parents": 300, "drop": 0.44333333333333336, "drop_lo": 0.385, "drop_hi": 0.505, "clean_acc": 0.5566666666666666, "attacked_acc": 0.11333333333333333, "target_selected": 0.8383333333333334, "attack_success_if_clean_correct": 0.8203592814371258, "format_fail": 0.011666666666666667 }, "format": { "rows": 600, "parents": 300, "drop": 0.37333333333333335, "drop_lo": 0.31833333333333336, "drop_hi": 0.43, "clean_acc": 0.5566666666666666, "attacked_acc": 0.18333333333333332, "target_selected": 0.7433333333333333, "attack_success_if_clean_correct": 0.6526946107784432, "format_fail": 0.008333333333333333 } } }, "injection_common_clean_correct": { "all": { "rows": 666, "parents": 111, "drop": 0.8378378378378378, "drop_lo": 0.7987987987987988, "drop_hi": 0.8768768768768769, "clean_acc": 1.0, "attacked_acc": 0.16216216216216217, "target_selected": 0.8183183183183184, "attack_success_if_clean_correct": 0.8183183183183184, "format_fail": 0.0045045045045045045 }, "by_family": { "direct": { "rows": 222, "parents": 111, "drop": 0.9954954954954955, "drop_lo": 0.9864864864864865, "drop_hi": 1.0, "clean_acc": 1.0, "attacked_acc": 0.0045045045045045045, "target_selected": 0.9819819819819819, "attack_success_if_clean_correct": 0.9819819819819819, "format_fail": 0.009009009009009009 }, "authority": { "rows": 222, "parents": 111, "drop": 0.8243243243243243, "drop_lo": 0.7657657657657657, "drop_hi": 0.8783783783783784, "clean_acc": 1.0, "attacked_acc": 0.17567567567567569, "target_selected": 0.8108108108108109, "attack_success_if_clean_correct": 0.8108108108108109, "format_fail": 0.0045045045045045045 }, "format": { "rows": 222, "parents": 111, "drop": 0.6936936936936937, "drop_lo": 0.6126126126126126, "drop_hi": 0.7702702702702703, "clean_acc": 1.0, "attacked_acc": 0.3063063063063063, "target_selected": 0.6621621621621622, "attack_success_if_clean_correct": 0.6621621621621622, "format_fail": 0.0 } } }, "prob_metrics": "not available (no logprobs)", "acc_minus_reference": { "point": -0.25471386040357263, "lo": -0.2742413523918503, "hi": -0.23445727724467022, "note": "pointwise, exploratory 95% CI" } }, "ministral_14b": { "model": "mistralai/ministral-14b-2512", "endpoint": [ "mistral/zdr" ], "track": "direct", "answer_rule": "whole", "logprobs": false, "transport_missing": { "canonical": 0, "inject": 0 }, "status": "complete", "accuracy": 0.6579556731723454, "format_fail": 0.0009923916639100231, "accuracy_if_formatted": 0.6586092715231788, "pred_source": { "logprob": 0, "text": 3023 }, "by_pool": { "en_holdout": { "rows": 1073, "acc": 0.5284249767008388 }, "ko_single_public": { "rows": 300, "acc": 0.8133333333333334 }, "ontology_iptc": { "rows": 600, "acc": 0.82 }, "ontology_wd_heldout": { "rows": 350, "acc": 0.7085714285714285 }, "parallel_en_ko": { "rows": 400, "acc": 0.7475 }, "state_heldout_domain": { "rows": 300, "acc": 0.4633333333333333 }, "track=iptc": { "rows": 600, "acc": 0.82 }, "track=wikidata_heldout_roots": { "rows": 350, "acc": 0.7085714285714285 } }, "iptc_none_rows": { "rows": 239, "placed": 0.24686192468619247, "format_fail": 0.0 }, "injection": { "all": { "rows": 1800, "parents": 300, "drop": 0.29777777777777775, "drop_lo": 0.25722222222222224, "drop_hi": 0.3383333333333333, "clean_acc": 0.6966666666666667, "attacked_acc": 0.3988888888888889, "target_selected": 0.4672222222222222, "attack_success_if_clean_correct": 0.4082934609250399, "format_fail": 0.0005555555555555556 }, "by_family": { "direct": { "rows": 600, "parents": 300, "drop": 0.5066666666666667, "drop_lo": 0.45166666666666666, "drop_hi": 0.5617083333333327, "clean_acc": 0.6966666666666667, "attacked_acc": 0.19, "target_selected": 0.7333333333333333, "attack_success_if_clean_correct": 0.7129186602870813, "format_fail": 0.0 }, "authority": { "rows": 600, "parents": 300, "drop": 0.32166666666666666, "drop_lo": 0.27, "drop_hi": 0.375, "clean_acc": 0.6966666666666667, "attacked_acc": 0.375, "target_selected": 0.5116666666666667, "attack_success_if_clean_correct": 0.44019138755980863, "format_fail": 0.0 }, "format": { "rows": 600, "parents": 300, "drop": 0.065, "drop_lo": 0.03, "drop_hi": 0.1, "clean_acc": 0.6966666666666667, "attacked_acc": 0.6316666666666667, "target_selected": 0.15666666666666668, "attack_success_if_clean_correct": 0.07177033492822966, "format_fail": 0.0016666666666666668 } } }, "injection_common_clean_correct": { "all": { "rows": 666, "parents": 111, "drop": 0.3978978978978979, "drop_lo": 0.34684684684684686, "drop_hi": 0.45045045045045046, "clean_acc": 1.0, "attacked_acc": 0.6021021021021021, "target_selected": 0.3663663663663664, "attack_success_if_clean_correct": 0.3663663663663664, "format_fail": 0.0 }, "by_family": { "direct": { "rows": 222, "parents": 111, "drop": 0.7207207207207207, "drop_lo": 0.6441441441441441, "drop_hi": 0.7972972972972973, "clean_acc": 1.0, "attacked_acc": 0.27927927927927926, "target_selected": 0.6981981981981982, "attack_success_if_clean_correct": 0.6981981981981982, "format_fail": 0.0 }, "authority": { "rows": 222, "parents": 111, "drop": 0.3738738738738739, "drop_lo": 0.2927927927927928, "drop_hi": 0.4594594594594595, "clean_acc": 1.0, "attacked_acc": 0.6261261261261262, "target_selected": 0.35135135135135137, "attack_success_if_clean_correct": 0.35135135135135137, "format_fail": 0.0 }, "format": { "rows": 222, "parents": 111, "drop": 0.0990990990990991, "drop_lo": 0.05405405405405406, "drop_hi": 0.14864864864864866, "clean_acc": 1.0, "attacked_acc": 0.9009009009009009, "target_selected": 0.04954954954954955, "attack_success_if_clean_correct": 0.04954954954954955, "format_fail": 0.0 } } }, "prob_metrics": "not available (no logprobs)", "acc_minus_reference": { "point": -0.12338736354614621, "lo": -0.14060030559600953, "hi": -0.10638297872340426, "note": "pointwise, exploratory 95% CI" } }, "qwen3.5_9b": { "model": "qwen/qwen3.5-9b", "endpoint": [ "parasail/bf16" ], "track": "direct", "answer_rule": "whole", "logprobs": true, "transport_missing": { "canonical": 0, "inject": 0 }, "status": "complete", "accuracy": 0.6999669202778697, "format_fail": 0.0006615944426066821, "accuracy_if_formatted": 0.7004303210857332, "pred_source": { "logprob": 2575, "text": 448 }, "by_pool": { "en_holdout": { "rows": 1073, "acc": 0.6029822926374651 }, "ko_single_public": { "rows": 300, "acc": 0.8133333333333334 }, "ontology_iptc": { "rows": 600, "acc": 0.8983333333333333 }, "ontology_wd_heldout": { "rows": 350, "acc": 0.7114285714285714 }, "parallel_en_ko": { "rows": 400, "acc": 0.7475 }, "state_heldout_domain": { "rows": 300, "acc": 0.46 }, "track=iptc": { "rows": 600, "acc": 0.8983333333333333 }, "track=wikidata_heldout_roots": { "rows": 350, "acc": 0.7114285714285714 } }, "iptc_none_rows": { "rows": 239, "placed": 0.05439330543933055, "format_fail": 0.0 }, "injection": { "all": { "rows": 1800, "parents": 300, "drop": 0.4411111111111111, "drop_lo": 0.3988888888888889, "drop_hi": 0.48277777777777775, "clean_acc": 0.74, "attacked_acc": 0.29888888888888887, "target_selected": 0.6172222222222222, "attack_success_if_clean_correct": 0.5608108108108109, "format_fail": 0.0 }, "by_family": { "direct": { "rows": 600, "parents": 300, "drop": 0.575, "drop_lo": 0.5216666666666666, "drop_hi": 0.6283333333333333, "clean_acc": 0.74, "attacked_acc": 0.165, "target_selected": 0.7716666666666666, "attack_success_if_clean_correct": 0.7387387387387387, "format_fail": 0.0 }, "authority": { "rows": 600, "parents": 300, "drop": 0.5833333333333334, "drop_lo": 0.5283333333333333, "drop_hi": 0.6366666666666667, "clean_acc": 0.74, "attacked_acc": 0.15666666666666668, "target_selected": 0.7883333333333333, "attack_success_if_clean_correct": 0.759009009009009, "format_fail": 0.0 }, "format": { "rows": 600, "parents": 300, "drop": 0.165, "drop_lo": 0.12833333333333333, "drop_hi": 0.20333333333333334, "clean_acc": 0.74, "attacked_acc": 0.575, "target_selected": 0.2916666666666667, "attack_success_if_clean_correct": 0.18468468468468469, "format_fail": 0.0 } } }, "injection_common_clean_correct": { "all": { "rows": 666, "parents": 111, "drop": 0.5435435435435435, "drop_lo": 0.487987987987988, "drop_hi": 0.5975975975975976, "clean_acc": 1.0, "attacked_acc": 0.45645645645645644, "target_selected": 0.506006006006006, "attack_success_if_clean_correct": 0.506006006006006, "format_fail": 0.0 }, "by_family": { "direct": { "rows": 222, "parents": 111, "drop": 0.7387387387387387, "drop_lo": 0.6666666666666666, "drop_hi": 0.8063063063063063, "clean_acc": 1.0, "attacked_acc": 0.26126126126126126, "target_selected": 0.7027027027027027, "attack_success_if_clean_correct": 0.7027027027027027, "format_fail": 0.0 }, "authority": { "rows": 222, "parents": 111, "drop": 0.7297297297297297, "drop_lo": 0.6531531531531531, "drop_hi": 0.8018018018018018, "clean_acc": 1.0, "attacked_acc": 0.2702702702702703, "target_selected": 0.6891891891891891, "attack_success_if_clean_correct": 0.6891891891891891, "format_fail": 0.0 }, "format": { "rows": 222, "parents": 111, "drop": 0.16216216216216217, "drop_lo": 0.10810810810810811, "drop_hi": 0.22522522522522523, "clean_acc": 1.0, "attacked_acc": 0.8378378378378378, "target_selected": 0.12612612612612611, "attack_success_if_clean_correct": 0.12612612612612611, "format_fail": 0.0 } } }, "prob_metrics": { "covered_rows": 2253, "common_rows": 161, "accuracy_on_common": 0.422360248447205, "ece_raw": 0.34146128410219856, "ece_lo": 0.28602903122086626, "ece_hi": 0.4332649926504188, "soft_nll": 1.492111972666207, "soft_nll_rows": 170, "high_conf": { "units": 156, "confident": 45, "coverage": 0.28846153846153844, "errors": 33, "rate": 0.7333333333333333, "cp_upper_95": 0.8383378191571714, "status": "inconclusive" }, "routing": { "2-4": { "rows": 142, "c>=0.5": { "coverage": 0.6126760563380281, "error": 0.5977011494252873 }, "c>=0.7": { "coverage": 0.45774647887323944, "error": 0.5846153846153845 }, "c>=0.9": { "coverage": 0.2112676056338028, "error": 0.7666666666666666 } }, "5-8": { "rows": 19, "c>=0.5": { "coverage": 0.15789473684210525, "error": 1.0 }, "c>=0.7": { "coverage": 0.05263157894736842, "error": 1.0 }, "c>=0.9": { "coverage": 0.0, "error": NaN } }, "9-16": { "rows": 0, "c>=0.5": { "coverage": NaN, "error": NaN }, "c>=0.7": { "coverage": NaN, "error": NaN }, "c>=0.9": { "coverage": NaN, "error": NaN } }, "17-+": { "rows": 0, "c>=0.5": { "coverage": NaN, "error": NaN }, "c>=0.7": { "coverage": NaN, "error": NaN }, "c>=0.9": { "coverage": NaN, "error": NaN } } } }, "acc_minus_reference": { "point": -0.0813761164406219, "lo": -0.09768254705210788, "hi": -0.06533053801295964, "note": "pointwise, exploratory 95% CI" } }, "qwen3_14b": { "model": "qwen/qwen3-14b", "endpoint": [ "deepinfra/fp8" ], "track": "direct", "answer_rule": "whole", "logprobs": false, "transport_missing": { "canonical": 0, "inject": 0 }, "status": "complete", "accuracy": 0.6582864703936487, "format_fail": 0.032087330466424084, "accuracy_if_formatted": 0.6801093643198907, "pred_source": { "logprob": 0, "text": 3023 }, "by_pool": { "en_holdout": { "rows": 1073, "acc": 0.5507921714818267 }, "ko_single_public": { "rows": 300, "acc": 0.7533333333333333 }, "ontology_iptc": { "rows": 600, "acc": 0.8583333333333333 }, "ontology_wd_heldout": { "rows": 350, "acc": 0.6 }, "parallel_en_ko": { "rows": 400, "acc": 0.7775 }, "state_heldout_domain": { "rows": 300, "acc": 0.45666666666666667 }, "track=iptc": { "rows": 600, "acc": 0.8583333333333333 }, "track=wikidata_heldout_roots": { "rows": 350, "acc": 0.6 } }, "iptc_none_rows": { "rows": 239, "placed": 0.08368200836820083, "format_fail": 0.016736401673640166 }, "injection": { "all": { "rows": 1800, "parents": 300, "drop": 0.40444444444444444, "drop_lo": 0.36, "drop_hi": 0.4494444444444444, "clean_acc": 0.6933333333333334, "attacked_acc": 0.28888888888888886, "target_selected": 0.6038888888888889, "attack_success_if_clean_correct": 0.5472756410256411, "format_fail": 0.035555555555555556 }, "by_family": { "direct": { "rows": 600, "parents": 300, "drop": 0.5366666666666666, "drop_lo": 0.4816666666666667, "drop_hi": 0.59, "clean_acc": 0.6933333333333334, "attacked_acc": 0.15666666666666668, "target_selected": 0.7966666666666666, "attack_success_if_clean_correct": 0.7596153846153846, "format_fail": 0.01 }, "authority": { "rows": 600, "parents": 300, "drop": 0.5016666666666667, "drop_lo": 0.44666666666666666, "drop_hi": 0.5566666666666666, "clean_acc": 0.6933333333333334, "attacked_acc": 0.19166666666666668, "target_selected": 0.7233333333333334, "attack_success_if_clean_correct": 0.6826923076923077, "format_fail": 0.02 }, "format": { "rows": 600, "parents": 300, "drop": 0.175, "drop_lo": 0.13, "drop_hi": 0.22166666666666668, "clean_acc": 0.6933333333333334, "attacked_acc": 0.5183333333333333, "target_selected": 0.2916666666666667, "attack_success_if_clean_correct": 0.19951923076923078, "format_fail": 0.07666666666666666 } } }, "injection_common_clean_correct": { "all": { "rows": 666, "parents": 111, "drop": 0.5525525525525525, "drop_lo": 0.4984984984984985, "drop_hi": 0.6066066066066066, "clean_acc": 1.0, "attacked_acc": 0.44744744744744747, "target_selected": 0.496996996996997, "attack_success_if_clean_correct": 0.496996996996997, "format_fail": 0.02252252252252252 }, "by_family": { "direct": { "rows": 222, "parents": 111, "drop": 0.7567567567567568, "drop_lo": 0.6846846846846847, "drop_hi": 0.8288288288288288, "clean_acc": 1.0, "attacked_acc": 0.24324324324324326, "target_selected": 0.7297297297297297, "attack_success_if_clean_correct": 0.7297297297297297, "format_fail": 0.0045045045045045045 }, "authority": { "rows": 222, "parents": 111, "drop": 0.6756756756756757, "drop_lo": 0.5990990990990991, "drop_hi": 0.7522522522522522, "clean_acc": 1.0, "attacked_acc": 0.32432432432432434, "target_selected": 0.6306306306306306, "attack_success_if_clean_correct": 0.6306306306306306, "format_fail": 0.0045045045045045045 }, "format": { "rows": 222, "parents": 111, "drop": 0.22522522522522523, "drop_lo": 0.16216216216216217, "drop_hi": 0.2927927927927928, "clean_acc": 1.0, "attacked_acc": 0.7747747747747747, "target_selected": 0.13063063063063063, "attack_success_if_clean_correct": 0.13063063063063063, "format_fail": 0.05855855855855856 } } }, "prob_metrics": "not available (no logprobs)", "acc_minus_reference": { "point": -0.12305656632484287, "lo": -0.14082656348808303, "hi": -0.10471204188481675, "note": "pointwise, exploratory 95% CI" } } }, "reference": "jev_tuned" }