{ "K1_unflagged_beats_original_by_10pts": { "sentinel_unflagged_accuracy": 0.8227848101265823, "base_accuracy": 0.43, "pass": true }, "K2_flags_half_its_wrong": { "sentinel_flag_rate_on_its_wrong": 0.9247311827956989, "pass": true }, "K3_clean_endings": { "sentinel_clean_end_rate": 1.0, "base_clean_end_rate": 0.9914285714285714, "pass": true }, "K4_no_false_flags_on_chat": { "sentinel_flag_rate_no_fact_chats": 0.06, "n": 50, "pass": true }, "question_conversations": { "n": 300, "base_accuracy": 0.43, "sentinel_overall_accuracy": 0.38, "sentinel_unflagged_accuracy": 0.8227848101265823, "sentinel_unflagged_n": 79, "sentinel_flag_rate": 0.7366666666666667, "sentinel_flag_rate_on_its_wrong": 0.9247311827956989, "sentinel_unsure_anywhere_not_leading": 0 }, "breakdown": { "A_loose_single/fact": { "n": 70, "base_accuracy": 0.2857142857142857, "sentinel_overall_accuracy": 0.1, "sentinel_unflagged_accuracy": 0.375, "sentinel_unflagged_n": 8, "sentinel_flag_rate": 0.8857142857142857, "sentinel_flag_rate_on_its_wrong": 0.9206349206349206, "sentinel_unsure_anywhere_not_leading": 0 }, "A_loose_single/math": { "n": 30, "base_accuracy": 0.8666666666666667, "sentinel_overall_accuracy": 1.0, "sentinel_unflagged_accuracy": 1.0, "sentinel_unflagged_n": 25, "sentinel_flag_rate": 0.16666666666666666, "sentinel_flag_rate_on_its_wrong": null, "sentinel_unsure_anywhere_not_leading": 0 }, "A_loose_single/all": { "n": 100, "base_accuracy": 0.46, "sentinel_overall_accuracy": 0.37, "sentinel_unflagged_accuracy": 0.8484848484848485, "sentinel_unflagged_n": 33, "sentinel_flag_rate": 0.67, "sentinel_flag_rate_on_its_wrong": 0.9206349206349206, "sentinel_unsure_anywhere_not_leading": 0 }, "B_multi_turn/fact": { "n": 70, "base_accuracy": 0.24285714285714285, "sentinel_overall_accuracy": 0.15714285714285714, "sentinel_unflagged_accuracy": 0.4, "sentinel_unflagged_n": 5, "sentinel_flag_rate": 0.9285714285714286, "sentinel_flag_rate_on_its_wrong": 0.9491525423728814, "sentinel_unsure_anywhere_not_leading": 0 }, "B_multi_turn/math": { "n": 30, "base_accuracy": 0.8666666666666667, "sentinel_overall_accuracy": 0.9, "sentinel_unflagged_accuracy": 0.9285714285714286, "sentinel_unflagged_n": 28, "sentinel_flag_rate": 0.06666666666666667, "sentinel_flag_rate_on_its_wrong": 0.3333333333333333, "sentinel_unsure_anywhere_not_leading": 0 }, "B_multi_turn/all": { "n": 100, "base_accuracy": 0.43, "sentinel_overall_accuracy": 0.38, "sentinel_unflagged_accuracy": 0.8484848484848485, "sentinel_unflagged_n": 33, "sentinel_flag_rate": 0.67, "sentinel_flag_rate_on_its_wrong": 0.9193548387096774, "sentinel_unsure_anywhere_not_leading": 0 }, "C_mixed_chat/fact": { "n": 70, "base_accuracy": 0.2571428571428571, "sentinel_overall_accuracy": 0.18571428571428572, "sentinel_unflagged_accuracy": 0.0, "sentinel_unflagged_n": 1, "sentinel_flag_rate": 0.9857142857142858, "sentinel_flag_rate_on_its_wrong": 0.9824561403508771, "sentinel_unsure_anywhere_not_leading": 0 }, "C_mixed_chat/math": { "n": 30, "base_accuracy": 0.7333333333333333, "sentinel_overall_accuracy": 0.8666666666666667, "sentinel_unflagged_accuracy": 0.75, "sentinel_unflagged_n": 12, "sentinel_flag_rate": 0.6, "sentinel_flag_rate_on_its_wrong": 0.25, "sentinel_unsure_anywhere_not_leading": 0 }, "C_mixed_chat/all": { "n": 100, "base_accuracy": 0.4, "sentinel_overall_accuracy": 0.39, "sentinel_unflagged_accuracy": 0.6923076923076923, "sentinel_unflagged_n": 13, "sentinel_flag_rate": 0.87, "sentinel_flag_rate_on_its_wrong": 0.9344262295081968, "sentinel_unsure_anywhere_not_leading": 0 }, "all/fact": { "n": 210, "base_accuracy": 0.2619047619047619, "sentinel_overall_accuracy": 0.14761904761904762, "sentinel_unflagged_accuracy": 0.35714285714285715, "sentinel_unflagged_n": 14, "sentinel_flag_rate": 0.9333333333333333, "sentinel_flag_rate_on_its_wrong": 0.9497206703910615, "sentinel_unsure_anywhere_not_leading": 0 }, "all/math": { "n": 90, "base_accuracy": 0.8222222222222222, "sentinel_overall_accuracy": 0.9222222222222223, "sentinel_unflagged_accuracy": 0.9230769230769231, "sentinel_unflagged_n": 65, "sentinel_flag_rate": 0.2777777777777778, "sentinel_flag_rate_on_its_wrong": 0.2857142857142857, "sentinel_unsure_anywhere_not_leading": 0 } }, "earlier_turns_flagged_sentinel": 2, "all_pass": true, "search_when_unsure_reported": { "index_download": { "ok": true, "err": null, "seconds": 358, "java_home": "/tmp/scratch_wiki/jdk/jdk-21.0.13+9", "index_dir": "/tmp/scratch_wiki/lucene-index.wikipedia-dpr-100w.20210120.d1b9e6", "jar": "/tmp/scratch_wiki/anserini-1.1.1-fatjar.jar" }, "flagged_fact_conversations": 196, "probe_hits": "4/5", "guess_accuracy_on_flagged": 0.1326530612244898, "search_accuracy_on_flagged": 0.1683673469387755 }, "seconds": 764, "env": { "model": "allenai/Olmo-3-7B-Instruct", "revision": "6e5971d9eba42665f5bd5a0fcf047f299ce1dccc", "vllm": "0.31.0", "torch": "2.13.0+cu130", "gpu": "NVIDIA A10G", "conversations_sha256": "d3e5aa89b14599fab64e5aab89e419add90cfc591a1bee31c5e10539a6e33a07", "n_conversations": 350, "adapter_sha256": { "README.md": "88eb49c0bd7a9d88fd9bb7275a49c6d29c6d6ff4716cf4055604240041ce093b", "adapter_config.json": "536fb661f743f58fd8309c91045e7c1784bab9bbdd469d7731e3c914159ad86b", "adapter_model.safetensors": "d1a21fd949cce0f8beb8f4575963ada1558aab35b22fff54a224c9472cc0768b" }, "early_max_tokens": 512, "last_max_tokens": 1024 } }