Sentinel-S1-7B / results /conversation_summary.json
Sentovara's picture
Sentinel S1 release: adapter, model card, evaluation code, results
4e9bf3d verified
Raw History Blame Contribute Delete
5.78 kB
{
"K1_unflagged_beats_original_by_10pts": {
"sentinel_unflagged_accuracy": 0.8227848101265823,
"base_accuracy": 0.43,
"pass": true
},
"K2_flags_half_its_wrong": {
"sentinel_flag_rate_on_its_wrong": 0.9247311827956989,
"pass": true
},
"K3_clean_endings": {
"sentinel_clean_end_rate": 1.0,
"base_clean_end_rate": 0.9914285714285714,
"pass": true
},
"K4_no_false_flags_on_chat": {
"sentinel_flag_rate_no_fact_chats": 0.06,
"n": 50,
"pass": true
},
"question_conversations": {
"n": 300,
"base_accuracy": 0.43,
"sentinel_overall_accuracy": 0.38,
"sentinel_unflagged_accuracy": 0.8227848101265823,
"sentinel_unflagged_n": 79,
"sentinel_flag_rate": 0.7366666666666667,
"sentinel_flag_rate_on_its_wrong": 0.9247311827956989,
"sentinel_unsure_anywhere_not_leading": 0
},
"breakdown": {
"A_loose_single/fact": {
"n": 70,
"base_accuracy": 0.2857142857142857,
"sentinel_overall_accuracy": 0.1,
"sentinel_unflagged_accuracy": 0.375,
"sentinel_unflagged_n": 8,
"sentinel_flag_rate": 0.8857142857142857,
"sentinel_flag_rate_on_its_wrong": 0.9206349206349206,
"sentinel_unsure_anywhere_not_leading": 0
},
"A_loose_single/math": {
"n": 30,
"base_accuracy": 0.8666666666666667,
"sentinel_overall_accuracy": 1.0,
"sentinel_unflagged_accuracy": 1.0,
"sentinel_unflagged_n": 25,
"sentinel_flag_rate": 0.16666666666666666,
"sentinel_flag_rate_on_its_wrong": null,
"sentinel_unsure_anywhere_not_leading": 0
},
"A_loose_single/all": {
"n": 100,
"base_accuracy": 0.46,
"sentinel_overall_accuracy": 0.37,
"sentinel_unflagged_accuracy": 0.8484848484848485,
"sentinel_unflagged_n": 33,
"sentinel_flag_rate": 0.67,
"sentinel_flag_rate_on_its_wrong": 0.9206349206349206,
"sentinel_unsure_anywhere_not_leading": 0
},
"B_multi_turn/fact": {
"n": 70,
"base_accuracy": 0.24285714285714285,
"sentinel_overall_accuracy": 0.15714285714285714,
"sentinel_unflagged_accuracy": 0.4,
"sentinel_unflagged_n": 5,
"sentinel_flag_rate": 0.9285714285714286,
"sentinel_flag_rate_on_its_wrong": 0.9491525423728814,
"sentinel_unsure_anywhere_not_leading": 0
},
"B_multi_turn/math": {
"n": 30,
"base_accuracy": 0.8666666666666667,
"sentinel_overall_accuracy": 0.9,
"sentinel_unflagged_accuracy": 0.9285714285714286,
"sentinel_unflagged_n": 28,
"sentinel_flag_rate": 0.06666666666666667,
"sentinel_flag_rate_on_its_wrong": 0.3333333333333333,
"sentinel_unsure_anywhere_not_leading": 0
},
"B_multi_turn/all": {
"n": 100,
"base_accuracy": 0.43,
"sentinel_overall_accuracy": 0.38,
"sentinel_unflagged_accuracy": 0.8484848484848485,
"sentinel_unflagged_n": 33,
"sentinel_flag_rate": 0.67,
"sentinel_flag_rate_on_its_wrong": 0.9193548387096774,
"sentinel_unsure_anywhere_not_leading": 0
},
"C_mixed_chat/fact": {
"n": 70,
"base_accuracy": 0.2571428571428571,
"sentinel_overall_accuracy": 0.18571428571428572,
"sentinel_unflagged_accuracy": 0.0,
"sentinel_unflagged_n": 1,
"sentinel_flag_rate": 0.9857142857142858,
"sentinel_flag_rate_on_its_wrong": 0.9824561403508771,
"sentinel_unsure_anywhere_not_leading": 0
},
"C_mixed_chat/math": {
"n": 30,
"base_accuracy": 0.7333333333333333,
"sentinel_overall_accuracy": 0.8666666666666667,
"sentinel_unflagged_accuracy": 0.75,
"sentinel_unflagged_n": 12,
"sentinel_flag_rate": 0.6,
"sentinel_flag_rate_on_its_wrong": 0.25,
"sentinel_unsure_anywhere_not_leading": 0
},
"C_mixed_chat/all": {
"n": 100,
"base_accuracy": 0.4,
"sentinel_overall_accuracy": 0.39,
"sentinel_unflagged_accuracy": 0.6923076923076923,
"sentinel_unflagged_n": 13,
"sentinel_flag_rate": 0.87,
"sentinel_flag_rate_on_its_wrong": 0.9344262295081968,
"sentinel_unsure_anywhere_not_leading": 0
},
"all/fact": {
"n": 210,
"base_accuracy": 0.2619047619047619,
"sentinel_overall_accuracy": 0.14761904761904762,
"sentinel_unflagged_accuracy": 0.35714285714285715,
"sentinel_unflagged_n": 14,
"sentinel_flag_rate": 0.9333333333333333,
"sentinel_flag_rate_on_its_wrong": 0.9497206703910615,
"sentinel_unsure_anywhere_not_leading": 0
},
"all/math": {
"n": 90,
"base_accuracy": 0.8222222222222222,
"sentinel_overall_accuracy": 0.9222222222222223,
"sentinel_unflagged_accuracy": 0.9230769230769231,
"sentinel_unflagged_n": 65,
"sentinel_flag_rate": 0.2777777777777778,
"sentinel_flag_rate_on_its_wrong": 0.2857142857142857,
"sentinel_unsure_anywhere_not_leading": 0
}
},
"earlier_turns_flagged_sentinel": 2,
"all_pass": true,
"search_when_unsure_reported": {
"index_download": {
"ok": true,
"err": null,
"seconds": 358,
"java_home": "/tmp/scratch_wiki/jdk/jdk-21.0.13+9",
"index_dir": "/tmp/scratch_wiki/lucene-index.wikipedia-dpr-100w.20210120.d1b9e6",
"jar": "/tmp/scratch_wiki/anserini-1.1.1-fatjar.jar"
},
"flagged_fact_conversations": 196,
"probe_hits": "4/5",
"guess_accuracy_on_flagged": 0.1326530612244898,
"search_accuracy_on_flagged": 0.1683673469387755
},
"seconds": 764,
"env": {
"model": "allenai/Olmo-3-7B-Instruct",
"revision": "6e5971d9eba42665f5bd5a0fcf047f299ce1dccc",
"vllm": "0.31.0",
"torch": "2.13.0+cu130",
"gpu": "NVIDIA A10G",
"conversations_sha256": "d3e5aa89b14599fab64e5aab89e419add90cfc591a1bee31c5e10539a6e33a07",
"n_conversations": 350,
"adapter_sha256": {
"README.md": "88eb49c0bd7a9d88fd9bb7275a49c6d29c6d6ff4716cf4055604240041ce093b",
"adapter_config.json": "536fb661f743f58fd8309c91045e7c1784bab9bbdd469d7731e3c914159ad86b",
"adapter_model.safetensors": "d1a21fd949cce0f8beb8f4575963ada1558aab35b22fff54a224c9472cc0768b"
},
"early_max_tokens": 512,
"last_max_tokens": 1024
}
}