system's picture
system HF Staff
Mirror GitHub f31f2ceadd54
fd74a4d verified
Raw History Blame Contribute Delete
10 kB
{
"fixture": {
"purpose": "End-to-end replay of the existing Figure 1 family, selected by its role in the paper; uses both models and arms at base five, without choosing outputs by success.",
"sample_id": "3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e",
"base_count": 5,
"reference_rule_pairs": [
[
"ex",
"azw"
],
[
"wa",
"y"
]
],
"task_pair": {
"CONTROL": {
"inputs": [
"gud",
"czua",
"bwcied",
"dzexa",
"yf",
"iiiexa"
],
"outputs": [
"gud",
"czua",
"bwcied",
"dzazy",
"yf",
"iiiazy"
]
},
"SEPARATION": {
"inputs": [
"gud",
"czua",
"bwcied",
"dzexa",
"yf",
"wbawex"
],
"outputs": [
"gud",
"czua",
"bwcied",
"dzazy",
"yf",
"wbayzw"
]
}
},
"request_ids": [
"M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL",
"M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION",
"M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL",
"M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION"
],
"expected": {
"status": "PASS",
"family": "3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e",
"base_count": 5,
"certificates": {
"CONTROL": {
"complete_one_rule_solutions": [
[
"exa",
"azy"
]
],
"minimum_length": 1,
"two_rule_reference_passes": true
},
"SEPARATION": {
"complete_one_rule_solutions": [],
"minimum_length": 2,
"two_rule_reference_passes": true
}
},
"saved_outputs": {
"M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION": {
"selected_rules": null,
"legal_selected_program": false,
"targets": {
"displayed": {
"hits": 0,
"inputs": 6,
"all_pass": false
},
"common_nine": {
"hits": 0,
"inputs": 9,
"all_pass": false
},
"wholly_unshown_pair": {
"hits": 0,
"inputs": 2,
"all_pass": false
},
"uniform": {
"hits": 0,
"inputs": 32,
"all_pass": false
},
"active": {
"hits": 0,
"inputs": 32,
"all_pass": false
}
}
},
"M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL": {
"selected_rules": [
[
"a",
"y"
],
[
"e",
"a"
],
[
"x",
"z"
]
],
"legal_selected_program": true,
"targets": {
"displayed": {
"hits": 4,
"inputs": 6,
"all_pass": false
},
"common_nine": {
"hits": 5,
"inputs": 9,
"all_pass": false
},
"wholly_unshown_pair": {
"hits": 1,
"inputs": 2,
"all_pass": false
},
"uniform": {
"hits": 11,
"inputs": 32,
"all_pass": false
},
"active": {
"hits": 5,
"inputs": 32,
"all_pass": false
}
}
},
"M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION": {
"selected_rules": [
[
"exa",
"azy"
],
[
"wex",
"yzw"
]
],
"legal_selected_program": true,
"targets": {
"displayed": {
"hits": 6,
"inputs": 6,
"all_pass": true
},
"common_nine": {
"hits": 9,
"inputs": 9,
"all_pass": true
},
"wholly_unshown_pair": {
"hits": 2,
"inputs": 2,
"all_pass": true
},
"uniform": {
"hits": 32,
"inputs": 32,
"all_pass": true
},
"active": {
"hits": 0,
"inputs": 32,
"all_pass": false
}
}
},
"M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL": {
"selected_rules": null,
"legal_selected_program": false,
"targets": {
"displayed": {
"hits": 0,
"inputs": 6,
"all_pass": false
},
"common_nine": {
"hits": 0,
"inputs": 9,
"all_pass": false
},
"wholly_unshown_pair": {
"hits": 0,
"inputs": 2,
"all_pass": false
},
"uniform": {
"hits": 0,
"inputs": 32,
"all_pass": false
},
"active": {
"hits": 0,
"inputs": 32,
"all_pass": false
}
}
}
},
"model_inference_calls": 0
}
},
"panel": [
{
"model": "Qwen3-8B",
"base_count": "3",
"displayed_examples": "4",
"primary_family_N": "119",
"CONTROL_displayed_successes": "59",
"SEPARATION_displayed_successes": "15",
"displayed_delta_pp_and_95CI": "-36.97 [-46.22,-27.73]",
"unshown_pair_delta_pp_and_95CI": "+13.45 [+7.56,+20.17]",
"all_legal_N": "120",
"all_legal_displayed_delta_pp_and_95CI": "-36.67 [-45.83,-27.50]",
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
},
{
"model": "Qwen3-8B",
"base_count": "4",
"displayed_examples": "5",
"primary_family_N": "119",
"CONTROL_displayed_successes": "61",
"SEPARATION_displayed_successes": "15",
"displayed_delta_pp_and_95CI": "-38.66 [-47.90,-28.57]",
"unshown_pair_delta_pp_and_95CI": "+10.92 [+5.88,+16.81]",
"all_legal_N": "120",
"all_legal_displayed_delta_pp_and_95CI": "-39.17 [-48.33,-29.17]",
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
},
{
"model": "Qwen3-8B",
"base_count": "5",
"displayed_examples": "6",
"primary_family_N": "120",
"CONTROL_displayed_successes": "67",
"SEPARATION_displayed_successes": "12",
"displayed_delta_pp_and_95CI": "-45.83 [-55.00,-36.67]",
"unshown_pair_delta_pp_and_95CI": "+9.17 [+4.17,+15.00]",
"all_legal_N": "120",
"all_legal_displayed_delta_pp_and_95CI": "-45.83 [-55.00,-36.67]",
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
},
{
"model": "Qwen3-8B",
"base_count": "6",
"displayed_examples": "7",
"primary_family_N": "120",
"CONTROL_displayed_successes": "68",
"SEPARATION_displayed_successes": "19",
"displayed_delta_pp_and_95CI": "-40.83 [-50.00,-31.67]",
"unshown_pair_delta_pp_and_95CI": "+12.50 [+6.67,+18.33]",
"all_legal_N": "120",
"all_legal_displayed_delta_pp_and_95CI": "-40.83 [-50.00,-31.67]",
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
},
{
"model": "Olmo-3-7B-Think",
"base_count": "3",
"displayed_examples": "4",
"primary_family_N": "119",
"CONTROL_displayed_successes": "53",
"SEPARATION_displayed_successes": "14",
"displayed_delta_pp_and_95CI": "-32.77 [-42.86,-22.69]",
"unshown_pair_delta_pp_and_95CI": "+8.40 [+3.36,+13.45]",
"all_legal_N": "120",
"all_legal_displayed_delta_pp_and_95CI": "-33.33 [-43.33,-23.33]",
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
},
{
"model": "Olmo-3-7B-Think",
"base_count": "4",
"displayed_examples": "5",
"primary_family_N": "119",
"CONTROL_displayed_successes": "47",
"SEPARATION_displayed_successes": "19",
"displayed_delta_pp_and_95CI": "-23.53 [-34.45,-12.61]",
"unshown_pair_delta_pp_and_95CI": "+11.76 [+5.88,+17.65]",
"all_legal_N": "120",
"all_legal_displayed_delta_pp_and_95CI": "-23.33 [-34.17,-12.50]",
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
},
{
"model": "Olmo-3-7B-Think",
"base_count": "5",
"displayed_examples": "6",
"primary_family_N": "120",
"CONTROL_displayed_successes": "52",
"SEPARATION_displayed_successes": "15",
"displayed_delta_pp_and_95CI": "-30.83 [-40.83,-20.83]",
"unshown_pair_delta_pp_and_95CI": "+11.67 [+6.67,+17.50]",
"all_legal_N": "120",
"all_legal_displayed_delta_pp_and_95CI": "-30.83 [-40.83,-20.83]",
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
},
{
"model": "Olmo-3-7B-Think",
"base_count": "6",
"displayed_examples": "7",
"primary_family_N": "120",
"CONTROL_displayed_successes": "54",
"SEPARATION_displayed_successes": "13",
"displayed_delta_pp_and_95CI": "-34.17 [-43.33,-25.00]",
"unshown_pair_delta_pp_and_95CI": "+9.17 [+4.17,+15.00]",
"all_legal_N": "120",
"all_legal_displayed_delta_pp_and_95CI": "-34.17 [-43.33,-25.00]",
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
}
]
}