{
  "tasks": {
    "r7|full|chess_legal_v2": {
      "model": "r7",
      "stage": "full",
      "task": "chess_legal_v2",
      "rows": 500,
      "rotations": 4,
      "k": 4,
      "rotation_mean_accuracy": 0.4005,
      "debiased_accuracy": 0.426,
      "debiased_ci95": [
        0.38338257803751286,
        0.4697458243684626
      ],
      "order_consistency": 0.466,
      "nll_mean_probs": 1.2895886544670165,
      "ln_k": 1.3862943611198906,
      "pred_position_share": [
        0.4155,
        0.2225,
        0.183,
        0.179
      ]
    },
    "r7|full|gpqa_v2": {
      "model": "r7",
      "stage": "full",
      "task": "gpqa_v2",
      "rows": 198,
      "rotations": 4,
      "k": 4,
      "rotation_mean_accuracy": 0.3371212121212121,
      "debiased_accuracy": 0.3282828282828283,
      "debiased_ci95": [
        0.2666857032513807,
        0.3964162165579131
      ],
      "order_consistency": 0.23232323232323232,
      "nll_mean_probs": 1.3462829985829794,
      "ln_k": 1.3862943611198906,
      "pred_position_share": [
        0.2032828282828283,
        0.18686868686868688,
        0.2777777777777778,
        0.33207070707070707
      ]
    },
    "r7|full|gsm8k_mc10_v2": {
      "model": "r7",
      "stage": "full",
      "task": "gsm8k_mc10_v2",
      "rows": 1319,
      "rotations": 10,
      "k": 10,
      "rotation_mean_accuracy": 0.2858225928733891,
      "debiased_accuracy": 0.3062926459438969,
      "debiased_ci95": [
        0.282008848029071,
        0.3317014748752748
      ],
      "order_consistency": 0.14935557240333586,
      "nll_mean_probs": 1.9325159750133525,
      "ln_k": 2.302585092994046,
      "pred_position_share": [
        0.06739954510993176,
        0.07816527672479151,
        0.08695981804397271,
        0.09833206974981046,
        0.10151630022744504,
        0.100303260045489,
        0.10962850644427596,
        0.11940864291129644,
        0.11834723275208492,
        0.1199393479909022
      ]
    },
    "r7|full|gsm8k_mc4_v2": {
      "model": "r7",
      "stage": "full",
      "task": "gsm8k_mc4_v2",
      "rows": 1319,
      "rotations": 4,
      "k": 4,
      "rotation_mean_accuracy": 0.515352539802881,
      "debiased_accuracy": 0.5382865807429871,
      "debiased_ci95": [
        0.5113101453003895,
        0.5650406519337808
      ],
      "order_consistency": 0.43290371493555724,
      "nll_mean_probs": 1.096663031900488,
      "ln_k": 1.3862943611198906,
      "pred_position_share": [
        0.19143290371493557,
        0.23294162244124336,
        0.27369219105382864,
        0.30193328278999243
      ]
    },
    "r7|options_only|chess_legal_v2": {
      "model": "r7",
      "stage": "options_only",
      "task": "chess_legal_v2",
      "rows": 500,
      "rotations": 4,
      "k": 4,
      "rotation_mean_accuracy": 0.294,
      "debiased_accuracy": 0.308,
      "debiased_ci95": [
        0.26912574776130455,
        0.34980199902176684
      ],
      "order_consistency": 0.11,
      "nll_mean_probs": 1.4122002220138676,
      "ln_k": 1.3862943611198906,
      "pred_position_share": [
        0.154,
        0.143,
        0.259,
        0.444
      ]
    },
    "r7|options_only|gpqa_v2": {
      "model": "r7",
      "stage": "options_only",
      "task": "gpqa_v2",
      "rows": 198,
      "rotations": 4,
      "k": 4,
      "rotation_mean_accuracy": 0.3068181818181818,
      "debiased_accuracy": 0.32323232323232326,
      "debiased_ci95": [
        0.2619851056969361,
        0.3912080470479252
      ],
      "order_consistency": 0.06565656565656566,
      "nll_mean_probs": 1.4115891851171167,
      "ln_k": 1.3862943611198906,
      "pred_position_share": [
        0.15025252525252525,
        0.16540404040404041,
        0.3068181818181818,
        0.37752525252525254
      ]
    },
    "r7|options_only|gsm8k_mc10_v2": {
      "model": "r7",
      "stage": "options_only",
      "task": "gsm8k_mc10_v2",
      "rows": 1319,
      "rotations": 5,
      "k": 10,
      "rotation_mean_accuracy": 0.10947687642153177,
      "debiased_accuracy": 0.10841546626231995,
      "debiased_ci95": [
        0.0927599612966865,
        0.12634525114986767
      ],
      "order_consistency": 0.003032600454890068,
      "nll_mean_probs": 2.377036472907572,
      "ln_k": 2.302585092994046,
      "pred_position_share": [
        0.011372251705837756,
        0.004397270659590599,
        0.014859742228961335,
        0.0357846853677028,
        0.047611827141774074,
        0.08809704321455648,
        0.1420773313115997,
        0.20272934040940105,
        0.18832448824867323,
        0.264746019711903
      ]
    },
    "r7|options_only|gsm8k_mc4_v2": {
      "model": "r7",
      "stage": "options_only",
      "task": "gsm8k_mc4_v2",
      "rows": 1319,
      "rotations": 4,
      "k": 4,
      "rotation_mean_accuracy": 0.2505686125852919,
      "debiased_accuracy": 0.24336618650492797,
      "debiased_ci95": [
        0.22097520606722637,
        0.2672476679175918
      ],
      "order_consistency": 0.05458680818802123,
      "nll_mean_probs": 1.4384991214185934,
      "ln_k": 1.3862943611198906,
      "pred_position_share": [
        0.24298711144806673,
        0.07695223654283548,
        0.22915087187263078,
        0.45090978013646704
      ]
    },
    "raw|full|chess_legal_v2": {
      "model": "raw",
      "stage": "full",
      "task": "chess_legal_v2",
      "rows": 500,
      "rotations": 4,
      "k": 4,
      "rotation_mean_accuracy": 0.315,
      "debiased_accuracy": 0.438,
      "debiased_ci95": [
        0.39514834375489416,
        0.4817970744771394
      ],
      "order_consistency": 0.014,
      "nll_mean_probs": 1.2759298211025412,
      "ln_k": 1.3862943611198906,
      "pred_position_share": [
        0.6895,
        0.29,
        0.0125,
        0.008
      ]
    },
    "raw|full|gpqa_v2": {
      "model": "raw",
      "stage": "full",
      "task": "gpqa_v2",
      "rows": 198,
      "rotations": 4,
      "k": 4,
      "rotation_mean_accuracy": 0.31313131313131315,
      "debiased_accuracy": 0.3282828282828283,
      "debiased_ci95": [
        0.2666857032513807,
        0.3964162165579131
      ],
      "order_consistency": 0.10101010101010101,
      "nll_mean_probs": 1.554143105728764,
      "ln_k": 1.3862943611198906,
      "pred_position_share": [
        0.10984848484848485,
        0.63510101010101,
        0.13383838383838384,
        0.12121212121212122
      ]
    },
    "raw|full|gsm8k_mc10_v2": {
      "model": "raw",
      "stage": "full",
      "task": "gsm8k_mc10_v2",
      "rows": 1319,
      "rotations": 10,
      "k": 10,
      "rotation_mean_accuracy": 0.16467020470052868,
      "debiased_accuracy": 0.20545868081880211,
      "debiased_ci95": [
        0.18452439827071845,
        0.22810362696739636
      ],
      "order_consistency": 0.006065200909780136,
      "nll_mean_probs": 2.160294692789794,
      "ln_k": 2.302585092994046,
      "pred_position_share": [
        0.17505686125852918,
        0.523502653525398,
        0.05473843821076573,
        0.054510993176648974,
        0.013722517058377558,
        0.03752843062926459,
        0.019257012888551934,
        0.06110689916603487,
        0.030780894617134194,
        0.02979529946929492
      ]
    },
    "raw|full|gsm8k_mc4_v2": {
      "model": "raw",
      "stage": "full",
      "task": "gsm8k_mc4_v2",
      "rows": 1319,
      "rotations": 4,
      "k": 4,
      "rotation_mean_accuracy": 0.36713419257012886,
      "debiased_accuracy": 0.43745261561789234,
      "debiased_ci95": [
        0.4109011521947761,
        0.4643673473731633
      ],
      "order_consistency": 0.06899166034874905,
      "nll_mean_probs": 1.2742964931019587,
      "ln_k": 1.3862943611198906,
      "pred_position_share": [
        0.2579605761940864,
        0.5869977255496588,
        0.11694465504169825,
        0.03809704321455648
      ]
    },
    "raw|options_only|chess_legal_v2": {
      "model": "raw",
      "stage": "options_only",
      "task": "chess_legal_v2",
      "rows": 500,
      "rotations": 4,
      "k": 4,
      "rotation_mean_accuracy": 0.252,
      "debiased_accuracy": 0.29,
      "debiased_ci95": [
        0.2519474195915272,
        0.33125480345245706
      ],
      "order_consistency": 0.0,
      "nll_mean_probs": 1.4114726259217971,
      "ln_k": 1.3862943611198906,
      "pred_position_share": [
        0.0,
        0.9815,
        0.0075,
        0.011
      ]
    },
    "raw|options_only|gpqa_v2": {
      "model": "raw",
      "stage": "options_only",
      "task": "gpqa_v2",
      "rows": 198,
      "rotations": 4,
      "k": 4,
      "rotation_mean_accuracy": 0.25757575757575757,
      "debiased_accuracy": 0.25757575757575757,
      "debiased_ci95": [
        0.20168506443256926,
        0.32269411647466895
      ],
      "order_consistency": 0.020202020202020204,
      "nll_mean_probs": 1.5601990009952065,
      "ln_k": 1.3862943611198906,
      "pred_position_share": [
        0.015151515151515152,
        0.8434343434343434,
        0.07702020202020202,
        0.06439393939393939
      ]
    },
    "raw|options_only|gsm8k_mc10_v2": {
      "model": "raw",
      "stage": "options_only",
      "task": "gsm8k_mc10_v2",
      "rows": 1319,
      "rotations": 5,
      "k": 10,
      "rotation_mean_accuracy": 0.10553449583017482,
      "debiased_accuracy": 0.10538286580742987,
      "debiased_ci95": [
        0.08994299481674577,
        0.1231146297300721
      ],
      "order_consistency": 0.0,
      "nll_mean_probs": 2.7989174437564013,
      "ln_k": 2.302585092994046,
      "pred_position_share": [
        0.0,
        0.13525398028809704,
        0.0037907505686125853,
        0.0,
        0.0012130401819560274,
        0.00030326004548900684,
        0.4363912054586808,
        0.0018195602729340408,
        0.3905989385898408,
        0.03062926459438969
      ]
    },
    "raw|options_only|gsm8k_mc4_v2": {
      "model": "raw",
      "stage": "options_only",
      "task": "gsm8k_mc4_v2",
      "rows": 1319,
      "rotations": 4,
      "k": 4,
      "rotation_mean_accuracy": 0.24981046247156938,
      "debiased_accuracy": 0.24715693707354056,
      "debiased_ci95": [
        0.22463441088141858,
        0.27114794797806996
      ],
      "order_consistency": 0.0,
      "nll_mean_probs": 1.4600850953045421,
      "ln_k": 1.3862943611198906,
      "pred_position_share": [
        0.0,
        0.9474981046247157,
        0.041319181197877176,
        0.011182714177407127
      ]
    }
  },
  "pairs": {
    "raw->r7|full|chess_legal_v2": {
      "rows": 500,
      "delta_pp": -1.2,
      "ci95_pp": [
        -6.4,
        3.8
      ],
      "raw_only": 90,
      "r7_only": 84,
      "mcnemar_p": 0.7047680231195156
    },
    "raw->r7|full|gpqa_v2": {
      "rows": 198,
      "delta_pp": 0.0,
      "ci95_pp": [
        -7.07070707070707,
        7.07070707070707
      ],
      "raw_only": 26,
      "r7_only": 26,
      "mcnemar_p": 1.0
    },
    "raw->r7|full|gsm8k_mc10_v2": {
      "rows": 1319,
      "delta_pp": 10.083396512509477,
      "ci95_pp": [
        7.050796057619409,
        13.040181956027293
      ],
      "raw_only": 150,
      "r7_only": 283,
      "mcnemar_p": 1.625471377453816e-10
    },
    "raw->r7|full|gsm8k_mc4_v2": {
      "rows": 1319,
      "delta_pp": 10.083396512509477,
      "ci95_pp": [
        6.444275966641396,
        13.722517058377559
      ],
      "raw_only": 226,
      "r7_only": 359,
      "mcnemar_p": 4.2453654446737796e-08
    },
    "raw->r7|options_only|chess_legal_v2": {
      "rows": 500,
      "delta_pp": 1.8,
      "ci95_pp": [
        -3.0,
        7.199999999999999
      ],
      "raw_only": 79,
      "r7_only": 88,
      "mcnemar_p": 0.5360050623796357
    },
    "raw->r7|options_only|gpqa_v2": {
      "rows": 198,
      "delta_pp": 6.565656565656566,
      "ci95_pp": [
        -1.0101010101010102,
        14.14141414141414
      ],
      "raw_only": 24,
      "r7_only": 37,
      "mcnemar_p": 0.12373144253809217
    },
    "raw->r7|options_only|gsm8k_mc10_v2": {
      "rows": 1319,
      "delta_pp": 0.3032600454890068,
      "ci95_pp": [
        -2.047005307050796,
        2.577710386656558
      ],
      "raw_only": 119,
      "r7_only": 123,
      "mcnemar_p": 0.8471289384306088
    },
    "raw->r7|options_only|gsm8k_mc4_v2": {
      "rows": 1319,
      "delta_pp": -0.37907505686125853,
      "ci95_pp": [
        -3.790750568612585,
        2.9567854435178167
      ],
      "raw_only": 261,
      "r7_only": 256,
      "mcnemar_p": 0.8603786002624875
    }
  }
}