{
  "schema": "ladderbench.report/v1",
  "generated_at": "2026-09-29T12:17:59.114016+00:00",
  "ladderbench_version": "0.1.0",
  "model": "hf:bottlecapai/ThinkingCap-Qwen3.8-27B",
  "endpoint": "vllm-offline",
  "probe_set": "all@v1",
  "levels_ordered": [
    "xhigh",
    "medium",
    "low"
  ],
  "summary": {
    "levels": [
      "xhigh",
      "medium",
      "low"
    ],
    "accuracy": [
      0.8823529411764706,
      0.8970588235294118,
      0.9264705882352942
    ],
    "median_thinking_tokens": [
      33.5,
      39.5,
      40.5
    ],
    "spearman_tokens_vs_effort": -0.9999999999999998,
    "spearman_accuracy_vs_effort": -0.9999999999999998,
    "top_level": {
      "level": "xhigh",
      "accuracy": 0.8823529411764706,
      "median_thinking_tokens": 33.5,
      "empty_trace_rate": 0.0
    },
    "token_spread_max_over_min": 1.208955223880597
  },
  "ladder_integrity": "degraded",
  "findings": [
    "inverted dial: tokens vs effort rho=-1.00",
    "misaligned: accuracy vs effort rho=-1.00"
  ],
  "calibration_delta": [
    {
      "level": "xhigh",
      "d_accuracy": 0.0,
      "d_thinking_tokens": -31.5
    },
    {
      "level": "medium",
      "d_accuracy": 0.0147,
      "d_thinking_tokens": -8.0
    },
    {
      "level": "low",
      "d_accuracy": 0.0294,
      "d_thinking_tokens": -4.5
    }
  ],
  "per_level": {
    "xhigh": {
      "n": 68,
      "accuracy": 0.8824,
      "median_thinking_tokens": 33.5,
      "empty_trace_rate": 0.0,
      "responses": [
        {
          "id": "e01",
          "correct": true,
          "predicted": "43",
          "trace_chars": 70
        },
        {
          "id": "e02",
          "correct": true,
          "predicted": "62",
          "trace_chars": 76
        },
        {
          "id": "e03",
          "correct": true,
          "predicted": "12",
          "trace_chars": 70
        },
        {
          "id": "e04",
          "correct": true,
          "predicted": "12",
          "trace_chars": 85
        },
        {
          "id": "e05",
          "correct": true,
          "predicted": "150",
          "trace_chars": 79
        },
        {
          "id": "e06",
          "correct": true,
          "predicted": "5.75",
          "trace_chars": 162
        },
        {
          "id": "e07",
          "correct": true,
          "predicted": "33",
          "trace_chars": 99
        },
        {
          "id": "e08",
          "correct": true,
          "predicted": "1024",
          "trace_chars": 41
        },
        {
          "id": "e09",
          "correct": true,
          "predicted": "210",
          "trace_chars": 95
        },
        {
          "id": "e10",
          "correct": true,
          "predicted": "495",
          "trace_chars": 70
        },
        {
          "id": "e11",
          "correct": true,
          "predicted": "28",
          "trace_chars": 211
        },
        {
          "id": "e12",
          "correct": true,
          "predicted": "63",
          "trace_chars": 70
        },
        {
          "id": "e13",
          "correct": true,
          "predicted": "64",
          "trace_chars": 100
        },
        {
          "id": "e14",
          "correct": true,
          "predicted": "144",
          "trace_chars": 70
        },
        {
          "id": "e15",
          "correct": true,
          "predicted": "7",
          "trace_chars": 190
        },
        {
          "id": "e16",
          "correct": true,
          "predicted": "8",
          "trace_chars": 86
        },
        {
          "id": "e17",
          "correct": true,
          "predicted": "12",
          "trace_chars": 218
        },
        {
          "id": "e18",
          "correct": true,
          "predicted": "100",
          "trace_chars": 87
        },
        {
          "id": "e19",
          "correct": true,
          "predicted": "90",
          "trace_chars": 93
        },
        {
          "id": "e20",
          "correct": true,
          "predicted": "125",
          "trace_chars": 41
        },
        {
          "id": "m01",
          "correct": true,
          "predicted": "1081",
          "trace_chars": 98
        },
        {
          "id": "m02",
          "correct": true,
          "predicted": "32768",
          "trace_chars": 73
        },
        {
          "id": "m03",
          "correct": true,
          "predicted": "5050",
          "trace_chars": 71
        },
        {
          "id": "m04",
          "correct": true,
          "predicted": "5",
          "trace_chars": 311
        },
        {
          "id": "m05",
          "correct": true,
          "predicted": "36",
          "trace_chars": 107
        },
        {
          "id": "m06",
          "correct": true,
          "predicted": "12",
          "trace_chars": 111
        },
        {
          "id": "m07",
          "correct": true,
          "predicted": "300",
          "trace_chars": 101
        },
        {
          "id": "m08",
          "correct": true,
          "predicted": "204",
          "trace_chars": 187
        },
        {
          "id": "m09",
          "correct": true,
          "predicted": "72",
          "trace_chars": 203
        },
        {
          "id": "m10",
          "correct": true,
          "predicted": "1210",
          "trace_chars": 155
        },
        {
          "id": "m11",
          "correct": true,
          "predicted": "8",
          "trace_chars": 204
        },
        {
          "id": "m12",
          "correct": true,
          "predicted": "28",
          "trace_chars": 207
        },
        {
          "id": "m13",
          "correct": true,
          "predicted": "154",
          "trace_chars": 147
        },
        {
          "id": "m14",
          "correct": true,
          "predicted": "13",
          "trace_chars": 194
        },
        {
          "id": "m15",
          "correct": true,
          "predicted": "75",
          "trace_chars": 70
        },
        {
          "id": "m16",
          "correct": true,
          "predicted": "6",
          "trace_chars": 248
        },
        {
          "id": "m17",
          "correct": true,
          "predicted": "48",
          "trace_chars": 101
        },
        {
          "id": "m18",
          "correct": true,
          "predicted": "18",
          "trace_chars": 256
        },
        {
          "id": "m19",
          "correct": true,
          "predicted": "0.5",
          "trace_chars": 166
        },
        {
          "id": "m20",
          "correct": true,
          "predicted": "216",
          "trace_chars": 102
        },
        {
          "id": "h01",
          "correct": true,
          "predicted": "42",
          "trace_chars": 200
        },
        {
          "id": "h02",
          "correct": true,
          "predicted": "1048576",
          "trace_chars": 77
        },
        {
          "id": "h03",
          "correct": true,
          "predicted": "10",
          "trace_chars": 84
        },
        {
          "id": "h04",
          "correct": true,
          "predicted": "8",
          "trace_chars": 158
        },
        {
          "id": "h05",
          "correct": true,
          "predicted": "100",
          "trace_chars": 107
        },
        {
          "id": "h06",
          "correct": true,
          "predicted": "56",
          "trace_chars": 213
        },
        {
          "id": "h07",
          "correct": true,
          "predicted": "20",
          "trace_chars": 195
        },
        {
          "id": "h08",
          "correct": true,
          "predicted": "10",
          "trace_chars": 191
        },
        {
          "id": "h09",
          "correct": true,
          "predicted": "12",
          "trace_chars": 96
        },
        {
          "id": "h10",
          "correct": true,
          "predicted": "13",
          "trace_chars": 154
        },
        {
          "id": "h11",
          "correct": true,
          "predicted": "17",
          "trace_chars": 173
        },
        {
          "id": "h12",
          "correct": true,
          "predicted": "25",
          "trace_chars": 96
        },
        {
          "id": "h13",
          "correct": true,
          "predicted": "30",
          "trace_chars": 209
        },
        {
          "id": "h14",
          "correct": true,
          "predicted": "132",
          "trace_chars": 95
        },
        {
          "id": "h15",
          "correct": true,
          "predicted": "12",
          "trace_chars": 81
        },
        {
          "id": "h16",
          "correct": true,
          "predicted": "720",
          "trace_chars": 93
        },
        {
          "id": "h17",
          "correct": true,
          "predicted": "1",
          "trace_chars": 164
        },
        {
          "id": "h18",
          "correct": true,
          "predicted": "60",
          "trace_chars": 127
        },
        {
          "id": "h19",
          "correct": true,
          "predicted": "0.375",
          "trace_chars": 227
        },
        {
          "id": "h20",
          "correct": true,
          "predicted": "55",
          "trace_chars": 137
        },
        {
          "id": "i01",
          "correct": false,
          "predicted": null,
          "trace_chars": 321
        },
        {
          "id": "i02",
          "correct": false,
          "predicted": null,
          "trace_chars": 612
        },
        {
          "id": "i03",
          "correct": false,
          "predicted": null,
          "trace_chars": 650
        },
        {
          "id": "i04",
          "correct": false,
          "predicted": null,
          "trace_chars": 375
        },
        {
          "id": "i05",
          "correct": false,
          "predicted": null,
          "trace_chars": 1425
        },
        {
          "id": "i06",
          "correct": false,
          "predicted": null,
          "trace_chars": 524
        },
        {
          "id": "i07",
          "correct": false,
          "predicted": null,
          "trace_chars": 388
        },
        {
          "id": "i08",
          "correct": false,
          "predicted": null,
          "trace_chars": 708
        }
      ]
    },
    "medium": {
      "n": 68,
      "accuracy": 0.8971,
      "median_thinking_tokens": 39.5,
      "empty_trace_rate": 0.0,
      "responses": [
        {
          "id": "e01",
          "correct": true,
          "predicted": "43",
          "trace_chars": 68
        },
        {
          "id": "e02",
          "correct": true,
          "predicted": "62",
          "trace_chars": 168
        },
        {
          "id": "e03",
          "correct": true,
          "predicted": "12",
          "trace_chars": 109
        },
        {
          "id": "e04",
          "correct": true,
          "predicted": "12",
          "trace_chars": 56
        },
        {
          "id": "e05",
          "correct": true,
          "predicted": "150",
          "trace_chars": 88
        },
        {
          "id": "e06",
          "correct": true,
          "predicted": "5.75",
          "trace_chars": 61
        },
        {
          "id": "e07",
          "correct": true,
          "predicted": "33",
          "trace_chars": 68
        },
        {
          "id": "e08",
          "correct": true,
          "predicted": "1024",
          "trace_chars": 174
        },
        {
          "id": "e09",
          "correct": true,
          "predicted": "210",
          "trace_chars": 164
        },
        {
          "id": "e10",
          "correct": true,
          "predicted": "495",
          "trace_chars": 88
        },
        {
          "id": "e11",
          "correct": true,
          "predicted": "28",
          "trace_chars": 145
        },
        {
          "id": "e12",
          "correct": true,
          "predicted": "63",
          "trace_chars": 73
        },
        {
          "id": "e13",
          "correct": true,
          "predicted": "64",
          "trace_chars": 103
        },
        {
          "id": "e14",
          "correct": true,
          "predicted": "144",
          "trace_chars": 76
        },
        {
          "id": "e15",
          "correct": true,
          "predicted": "7",
          "trace_chars": 120
        },
        {
          "id": "e16",
          "correct": true,
          "predicted": "8",
          "trace_chars": 92
        },
        {
          "id": "e17",
          "correct": true,
          "predicted": "12",
          "trace_chars": 145
        },
        {
          "id": "e18",
          "correct": true,
          "predicted": "100",
          "trace_chars": 60
        },
        {
          "id": "e19",
          "correct": true,
          "predicted": "90",
          "trace_chars": 75
        },
        {
          "id": "e20",
          "correct": true,
          "predicted": "125",
          "trace_chars": 74
        },
        {
          "id": "m01",
          "correct": true,
          "predicted": "1081",
          "trace_chars": 91
        },
        {
          "id": "m02",
          "correct": true,
          "predicted": "32768",
          "trace_chars": 224
        },
        {
          "id": "m03",
          "correct": true,
          "predicted": "5050",
          "trace_chars": 171
        },
        {
          "id": "m04",
          "correct": true,
          "predicted": "5",
          "trace_chars": 490
        },
        {
          "id": "m05",
          "correct": true,
          "predicted": "36",
          "trace_chars": 74
        },
        {
          "id": "m06",
          "correct": true,
          "predicted": "12",
          "trace_chars": 236
        },
        {
          "id": "m07",
          "correct": true,
          "predicted": "300",
          "trace_chars": 268
        },
        {
          "id": "m08",
          "correct": true,
          "predicted": "204",
          "trace_chars": 130
        },
        {
          "id": "m09",
          "correct": true,
          "predicted": "72",
          "trace_chars": 223
        },
        {
          "id": "m10",
          "correct": true,
          "predicted": "1210",
          "trace_chars": 197
        },
        {
          "id": "m11",
          "correct": true,
          "predicted": "8",
          "trace_chars": 85
        },
        {
          "id": "m12",
          "correct": true,
          "predicted": "28",
          "trace_chars": 101
        },
        {
          "id": "m13",
          "correct": true,
          "predicted": "154",
          "trace_chars": 108
        },
        {
          "id": "m14",
          "correct": true,
          "predicted": "13",
          "trace_chars": 164
        },
        {
          "id": "m15",
          "correct": true,
          "predicted": "75",
          "trace_chars": 90
        },
        {
          "id": "m16",
          "correct": true,
          "predicted": "6",
          "trace_chars": 134
        },
        {
          "id": "m17",
          "correct": true,
          "predicted": "48",
          "trace_chars": 148
        },
        {
          "id": "m18",
          "correct": true,
          "predicted": "18",
          "trace_chars": 140
        },
        {
          "id": "m19",
          "correct": true,
          "predicted": "0.5",
          "trace_chars": 209
        },
        {
          "id": "m20",
          "correct": true,
          "predicted": "216",
          "trace_chars": 69
        },
        {
          "id": "h01",
          "correct": true,
          "predicted": "42",
          "trace_chars": 251
        },
        {
          "id": "h02",
          "correct": true,
          "predicted": "1048576",
          "trace_chars": 299
        },
        {
          "id": "h03",
          "correct": true,
          "predicted": "10",
          "trace_chars": 103
        },
        {
          "id": "h04",
          "correct": true,
          "predicted": "8",
          "trace_chars": 136
        },
        {
          "id": "h05",
          "correct": true,
          "predicted": "100",
          "trace_chars": 259
        },
        {
          "id": "h06",
          "correct": true,
          "predicted": "56",
          "trace_chars": 201
        },
        {
          "id": "h07",
          "correct": true,
          "predicted": "20",
          "trace_chars": 180
        },
        {
          "id": "h08",
          "correct": true,
          "predicted": "10",
          "trace_chars": 166
        },
        {
          "id": "h09",
          "correct": true,
          "predicted": "12",
          "trace_chars": 202
        },
        {
          "id": "h10",
          "correct": true,
          "predicted": "13",
          "trace_chars": 119
        },
        {
          "id": "h11",
          "correct": true,
          "predicted": "17",
          "trace_chars": 129
        },
        {
          "id": "h12",
          "correct": true,
          "predicted": "25",
          "trace_chars": 152
        },
        {
          "id": "h13",
          "correct": true,
          "predicted": "30",
          "trace_chars": 246
        },
        {
          "id": "h14",
          "correct": true,
          "predicted": "132",
          "trace_chars": 163
        },
        {
          "id": "h15",
          "correct": true,
          "predicted": "12",
          "trace_chars": 389
        },
        {
          "id": "h16",
          "correct": true,
          "predicted": "720",
          "trace_chars": 130
        },
        {
          "id": "h17",
          "correct": true,
          "predicted": "1",
          "trace_chars": 336
        },
        {
          "id": "h18",
          "correct": true,
          "predicted": "60",
          "trace_chars": 373
        },
        {
          "id": "h19",
          "correct": true,
          "predicted": "0.375",
          "trace_chars": 204
        },
        {
          "id": "h20",
          "correct": true,
          "predicted": "55",
          "trace_chars": 196
        },
        {
          "id": "i01",
          "correct": false,
          "predicted": "A",
          "trace_chars": 1677
        },
        {
          "id": "i02",
          "correct": false,
          "predicted": "A",
          "trace_chars": 793
        },
        {
          "id": "i03",
          "correct": false,
          "predicted": "B",
          "trace_chars": 824
        },
        {
          "id": "i04",
          "correct": false,
          "predicted": "A",
          "trace_chars": 860
        },
        {
          "id": "i05",
          "correct": false,
          "predicted": "A",
          "trace_chars": 1345
        },
        {
          "id": "i06",
          "correct": false,
          "predicted": "B",
          "trace_chars": 1138
        },
        {
          "id": "i07",
          "correct": false,
          "predicted": "A",
          "trace_chars": 1155
        },
        {
          "id": "i08",
          "correct": true,
          "predicted": "B",
          "trace_chars": 793
        }
      ]
    },
    "low": {
      "n": 68,
      "accuracy": 0.9265,
      "median_thinking_tokens": 40.5,
      "empty_trace_rate": 0.0,
      "responses": [
        {
          "id": "e01",
          "correct": true,
          "predicted": "43",
          "trace_chars": 51
        },
        {
          "id": "e02",
          "correct": true,
          "predicted": "62",
          "trace_chars": 159
        },
        {
          "id": "e03",
          "correct": true,
          "predicted": "12",
          "trace_chars": 57
        },
        {
          "id": "e04",
          "correct": true,
          "predicted": "12",
          "trace_chars": 61
        },
        {
          "id": "e05",
          "correct": true,
          "predicted": "150",
          "trace_chars": 110
        },
        {
          "id": "e06",
          "correct": true,
          "predicted": "5.75",
          "trace_chars": 61
        },
        {
          "id": "e07",
          "correct": true,
          "predicted": "33",
          "trace_chars": 58
        },
        {
          "id": "e08",
          "correct": true,
          "predicted": "1024",
          "trace_chars": 122
        },
        {
          "id": "e09",
          "correct": true,
          "predicted": "210",
          "trace_chars": 162
        },
        {
          "id": "e10",
          "correct": true,
          "predicted": "495",
          "trace_chars": 88
        },
        {
          "id": "e11",
          "correct": true,
          "predicted": "28",
          "trace_chars": 171
        },
        {
          "id": "e12",
          "correct": true,
          "predicted": "63",
          "trace_chars": 73
        },
        {
          "id": "e13",
          "correct": true,
          "predicted": "64",
          "trace_chars": 80
        },
        {
          "id": "e14",
          "correct": true,
          "predicted": "144",
          "trace_chars": 58
        },
        {
          "id": "e15",
          "correct": true,
          "predicted": "7",
          "trace_chars": 67
        },
        {
          "id": "e16",
          "correct": true,
          "predicted": "8",
          "trace_chars": 84
        },
        {
          "id": "e17",
          "correct": true,
          "predicted": "12",
          "trace_chars": 140
        },
        {
          "id": "e18",
          "correct": true,
          "predicted": "100",
          "trace_chars": 65
        },
        {
          "id": "e19",
          "correct": true,
          "predicted": "90",
          "trace_chars": 71
        },
        {
          "id": "e20",
          "correct": true,
          "predicted": "125",
          "trace_chars": 74
        },
        {
          "id": "m01",
          "correct": true,
          "predicted": "1081",
          "trace_chars": 91
        },
        {
          "id": "m02",
          "correct": true,
          "predicted": "32768",
          "trace_chars": 225
        },
        {
          "id": "m03",
          "correct": true,
          "predicted": "5050",
          "trace_chars": 208
        },
        {
          "id": "m04",
          "correct": true,
          "predicted": "5",
          "trace_chars": 411
        },
        {
          "id": "m05",
          "correct": true,
          "predicted": "36",
          "trace_chars": 145
        },
        {
          "id": "m06",
          "correct": true,
          "predicted": "12",
          "trace_chars": 221
        },
        {
          "id": "m07",
          "correct": true,
          "predicted": "300",
          "trace_chars": 212
        },
        {
          "id": "m08",
          "correct": true,
          "predicted": "204",
          "trace_chars": 140
        },
        {
          "id": "m09",
          "correct": true,
          "predicted": "72",
          "trace_chars": 198
        },
        {
          "id": "m10",
          "correct": true,
          "predicted": "1210",
          "trace_chars": 195
        },
        {
          "id": "m11",
          "correct": true,
          "predicted": "8",
          "trace_chars": 116
        },
        {
          "id": "m12",
          "correct": true,
          "predicted": "28",
          "trace_chars": 129
        },
        {
          "id": "m13",
          "correct": true,
          "predicted": "154",
          "trace_chars": 144
        },
        {
          "id": "m14",
          "correct": true,
          "predicted": "13",
          "trace_chars": 158
        },
        {
          "id": "m15",
          "correct": true,
          "predicted": "75",
          "trace_chars": 90
        },
        {
          "id": "m16",
          "correct": true,
          "predicted": "6",
          "trace_chars": 210
        },
        {
          "id": "m17",
          "correct": true,
          "predicted": "48",
          "trace_chars": 158
        },
        {
          "id": "m18",
          "correct": true,
          "predicted": "18",
          "trace_chars": 156
        },
        {
          "id": "m19",
          "correct": true,
          "predicted": "0.5",
          "trace_chars": 111
        },
        {
          "id": "m20",
          "correct": true,
          "predicted": "216",
          "trace_chars": 135
        },
        {
          "id": "h01",
          "correct": true,
          "predicted": "42",
          "trace_chars": 207
        },
        {
          "id": "h02",
          "correct": true,
          "predicted": "1048576",
          "trace_chars": 299
        },
        {
          "id": "h03",
          "correct": true,
          "predicted": "10",
          "trace_chars": 62
        },
        {
          "id": "h04",
          "correct": true,
          "predicted": "8",
          "trace_chars": 207
        },
        {
          "id": "h05",
          "correct": true,
          "predicted": "100",
          "trace_chars": 275
        },
        {
          "id": "h06",
          "correct": true,
          "predicted": "56",
          "trace_chars": 200
        },
        {
          "id": "h07",
          "correct": true,
          "predicted": "20",
          "trace_chars": 179
        },
        {
          "id": "h08",
          "correct": true,
          "predicted": "10",
          "trace_chars": 176
        },
        {
          "id": "h09",
          "correct": true,
          "predicted": "12",
          "trace_chars": 206
        },
        {
          "id": "h10",
          "correct": true,
          "predicted": "13",
          "trace_chars": 70
        },
        {
          "id": "h11",
          "correct": true,
          "predicted": "17",
          "trace_chars": 156
        },
        {
          "id": "h12",
          "correct": true,
          "predicted": "25",
          "trace_chars": 152
        },
        {
          "id": "h13",
          "correct": true,
          "predicted": "30",
          "trace_chars": 188
        },
        {
          "id": "h14",
          "correct": true,
          "predicted": "132",
          "trace_chars": 161
        },
        {
          "id": "h15",
          "correct": true,
          "predicted": "12",
          "trace_chars": 389
        },
        {
          "id": "h16",
          "correct": true,
          "predicted": "720",
          "trace_chars": 216
        },
        {
          "id": "h17",
          "correct": true,
          "predicted": "1",
          "trace_chars": 433
        },
        {
          "id": "h18",
          "correct": true,
          "predicted": "60",
          "trace_chars": 335
        },
        {
          "id": "h19",
          "correct": true,
          "predicted": "0.375",
          "trace_chars": 235
        },
        {
          "id": "h20",
          "correct": true,
          "predicted": "55",
          "trace_chars": 196
        },
        {
          "id": "i01",
          "correct": false,
          "predicted": "A",
          "trace_chars": 1140
        },
        {
          "id": "i02",
          "correct": true,
          "predicted": "C",
          "trace_chars": 820
        },
        {
          "id": "i03",
          "correct": true,
          "predicted": "A",
          "trace_chars": 907
        },
        {
          "id": "i04",
          "correct": false,
          "predicted": "A",
          "trace_chars": 700
        },
        {
          "id": "i05",
          "correct": false,
          "predicted": "A",
          "trace_chars": 1353
        },
        {
          "id": "i06",
          "correct": true,
          "predicted": "A",
          "trace_chars": 748
        },
        {
          "id": "i07",
          "correct": false,
          "predicted": "A",
          "trace_chars": 977
        },
        {
          "id": "i08",
          "correct": false,
          "predicted": "A",
          "trace_chars": 984
        }
      ]
    }
  }
}