File size: 14,293 Bytes
8399c74 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 | {
"component": "kda",
"gpu": "NVIDIA B300 SXM6 AC",
"torch": "2.12.1+cu130",
"method": "Identical inputs for both implementations, original judge unchanged, fp64 rebuilt truth.",
"rows": [
{
"batch_size": 16,
"family": "structured",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0023404883445068,
"max_individual_head_error_ratio": 1.006074351153129,
"individual_states_checked": 512,
"individual_states_passing_same_bound": 512,
"max_state_relative_mae": 0.005814296599920318
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0003884242962537,
"max_individual_head_error_ratio": 1.0057190231914948,
"individual_states_checked": 512,
"individual_states_passing_same_bound": 512,
"max_state_relative_mae": 0.005805816791951239
}
},
{
"batch_size": 16,
"family": "second_window",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.001995921353665,
"max_individual_head_error_ratio": 1.0066544648366493,
"individual_states_checked": 512,
"individual_states_passing_same_bound": 512,
"max_state_relative_mae": 0.0036491455137833747
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0005873698335468,
"max_individual_head_error_ratio": 1.0027614379675776,
"individual_states_checked": 512,
"individual_states_passing_same_bound": 512,
"max_state_relative_mae": 0.0036486360307457348
}
},
{
"batch_size": 16,
"family": "random",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0029677262722476,
"max_individual_head_error_ratio": 1.0084967170909311,
"individual_states_checked": 512,
"individual_states_passing_same_bound": 512,
"max_state_relative_mae": 0.004294319736538677
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0010557299854554,
"max_individual_head_error_ratio": 1.0074521427695602,
"individual_states_checked": 512,
"individual_states_passing_same_bound": 512,
"max_state_relative_mae": 0.004283246255273555
}
},
{
"batch_size": 32,
"family": "structured",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.001859667592005,
"max_individual_head_error_ratio": 1.007288920527367,
"individual_states_checked": 1024,
"individual_states_passing_same_bound": 1024,
"max_state_relative_mae": 0.00613288647257253
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.000374559298278,
"max_individual_head_error_ratio": 1.006046490834228,
"individual_states_checked": 1024,
"individual_states_passing_same_bound": 1024,
"max_state_relative_mae": 0.006122237864011208
}
},
{
"batch_size": 32,
"family": "second_window",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0018781088903055,
"max_individual_head_error_ratio": 1.0087159657086429,
"individual_states_checked": 1024,
"individual_states_passing_same_bound": 1024,
"max_state_relative_mae": 0.003767960967271697
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0004346967793898,
"max_individual_head_error_ratio": 1.0039219403792172,
"individual_states_checked": 1024,
"individual_states_passing_same_bound": 1024,
"max_state_relative_mae": 0.00376028316237159
}
},
{
"batch_size": 32,
"family": "random",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0027823839230108,
"max_individual_head_error_ratio": 1.0087438653780934,
"individual_states_checked": 1024,
"individual_states_passing_same_bound": 1024,
"max_state_relative_mae": 0.004820014387124738
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.00049773584226,
"max_individual_head_error_ratio": 1.005876845742881,
"individual_states_checked": 1024,
"individual_states_passing_same_bound": 1024,
"max_state_relative_mae": 0.004801880824288819
}
},
{
"batch_size": 64,
"family": "structured",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0015597078846517,
"max_individual_head_error_ratio": 1.0115407039016826,
"individual_states_checked": 2048,
"individual_states_passing_same_bound": 2047,
"max_state_relative_mae": 0.006038983078424852
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0004498411706473,
"max_individual_head_error_ratio": 1.0071467598471255,
"individual_states_checked": 2048,
"individual_states_passing_same_bound": 2048,
"max_state_relative_mae": 0.006028942499950136
}
},
{
"batch_size": 64,
"family": "second_window",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.001658339876203,
"max_individual_head_error_ratio": 1.0079195057490604,
"individual_states_checked": 2048,
"individual_states_passing_same_bound": 2048,
"max_state_relative_mae": 0.0040383280854628945
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0003413479955345,
"max_individual_head_error_ratio": 1.0045972224378705,
"individual_states_checked": 2048,
"individual_states_passing_same_bound": 2048,
"max_state_relative_mae": 0.0040048074177080764
}
},
{
"batch_size": 64,
"family": "random",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0025982273756666,
"max_individual_head_error_ratio": 1.0144644709226924,
"individual_states_checked": 2048,
"individual_states_passing_same_bound": 2045,
"max_state_relative_mae": 0.0052767051398082635
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0006642867533797,
"max_individual_head_error_ratio": 1.0064506108252624,
"individual_states_checked": 2048,
"individual_states_passing_same_bound": 2048,
"max_state_relative_mae": 0.005257746277318221
}
},
{
"batch_size": 128,
"family": "structured",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0017192663876373,
"max_individual_head_error_ratio": 1.013238615585311,
"individual_states_checked": 4096,
"individual_states_passing_same_bound": 4095,
"max_state_relative_mae": 0.006119204227107976
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.000351543146309,
"max_individual_head_error_ratio": 1.0095542900602699,
"individual_states_checked": 4096,
"individual_states_passing_same_bound": 4096,
"max_state_relative_mae": 0.0061135900161442
}
},
{
"batch_size": 128,
"family": "second_window",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0016918815912272,
"max_individual_head_error_ratio": 1.0127116564076568,
"individual_states_checked": 4096,
"individual_states_passing_same_bound": 4095,
"max_state_relative_mae": 0.004024117782495586
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.000165086203439,
"max_individual_head_error_ratio": 1.004243908619348,
"individual_states_checked": 4096,
"individual_states_passing_same_bound": 4096,
"max_state_relative_mae": 0.004004769181208124
}
},
{
"batch_size": 128,
"family": "random",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0024349888670567,
"max_individual_head_error_ratio": 1.015271042075488,
"individual_states_checked": 4096,
"individual_states_passing_same_bound": 4090,
"max_state_relative_mae": 0.004977186593824846
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0003664275392226,
"max_individual_head_error_ratio": 1.012560569248684,
"individual_states_checked": 4096,
"individual_states_passing_same_bound": 4095,
"max_state_relative_mae": 0.004972631753234797
}
},
{
"batch_size": 256,
"family": "structured",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0015036845092724,
"max_individual_head_error_ratio": 1.0122499937166671,
"individual_states_checked": 8192,
"individual_states_passing_same_bound": 8189,
"max_state_relative_mae": 0.006023172788241781
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.000234577199579,
"max_individual_head_error_ratio": 1.0093183226995546,
"individual_states_checked": 8192,
"individual_states_passing_same_bound": 8192,
"max_state_relative_mae": 0.006027195902602814
}
},
{
"batch_size": 256,
"family": "second_window",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0015014152495132,
"max_individual_head_error_ratio": 1.0082640884380387,
"individual_states_checked": 8192,
"individual_states_passing_same_bound": 8192,
"max_state_relative_mae": 0.004341165236607238
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.000115893549605,
"max_individual_head_error_ratio": 1.0046930658564874,
"individual_states_checked": 8192,
"individual_states_passing_same_bound": 8192,
"max_state_relative_mae": 0.004349394686179305
}
},
{
"batch_size": 256,
"family": "random",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0022296136176574,
"max_individual_head_error_ratio": 1.0134402418693471,
"individual_states_checked": 8192,
"individual_states_passing_same_bound": 8185,
"max_state_relative_mae": 0.005144512890380753
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0002245855889362,
"max_individual_head_error_ratio": 1.0084053441575165,
"individual_states_checked": 8192,
"individual_states_passing_same_bound": 8192,
"max_state_relative_mae": 0.005129051333732747
}
},
{
"batch_size": 512,
"family": "structured",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.001585994222947,
"max_individual_head_error_ratio": 1.0143067085964312,
"individual_states_checked": 16384,
"individual_states_passing_same_bound": 16379,
"max_state_relative_mae": 0.0060472408037226056
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0000920190918832,
"max_individual_head_error_ratio": 1.0092341243405343,
"individual_states_checked": 16384,
"individual_states_passing_same_bound": 16384,
"max_state_relative_mae": 0.0060387970828057825
}
},
{
"batch_size": 512,
"family": "second_window",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0015524522196704,
"max_individual_head_error_ratio": 1.007722344227275,
"individual_states_checked": 16384,
"individual_states_passing_same_bound": 16384,
"max_state_relative_mae": 0.0046480774404553055
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0000797570481763,
"max_individual_head_error_ratio": 1.0045844945863005,
"individual_states_checked": 16384,
"individual_states_passing_same_bound": 16384,
"max_state_relative_mae": 0.004645123687680978
}
},
{
"batch_size": 512,
"family": "random",
"baseline": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0021663627472086,
"max_individual_head_error_ratio": 1.0131403603700213,
"individual_states_checked": 16384,
"individual_states_passing_same_bound": 16367,
"max_state_relative_mae": 0.005146918931758928
},
"current": {
"original_judge_passed": true,
"worst_head_error_ratio": 1.0001660871100089,
"max_individual_head_error_ratio": 1.0088966612517987,
"individual_states_checked": 16384,
"individual_states_passing_same_bound": 16384,
"max_state_relative_mae": 0.005133520960083807
}
}
],
"status": "original_benchmark_pass",
"global_seed_plus_batch": 20261026,
"relative_tolerance": 0.01,
"absolute_cap_factor": 0.001,
"reference_error_floor_factor": 0.0001,
"individual_state_check_is_additional": true,
"summary": {
"baseline": {
"original_cases_passed": 18,
"worst_original_benchmark_error_ratio": 1.0029677262722476,
"individual_states_checked": 96768,
"additional_individual_state_failures": 44,
"max_individual_head_error_ratio": 1.015271042075488
},
"current": {
"original_cases_passed": 18,
"worst_original_benchmark_error_ratio": 1.0010557299854554,
"individual_states_checked": 96768,
"additional_individual_state_failures": 1,
"max_individual_head_error_ratio": 1.012560569248684
}
}
}
|