| { |
| "experiment": "D2 \u2014 cross-validating CUDA-scored reads against CPU-scored reads", |
| "status": "DIAGNOSTIC. Joins two stored results files; measures nothing new.", |
| "why": "CONTEXT.md \u00a77: E65's CUDA scoring branch is wrong by 37% on an arm with no operator, and ladder410/ladder1b are CUDA-scored and feed the asymptotes and the interaction. No CUDA-vs-CPU cross-check had ever been run.", |
| "inputs": [ |
| "results/ladder410/*.json (device=cuda)", |
| "results/e48_crossover_410m.json (device=cpu)" |
| ], |
| "shared_conditions": { |
| "lenses": "results/e28_<corpus>_410m_n200_s<seed>.pt", |
| "band": [ |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21 |
| ], |
| "K": [ |
| 1, |
| 2, |
| 5, |
| 10, |
| 20, |
| 50, |
| 100 |
| ], |
| "admitted_sets": [ |
| "multihop", |
| "multilingual", |
| "order-ops", |
| "poetry", |
| "typo" |
| ], |
| "rung": 200 |
| }, |
| "ladder_devices_seen": [ |
| "cuda" |
| ], |
| "tolerance": 1e-06, |
| "tolerance_derivation": "one (item,intermediate) pair flipping at one k moves a set mean by >= 1/(893*7) = 1.6e-4, so 1e-6 separates fp accumulation from a single changed decision", |
| "n_cells_compared": 30, |
| "max_abs_diff": 0.0002773862864289822, |
| "worst_cell": { |
| "corpus": "USPTO_Backgrounds", |
| "seed": 2, |
| "aggregation": "min", |
| "cuda_ladder": 0.1574542057301317, |
| "cpu_e48": 0.1571768194437027, |
| "abs_diff": 0.0002773862864289822, |
| "rel_diff": 0.0017648040430563356 |
| }, |
| "per_cell": [ |
| { |
| "corpus": "Github", |
| "seed": 0, |
| "aggregation": "persist", |
| "cuda_ladder": 0.02704486567527056, |
| "cpu_e48": 0.02704486474394798, |
| "abs_diff": 9.313225780849255e-10, |
| "rel_diff": 3.443620764616077e-08 |
| }, |
| { |
| "corpus": "Github", |
| "seed": 0, |
| "aggregation": "min", |
| "cuda_ladder": 0.10961288312183959, |
| "cpu_e48": 0.10961288511753083, |
| "abs_diff": 1.995691237266506e-09, |
| "rel_diff": 1.8206721181790396e-08 |
| }, |
| { |
| "corpus": "Github", |
| "seed": 1, |
| "aggregation": "persist", |
| "cuda_ladder": 0.029035190998443535, |
| "cpu_e48": 0.02903519310057163, |
| "abs_diff": 2.1021280965927147e-09, |
| "rel_diff": 7.23993151797337e-08 |
| }, |
| { |
| "corpus": "Github", |
| "seed": 1, |
| "aggregation": "min", |
| "cuda_ladder": 0.11494814133537667, |
| "cpu_e48": 0.1149481400847435, |
| "abs_diff": 1.2506331803496806e-09, |
| "rel_diff": 1.0879977522278075e-08 |
| }, |
| { |
| "corpus": "Github", |
| "seed": 2, |
| "aggregation": "persist", |
| "cuda_ladder": 0.027610584854015283, |
| "cpu_e48": 0.027610587328672408, |
| "abs_diff": 2.4746571250511273e-09, |
| "rel_diff": 8.962710918073453e-08 |
| }, |
| { |
| "corpus": "Github", |
| "seed": 2, |
| "aggregation": "min", |
| "cuda_ladder": 0.11328674435083355, |
| "cpu_e48": 0.11328674778342247, |
| "abs_diff": 3.432588921437052e-09, |
| "rel_diff": 3.0300004092264633e-08 |
| }, |
| { |
| "corpus": "Pile-CC", |
| "seed": 0, |
| "aggregation": "persist", |
| "cuda_ladder": 0.04509159454277583, |
| "cpu_e48": 0.04509159799199551, |
| "abs_diff": 3.449219680706772e-09, |
| "rel_diff": 7.649362263273668e-08 |
| }, |
| { |
| "corpus": "Pile-CC", |
| "seed": 0, |
| "aggregation": "min", |
| "cuda_ladder": 0.14523330333509615, |
| "cpu_e48": 0.1452333066612482, |
| "abs_diff": 3.3261520482330553e-09, |
| "rel_diff": 2.2902129853664994e-08 |
| }, |
| { |
| "corpus": "Pile-CC", |
| "seed": 1, |
| "aggregation": "persist", |
| "cuda_ladder": 0.04611681202160461, |
| "cpu_e48": 0.046116814273409544, |
| "abs_diff": 2.2518049369590898e-09, |
| "rel_diff": 4.882828470346999e-08 |
| }, |
| { |
| "corpus": "Pile-CC", |
| "seed": 1, |
| "aggregation": "min", |
| "cuda_ladder": 0.1453225617696132, |
| "cpu_e48": 0.1453225638717413, |
| "abs_diff": 2.1021281104705025e-09, |
| "rel_diff": 1.4465256147873893e-08 |
| }, |
| { |
| "corpus": "Pile-CC", |
| "seed": 2, |
| "aggregation": "persist", |
| "cuda_ladder": 0.045189316916678636, |
| "cpu_e48": 0.04518931868951768, |
| "abs_diff": 1.7728390422355744e-09, |
| "rel_diff": 3.923137355568962e-08 |
| }, |
| { |
| "corpus": "Pile-CC", |
| "seed": 2, |
| "aggregation": "min", |
| "cuda_ladder": 0.14300646082098994, |
| "cpu_e48": 0.14300646334886552, |
| "abs_diff": 2.5278755755309135e-09, |
| "rel_diff": 1.767665262348415e-08 |
| }, |
| { |
| "corpus": "StackExchange", |
| "seed": 0, |
| "aggregation": "persist", |
| "cuda_ladder": 0.04637858061386006, |
| "cpu_e48": 0.04637858076021075, |
| "abs_diff": 1.4635068851243105e-10, |
| "rel_diff": 3.1555663436339706e-09 |
| }, |
| { |
| "corpus": "StackExchange", |
| "seed": 0, |
| "aggregation": "min", |
| "cuda_ladder": 0.14699955502791065, |
| "cpu_e48": 0.1469995591789484, |
| "abs_diff": 4.151037763522325e-09, |
| "rel_diff": 2.8238436813739703e-08 |
| }, |
| { |
| "corpus": "StackExchange", |
| "seed": 1, |
| "aggregation": "persist", |
| "cuda_ladder": 0.04555068582828556, |
| "cpu_e48": 0.04555068640038371, |
| "abs_diff": 5.72098153572842e-10, |
| "rel_diff": 1.2559594569973873e-08 |
| }, |
| { |
| "corpus": "StackExchange", |
| "seed": 1, |
| "aggregation": "min", |
| "cuda_ladder": 0.14416987733649356, |
| "cpu_e48": 0.14416988678276538, |
| "abs_diff": 9.446271820312546e-09, |
| "rel_diff": 6.552180924263437e-08 |
| }, |
| { |
| "corpus": "StackExchange", |
| "seed": 2, |
| "aggregation": "persist", |
| "cuda_ladder": 0.045260619930922986, |
| "cpu_e48": 0.04526062114164233, |
| "abs_diff": 1.2107193442245645e-09, |
| "rel_diff": 2.6749949816986367e-08 |
| }, |
| { |
| "corpus": "StackExchange", |
| "seed": 2, |
| "aggregation": "min", |
| "cuda_ladder": 0.14551294004278523, |
| "cpu_e48": 0.14551294930279254, |
| "abs_diff": 9.26000731649168e-09, |
| "rel_diff": 6.36369983624129e-08 |
| }, |
| { |
| "corpus": "USPTO_Backgrounds", |
| "seed": 0, |
| "aggregation": "persist", |
| "cuda_ladder": 0.04964359144547156, |
| "cpu_e48": 0.04964358932338655, |
| "abs_diff": 2.1220850146552728e-09, |
| "rel_diff": 4.274640580139482e-08 |
| }, |
| { |
| "corpus": "USPTO_Backgrounds", |
| "seed": 0, |
| "aggregation": "min", |
| "cuda_ladder": 0.15465880749481065, |
| "cpu_e48": 0.15465880744159222, |
| "abs_diff": 5.3218429663104416e-11, |
| "rel_diff": 3.4410215973767067e-10 |
| }, |
| { |
| "corpus": "USPTO_Backgrounds", |
| "seed": 1, |
| "aggregation": "persist", |
| "cuda_ladder": 0.04933665746024677, |
| "cpu_e48": 0.04933666000142693, |
| "abs_diff": 2.541180162130008e-09, |
| "rel_diff": 5.150693545238999e-08 |
| }, |
| { |
| "corpus": "USPTO_Backgrounds", |
| "seed": 1, |
| "aggregation": "min", |
| "cuda_ladder": 0.15324432841901267, |
| "cpu_e48": 0.153244324401021, |
| "abs_diff": 4.017991661608988e-09, |
| "rel_diff": 2.6219513690402082e-08 |
| }, |
| { |
| "corpus": "USPTO_Backgrounds", |
| "seed": 2, |
| "aggregation": "persist", |
| "cuda_ladder": 0.04986586142331362, |
| "cpu_e48": 0.04986586356535554, |
| "abs_diff": 2.142041918840043e-09, |
| "rel_diff": 4.295607787946208e-08 |
| }, |
| { |
| "corpus": "USPTO_Backgrounds", |
| "seed": 2, |
| "aggregation": "min", |
| "cuda_ladder": 0.1574542057301317, |
| "cpu_e48": 0.1571768194437027, |
| "abs_diff": 0.0002773862864289822, |
| "rel_diff": 0.0017648040430563356 |
| }, |
| { |
| "corpus": "Wikipedia_en", |
| "seed": 0, |
| "aggregation": "persist", |
| "cuda_ladder": 0.03809930235147477, |
| "cpu_e48": 0.038099304959177974, |
| "abs_diff": 2.6077032061477823e-09, |
| "rel_diff": 6.844490231362073e-08 |
| }, |
| { |
| "corpus": "Wikipedia_en", |
| "seed": 0, |
| "aggregation": "min", |
| "cuda_ladder": 0.12926935620073762, |
| "cpu_e48": 0.12926936261355876, |
| "abs_diff": 6.412821135226565e-09, |
| "rel_diff": 4.960820573082906e-08 |
| }, |
| { |
| "corpus": "Wikipedia_en", |
| "seed": 1, |
| "aggregation": "persist", |
| "cuda_ladder": 0.03906317305352006, |
| "cpu_e48": 0.039063173532485965, |
| "abs_diff": 4.789659016624093e-10, |
| "rel_diff": 1.2261315667660452e-08 |
| }, |
| { |
| "corpus": "Wikipedia_en", |
| "seed": 1, |
| "aggregation": "min", |
| "cuda_ladder": 0.13214441066873925, |
| "cpu_e48": 0.1321444157510996, |
| "abs_diff": 5.0823603381378035e-09, |
| "rel_diff": 3.846065162307483e-08 |
| }, |
| { |
| "corpus": "Wikipedia_en", |
| "seed": 2, |
| "aggregation": "persist", |
| "cuda_ladder": 0.037466185060994965, |
| "cpu_e48": 0.03746618703007698, |
| "abs_diff": 1.9690820154960598e-09, |
| "rel_diff": 5.2556242617251837e-08 |
| }, |
| { |
| "corpus": "Wikipedia_en", |
| "seed": 2, |
| "aggregation": "min", |
| "cuda_ladder": 0.1300129838021738, |
| "cpu_e48": 0.13001298978924752, |
| "abs_diff": 5.9870737256773054e-09, |
| "rel_diff": 4.6049811910197724e-08 |
| } |
| ], |
| "VERDICT": "DIVERGENT \u2014 max |CUDA - CPU| = 2.774e-04 over 30 cells (worst: USPTO_Backgrounds s2, min, 0.18% relative). STOP. Every ladder-derived number in the paper, meaning the per-corpus asymptotes, the flatness in N and the 1B replication, must be re-sourced from a CPU scoring pass before it is cited.", |
| "provenance": { |
| "schema_version": 1, |
| "experiment": "D2 \u2014 cross-validating CUDA-scored reads against CPU-scored reads", |
| "utc": "2026-08-20T00:11:35+00:00", |
| "script": { |
| "path": "experiments/d2_device_crossvalidation.py", |
| "sha256": "e1033efb2cbd9ea71d60d628b4f33ee8fcf866946445cec4a1d054a28046c916", |
| "n_lines": 125 |
| }, |
| "git": { |
| "commit": "2dfeda4b4d89650f970dc362f8183e2f6077c9fe", |
| "commit_short": "2dfeda4b4d89", |
| "branch": "pythia", |
| "dirty": true, |
| "dirty_files": [ |
| "experiments/d2_device_crossvalidation.py", |
| "figures/fB_band_width.png", |
| "figures/make_figures.py", |
| "igures/f0_grid_single.png", |
| "paper/paper.tex", |
| "paper/paper_5pp.bbl", |
| "paper/paper_5pp.blg", |
| "paper/paper_5pp.tex", |
| "paper/paper_9pp.tex", |
| "paper/refs.bib", |
| "paper/tables_auto.tex", |
| "tools/make_paper_tables.py" |
| ], |
| "dirty_diff_sha256": "0b3cf18514db7d8a624429659f62d811e911c001c158012b785b6f8927d12df6", |
| "describe": "2dfeda4-dirty" |
| }, |
| "argv": [ |
| "experiments/d2_device_crossvalidation.py" |
| ], |
| "cwd": ".", |
| "inputs": [ |
| { |
| "path": "results/e48_crossover_410m.json", |
| "sha256": "7335aab33c2ea0b6bc3e9deec90181823ab2fb68202626a66c8658ad2cae4051", |
| "bytes": 116036, |
| "exists": true |
| } |
| ], |
| "env": { |
| "python": "3.14.2", |
| "torch": "2.13.0", |
| "transformers": "5.14.1", |
| "numpy": "2.5.1", |
| "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "hostname": "<redacted for double-blind review>", |
| "tf32": { |
| "allow_tf32_matmul": false, |
| "allow_tf32_cudnn": true, |
| "NVIDIA_TF32_OVERRIDE": null |
| } |
| }, |
| "payload_sha256": "e3c1b0af60b548841f62ecd52aba7dcc0e8eb5db7f7528a2aee67a2044f53858" |
| } |
| } |
