| { | |
| "environment": { | |
| "model": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "revision": "989aa7980e4cf806f80c7fef2b1adb7bc71aa306", | |
| "torch": "2.11.0", | |
| "transformers": "4.57.6", | |
| "gpu": "mps", | |
| "platform": "macOS-26.6.2-arm64-arm-64bit", | |
| "dtype": "torch.float16", | |
| "attention": "sdpa" | |
| }, | |
| "arguments": { | |
| "model": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "presets": [ | |
| "fintech_fraud", | |
| "support_triage", | |
| "code_security" | |
| ], | |
| "fields": [ | |
| 4, | |
| 16, | |
| 28 | |
| ], | |
| "steps": 1, | |
| "repeats": 8, | |
| "warmup": 2, | |
| "output": "mac_mps_initial.json", | |
| "device": "mps", | |
| "dtype": "float16" | |
| }, | |
| "cases": [ | |
| { | |
| "preset": "fintech_fraud", | |
| "fields": 4, | |
| "prefix_tokens": 291, | |
| "suffix_tokens": 28, | |
| "batch_padded_tokens": 32, | |
| "correctness": { | |
| "max_abs_logit_error": 0.0390625, | |
| "mean_abs_logit_error": 0.0043123154900968075, | |
| "full_vocab_argmax_agreement": 1.0, | |
| "candidate_decisions_equal": true, | |
| "candidate_collision_fields": [] | |
| }, | |
| "decode": { | |
| "batch": { | |
| "median_ms": 47.26822950760834, | |
| "min_ms": 46.848250000039116, | |
| "p90_ms": 47.59587500302587, | |
| "samples_ms": [ | |
| 47.55291700712405, | |
| 47.04954198678024, | |
| 47.15175001183525, | |
| 47.52816700784024, | |
| 46.848250000039116, | |
| 47.05770799773745, | |
| 47.38470900338143, | |
| 47.59587500302587 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 33.41227099735988, | |
| "min_ms": 33.26170801301487, | |
| "p90_ms": 34.000125000602566, | |
| "samples_ms": [ | |
| 33.38362500653602, | |
| 33.440916988183744, | |
| 33.26170801301487, | |
| 33.57716600294225, | |
| 33.26737499446608, | |
| 33.32154100644402, | |
| 33.557167000253685, | |
| 34.000125000602566 | |
| ] | |
| } | |
| }, | |
| "end_to_end": { | |
| "batch": { | |
| "median_ms": 190.59108350484166, | |
| "min_ms": 189.7072909923736, | |
| "p90_ms": 192.90312500379514, | |
| "samples_ms": [ | |
| 191.3511250022566, | |
| 190.71195799915586, | |
| 190.8974169928115, | |
| 192.90312500379514, | |
| 190.13754100888036, | |
| 189.7072909923736, | |
| 189.88337500195485, | |
| 190.47020901052747 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 178.3205000028829, | |
| "min_ms": 177.8794160054531, | |
| "p90_ms": 178.77554200822487, | |
| "samples_ms": [ | |
| 178.30850000609644, | |
| 177.99999999988358, | |
| 178.43274999177083, | |
| 178.77554200822487, | |
| 178.43395899399184, | |
| 177.8794160054531, | |
| 177.99320800986607, | |
| 178.33249999966938 | |
| ] | |
| } | |
| }, | |
| "memory": { | |
| "batch": { | |
| "driver_high_water_mib": 3279.0, | |
| "extra_driver_mib": 112.0 | |
| }, | |
| "tree": { | |
| "driver_high_water_mib": 3199.0, | |
| "extra_driver_mib": 32.0 | |
| } | |
| }, | |
| "decode_speedup": 1.4146966996449688, | |
| "end_to_end_speedup": 1.0688119621791121, | |
| "decode_speedup_ci95": [ | |
| 1.4070674260782117, | |
| 1.4236970070965906 | |
| ], | |
| "end_to_end_speedup_ci95": [ | |
| 1.0659652991225994, | |
| 1.072772632844677 | |
| ] | |
| }, | |
| { | |
| "preset": "fintech_fraud", | |
| "fields": 16, | |
| "prefix_tokens": 468, | |
| "suffix_tokens": 127, | |
| "batch_padded_tokens": 160, | |
| "correctness": { | |
| "max_abs_logit_error": 0.046875, | |
| "mean_abs_logit_error": 0.005229064263403416, | |
| "full_vocab_argmax_agreement": 1.0, | |
| "candidate_decisions_equal": true, | |
| "candidate_collision_fields": [] | |
| }, | |
| "decode": { | |
| "batch": { | |
| "median_ms": 166.3752914973884, | |
| "min_ms": 166.01883299881592, | |
| "p90_ms": 166.64354100066703, | |
| "samples_ms": [ | |
| 166.01883299881592, | |
| 166.21908301021904, | |
| 166.53400000359397, | |
| 166.37366599752568, | |
| 166.27183300442994, | |
| 166.3769169972511, | |
| 166.59845800313633, | |
| 166.64354100066703 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 83.06970800185809, | |
| "min_ms": 82.75195899477694, | |
| "p90_ms": 83.72970900381915, | |
| "samples_ms": [ | |
| 82.94133399613202, | |
| 83.04516600037459, | |
| 83.72970900381915, | |
| 83.09425000334159, | |
| 83.04158299870323, | |
| 82.75195899477694, | |
| 83.13174999784678, | |
| 83.20774999447167 | |
| ] | |
| } | |
| }, | |
| "end_to_end": { | |
| "batch": { | |
| "median_ms": 400.43991649872623, | |
| "min_ms": 400.2200419927249, | |
| "p90_ms": 402.33149999403395, | |
| "samples_ms": [ | |
| 402.33149999403395, | |
| 400.3236669959733, | |
| 400.2200419927249, | |
| 400.5150419980055, | |
| 400.364790999447, | |
| 400.3123750007944, | |
| 401.0615840088576, | |
| 401.5582089923555 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 317.8602914995281, | |
| "min_ms": 317.6447910082061, | |
| "p90_ms": 318.26429200009443, | |
| "samples_ms": [ | |
| 317.882292001741, | |
| 317.8382909973152, | |
| 317.65129198902287, | |
| 317.7312080078991, | |
| 317.91475000500213, | |
| 317.6447910082061, | |
| 318.1417499872623, | |
| 318.26429200009443 | |
| ] | |
| } | |
| }, | |
| "memory": { | |
| "batch": { | |
| "driver_high_water_mib": 5620.40625, | |
| "extra_driver_mib": 2352.0 | |
| }, | |
| "tree": { | |
| "driver_high_water_mib": 3276.40625, | |
| "extra_driver_mib": 8.0 | |
| } | |
| }, | |
| "decode_speedup": 2.00283948865773, | |
| "end_to_end_speedup": 1.25979849389058, | |
| "decode_speedup_ci95": [ | |
| 2.0015503733169644, | |
| 2.00498794137586 | |
| ], | |
| "end_to_end_speedup_ci95": [ | |
| 1.259346384504486, | |
| 1.2624706917901076 | |
| ] | |
| }, | |
| { | |
| "preset": "fintech_fraud", | |
| "fields": 28, | |
| "prefix_tokens": 646, | |
| "suffix_tokens": 227, | |
| "batch_padded_tokens": 308, | |
| "correctness": { | |
| "max_abs_logit_error": 0.0390625, | |
| "mean_abs_logit_error": 0.004572090692818165, | |
| "full_vocab_argmax_agreement": 1.0, | |
| "candidate_decisions_equal": true, | |
| "candidate_collision_fields": [ | |
| "counterparty_jurisdiction_risk" | |
| ] | |
| }, | |
| "decode": { | |
| "batch": { | |
| "median_ms": 337.68993749981746, | |
| "min_ms": 337.26487500825897, | |
| "p90_ms": 338.20399999967776, | |
| "samples_ms": [ | |
| 338.20399999967776, | |
| 337.63237499806564, | |
| 337.26487500825897, | |
| 337.7475000015693, | |
| 337.82583399442956, | |
| 337.49304099183064, | |
| 337.31808399898, | |
| 337.8040000097826 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 142.53641650429927, | |
| "min_ms": 142.30208400113042, | |
| "p90_ms": 143.00833299057558, | |
| "samples_ms": [ | |
| 142.60750000539701, | |
| 142.54791699931957, | |
| 142.32374999846797, | |
| 142.30208400113042, | |
| 142.68516600714065, | |
| 143.00833299057558, | |
| 142.44258399412502, | |
| 142.52491600927897 | |
| ] | |
| } | |
| }, | |
| "end_to_end": { | |
| "batch": { | |
| "median_ms": 665.5089580017375, | |
| "min_ms": 664.9612079927465, | |
| "p90_ms": 671.5790419984842, | |
| "samples_ms": [ | |
| 671.5790419984842, | |
| 665.4760410019662, | |
| 666.8390420090873, | |
| 665.4567090008641, | |
| 664.9612079927465, | |
| 665.394916999503, | |
| 665.5418750015087, | |
| 668.8652919983724 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 471.9899789997726, | |
| "min_ms": 470.15170801023487, | |
| "p90_ms": 473.9503750024596, | |
| "samples_ms": [ | |
| 472.25875000003725, | |
| 473.07704100967385, | |
| 471.5012079977896, | |
| 471.7212079995079, | |
| 472.3985000018729, | |
| 470.7340419990942, | |
| 470.15170801023487, | |
| 473.9503750024596 | |
| ] | |
| } | |
| }, | |
| "memory": { | |
| "batch": { | |
| "driver_high_water_mib": 5294.984375, | |
| "extra_driver_mib": 2022.0 | |
| }, | |
| "tree": { | |
| "driver_high_water_mib": 3270.984375, | |
| "extra_driver_mib": 40.0 | |
| } | |
| }, | |
| "decode_speedup": 2.369148501005228, | |
| "end_to_end_speedup": 1.4100065416898568, | |
| "decode_speedup_ci95": [ | |
| 2.366431024764677, | |
| 2.3721027360653832 | |
| ], | |
| "end_to_end_speedup_ci95": [ | |
| 1.4077065183304176, | |
| 1.4155099921366008 | |
| ] | |
| }, | |
| { | |
| "preset": "support_triage", | |
| "fields": 4, | |
| "prefix_tokens": 298, | |
| "suffix_tokens": 28, | |
| "batch_padded_tokens": 40, | |
| "correctness": { | |
| "max_abs_logit_error": 0.0390625, | |
| "mean_abs_logit_error": 0.004746712744235992, | |
| "full_vocab_argmax_agreement": 1.0, | |
| "candidate_decisions_equal": true, | |
| "candidate_collision_fields": [] | |
| }, | |
| "decode": { | |
| "batch": { | |
| "median_ms": 51.41572900902247, | |
| "min_ms": 50.98366699530743, | |
| "p90_ms": 52.308957994682714, | |
| "samples_ms": [ | |
| 51.25137499999255, | |
| 50.98366699530743, | |
| 51.47558300814126, | |
| 51.355875009903684, | |
| 52.004624987603165, | |
| 52.308957994682714, | |
| 51.34412499319296, | |
| 51.97441601194441 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 33.48941699368879, | |
| "min_ms": 33.176375000039116, | |
| "p90_ms": 35.18329199869186, | |
| "samples_ms": [ | |
| 33.21337500528898, | |
| 33.488749992102385, | |
| 33.33837499667425, | |
| 33.89491600682959, | |
| 35.18329199869186, | |
| 33.57179199520033, | |
| 33.176375000039116, | |
| 33.4900839952752 | |
| ] | |
| } | |
| }, | |
| "end_to_end": { | |
| "batch": { | |
| "median_ms": 194.8159375024261, | |
| "min_ms": 194.30545800423715, | |
| "p90_ms": 196.05383300222456, | |
| "samples_ms": [ | |
| 194.53891599550843, | |
| 194.30545800423715, | |
| 194.7874170000432, | |
| 194.8301250085933, | |
| 196.05383300222456, | |
| 195.8960420015501, | |
| 195.53312499192543, | |
| 194.80174999625888 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 179.28256250161212, | |
| "min_ms": 178.82008400920313, | |
| "p90_ms": 181.21854099445045, | |
| "samples_ms": [ | |
| 179.84216600598302, | |
| 179.1274169954704, | |
| 178.82008400920313, | |
| 178.8337500038324, | |
| 180.26720799389295, | |
| 181.21854099445045, | |
| 179.43770800775383, | |
| 179.10737499187235 | |
| ] | |
| } | |
| }, | |
| "memory": { | |
| "batch": { | |
| "driver_high_water_mib": 3359.15625, | |
| "extra_driver_mib": 104.0 | |
| }, | |
| "tree": { | |
| "driver_high_water_mib": 3263.15625, | |
| "extra_driver_mib": 8.0 | |
| } | |
| }, | |
| "decode_speedup": 1.5352828930617681, | |
| "end_to_end_speedup": 1.0866418617854947, | |
| "decode_speedup_ci95": [ | |
| 1.5224064292082697, | |
| 1.5519344776584314 | |
| ], | |
| "end_to_end_speedup_ci95": [ | |
| 1.0831418894925375, | |
| 1.0896991895565011 | |
| ] | |
| }, | |
| { | |
| "preset": "support_triage", | |
| "fields": 16, | |
| "prefix_tokens": 481, | |
| "suffix_tokens": 126, | |
| "batch_padded_tokens": 160, | |
| "correctness": { | |
| "max_abs_logit_error": 0.046875, | |
| "mean_abs_logit_error": 0.0048467740416526794, | |
| "full_vocab_argmax_agreement": 1.0, | |
| "candidate_decisions_equal": true, | |
| "candidate_collision_fields": [] | |
| }, | |
| "decode": { | |
| "batch": { | |
| "median_ms": 168.7676039946382, | |
| "min_ms": 168.39800000889227, | |
| "p90_ms": 169.3617080018157, | |
| "samples_ms": [ | |
| 168.98479100200348, | |
| 168.74533299414907, | |
| 169.21074999845587, | |
| 168.39800000889227, | |
| 168.68324999813922, | |
| 168.61854199669324, | |
| 168.78987499512732, | |
| 169.3617080018157 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 83.00702099950286, | |
| "min_ms": 82.86941600090358, | |
| "p90_ms": 83.5149589984212, | |
| "samples_ms": [ | |
| 83.5149589984212, | |
| 82.96704200620297, | |
| 83.02208301029168, | |
| 82.99604199419264, | |
| 82.9257090081228, | |
| 82.86941600090358, | |
| 83.01800000481308, | |
| 83.453457991709 | |
| ] | |
| } | |
| }, | |
| "end_to_end": { | |
| "batch": { | |
| "median_ms": 408.5541460008244, | |
| "min_ms": 407.3758330050623, | |
| "p90_ms": 411.9953330082353, | |
| "samples_ms": [ | |
| 409.455084009096, | |
| 408.30883300805, | |
| 407.3758330050623, | |
| 407.4040419945959, | |
| 408.7994589935988, | |
| 407.83354100130964, | |
| 408.8325830089161, | |
| 411.9953330082353 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 322.66933350183535, | |
| "min_ms": 321.90650000120513, | |
| "p90_ms": 325.09766600560397, | |
| "samples_ms": [ | |
| 322.21112499246374, | |
| 325.09766600560397, | |
| 322.2837920038728, | |
| 321.90650000120513, | |
| 323.0462500068825, | |
| 322.482667004806, | |
| 323.7660419981694, | |
| 322.8559999988647 | |
| ] | |
| } | |
| }, | |
| "memory": { | |
| "batch": { | |
| "driver_high_water_mib": 5599.484375, | |
| "extra_driver_mib": 2352.0 | |
| }, | |
| "tree": { | |
| "driver_high_water_mib": 3271.484375, | |
| "extra_driver_mib": 24.0 | |
| } | |
| }, | |
| "decode_speedup": 2.033172639645133, | |
| "end_to_end_speedup": 1.2661697396740696, | |
| "decode_speedup_ci95": [ | |
| 2.0271568557257, | |
| 2.036780629549704 | |
| ], | |
| "end_to_end_speedup_ci95": [ | |
| 1.2619319354399532, | |
| 1.2698004562026703 | |
| ] | |
| }, | |
| { | |
| "preset": "support_triage", | |
| "fields": 28, | |
| "prefix_tokens": 655, | |
| "suffix_tokens": 218, | |
| "batch_padded_tokens": 280, | |
| "correctness": { | |
| "max_abs_logit_error": 0.046875, | |
| "mean_abs_logit_error": 0.004553103819489479, | |
| "full_vocab_argmax_agreement": 1.0, | |
| "candidate_decisions_equal": true, | |
| "candidate_collision_fields": [ | |
| "target_resolution_hours", | |
| "assigned_agent_tier" | |
| ] | |
| }, | |
| "decode": { | |
| "batch": { | |
| "median_ms": 333.23189599468606, | |
| "min_ms": 332.5708750053309, | |
| "p90_ms": 338.84008300083224, | |
| "samples_ms": [ | |
| 333.4462920029182, | |
| 332.5708750053309, | |
| 333.0174999864539, | |
| 333.4563749958761, | |
| 338.84008300083224, | |
| 333.65920900541823, | |
| 332.6203750038985, | |
| 332.86400001088623 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 140.46581200091168, | |
| "min_ms": 140.3020409925375, | |
| "p90_ms": 141.14466699538752, | |
| "samples_ms": [ | |
| 141.14466699538752, | |
| 140.44829200429376, | |
| 140.45941599761136, | |
| 140.65008399484213, | |
| 141.10125000297558, | |
| 140.3020409925375, | |
| 140.472208004212, | |
| 140.3820419945987 | |
| ] | |
| } | |
| }, | |
| "end_to_end": { | |
| "batch": { | |
| "median_ms": 664.504312997451, | |
| "min_ms": 662.5099170050817, | |
| "p90_ms": 666.6135840059724, | |
| "samples_ms": [ | |
| 664.545999999973, | |
| 664.5135419967119, | |
| 662.5099170050817, | |
| 664.9717500113184, | |
| 664.3663330032723, | |
| 666.6135840059724, | |
| 664.3937079934403, | |
| 664.4950839981902 | |
| ] | |
| }, | |
| "tree": { | |
| "median_ms": 471.8639579950832, | |
| "min_ms": 470.3940409963252, | |
| "p90_ms": 473.91804099606816, | |
| "samples_ms": [ | |
| 473.1918329925975, | |
| 470.3940409963252, | |
| 473.91804099606816, | |
| 471.0846249945462, | |
| 471.2021250015823, | |
| 472.3992079962045, | |
| 472.50158300448675, | |
| 471.32870799396187 | |
| ] | |
| } | |
| }, | |
| "memory": { | |
| "batch": { | |
| "driver_high_water_mib": 5292.015625, | |
| "extra_driver_mib": 2062.0 | |
| }, | |
| "tree": { | |
| "driver_high_water_mib": 3276.015625, | |
| "extra_driver_mib": 48.0 | |
| } | |
| }, | |
| "decode_speedup": 2.372334529291179, | |
| "end_to_end_speedup": 1.408254014188503, | |
| "decode_speedup_ci95": [ | |
| 2.366582759337612, | |
| 2.376035750477272 | |
| ], | |
| "end_to_end_speedup_ci95": [ | |
| 1.4043364565201133, | |
| 1.4109481535389878 | |
| ] | |
| } | |
| ] | |
| } | |