xDecision / evaluation /native_mlx.json
mccoysc's picture
Add native MLX inference with calibrated output parity
d97731b verified
Raw History Blame Contribute Delete
27.1 kB
{
"model": "xDecision",
"processor": "Apple M5",
"memory_bytes": 25769803776,
"machine": "macOS-26.6.2-arm64-arm-64bit",
"torch": "2.14.0",
"threads": 4,
"iterations": 30,
"warmup": 5,
"mlx_enable_tf32": "0",
"input_sha256": "5475c43729d48eb4ef55a952a571dd40f0206890a49ef5b0f51f0bda3f88313e",
"timing": "Sequential backends; synchronized end-to-end predict, excluding model load. No output cache.",
"limits": "One machine/session; agreement is not accuracy. Load and first request are not fresh-process cold starts. Process RSS is cumulative across backends; MLX allocation is separate.",
"results": {
"torch-cpu": {
"load_s": 1.8336434159427881,
"first_request_ms": 65.17608417198062,
"parity": {
"questions": 42,
"argmax_agreement": 42,
"max_probability_error": 0.0,
"max_act_probability_error": 0.0,
"max_score_error": 0.0,
"max_confidence_error": 0.0
},
"timings": {
"short-choice": {
"p50_ms": 25.009167031385005,
"p95_ms": 26.35008773067966,
"samples_ms": [
25.580249959602952,
24.904084159061313,
25.626499904319644,
25.12637502513826,
25.341583183035254,
24.616667069494724,
24.470083182677627,
23.965499829500914,
24.464874994009733,
23.81699997931719,
25.715457974001765,
23.56974990107119,
26.23041719198227,
24.62316700257361,
25.330750038847327,
23.640000028535724,
24.533582851290703,
23.842666065320373,
23.983041988685727,
28.93995796330273,
25.99345799535513,
25.292209116742015,
25.987499859184027,
25.286666117608547,
25.114249903708696,
24.546624859794974,
26.44799998961389,
25.22704191505909,
24.410415906459093,
24.01049993932247
]
},
"short-three-types": {
"p50_ms": 59.266437543556094,
"p95_ms": 61.17129607591778,
"samples_ms": [
58.44441708177328,
59.365374967455864,
59.219166869297624,
57.941083097830415,
57.72687494754791,
57.723040925338864,
58.920334093272686,
58.38920804671943,
58.05695801973343,
58.266458101570606,
58.117832988500595,
58.81129205226898,
60.072166845202446,
59.27404202520847,
60.32779207453132,
57.795916916802526,
61.952083138749,
60.658416943624616,
59.784374898299575,
58.85491706430912,
59.515458066016436,
58.75099985860288,
59.258833061903715,
60.75645820237696,
59.79149998165667,
60.46687485650182,
60.36554090678692,
59.47137484326959,
61.51070888154209,
59.39895799383521
]
},
"long-three-types": {
"p50_ms": 1033.5651044733822,
"p95_ms": 1116.2748997681774,
"samples_ms": [
1128.8698751013726,
1024.5591660495847,
1014.6183748729527,
1012.2548341751099,
1023.1867081020027,
1024.2838331032544,
1019.5401250384748,
1014.0557079575956,
1027.6388749480247,
1023.492667125538,
1041.8245408218354,
1042.373958043754,
1064.232167089358,
1050.7710000965744,
1054.0782499592751,
1100.8810410276055,
1031.363375019282,
1036.6499170195311,
1039.8580830078572,
1027.7309999801219,
1052.3458749521524,
1034.1426669619977,
1032.9875419847667,
1031.6930410917848,
1030.512792058289,
1026.6135421115905,
1074.8492919374257,
1056.9510420318693,
1081.474999897182,
1160.2122080512345
]
}
},
"peak_rss_bytes": 2708946944,
"mlx_peak_bytes": null
},
"torch-mps": {
"load_s": 0.1871800001244992,
"first_request_ms": 125.95162494108081,
"parity": {
"questions": 42,
"argmax_agreement": 42,
"max_probability_error": 0.0,
"max_act_probability_error": 0.0,
"max_score_error": 0.0,
"max_confidence_error": 0.0
},
"timings": {
"short-choice": {
"p50_ms": 14.111166470684111,
"p95_ms": 15.085989504586903,
"samples_ms": [
14.610416954383254,
13.652416877448559,
13.500791043043137,
14.008042169734836,
13.934707967564464,
14.189000008627772,
12.798999901860952,
14.285790966823697,
15.36308298818767,
14.334207866340876,
14.643166912719607,
13.611707836389542,
14.086582930758595,
14.164666179567575,
12.87633297033608,
14.085000148043036,
14.7192080039531,
13.887250097468495,
13.825541129335761,
12.815791182219982,
14.562875032424927,
13.600333128124475,
14.791625086218119,
14.135750010609627,
14.679709216579795,
14.545290963724256,
12.948499992489815,
13.461249880492687,
14.168542111292481,
15.326833119615912
]
},
"short-three-types": {
"p50_ms": 28.994853957556188,
"p95_ms": 32.37735977163538,
"samples_ms": [
27.80325012281537,
28.71349989436567,
28.994916938245296,
28.362750075757504,
29.32179206982255,
28.771250043064356,
29.572124825790524,
29.29999982006848,
29.796124901622534,
28.99479097686708,
30.674332985654473,
32.34566608443856,
33.36429107002914,
32.403290970250964,
32.118957955390215,
30.328291933983564,
29.698500176891685,
28.179042041301727,
29.035374987870455,
28.080582851544023,
28.442249866202474,
28.57420896179974,
27.672333177179098,
28.317665914073586,
29.480499913915992,
28.979500057175756,
27.944750152528286,
28.508624993264675,
29.175875009968877,
28.334291884675622
]
},
"long-three-types": {
"p50_ms": 424.481229390949,
"p95_ms": 447.1228920505382,
"samples_ms": [
416.01675003767014,
412.92400006204844,
417.44862496852875,
416.14104178734124,
416.77662497386336,
420.62212503515184,
416.1499999463558,
422.2178750205785,
417.6560000050813,
421.9182909000665,
421.87454109080136,
419.34454208239913,
424.60116697475314,
425.74237496592104,
444.5189170073718,
451.4110831078142,
449.2534170858562,
438.7376250233501,
437.65700003132224,
441.4112498052418,
442.23374989815056,
435.749749885872,
432.55587480962276,
427.5060829240829,
430.12141599319875,
429.78799995034933,
429.1073330678046,
419.95962522923946,
424.3612918071449,
421.29729106090963
]
}
},
"peak_rss_bytes": 4395679744,
"mlx_peak_bytes": null
},
"mlx-f32": {
"load_s": 0.8362782921176404,
"first_request_ms": 62.71808291785419,
"parity": {
"questions": 42,
"argmax_agreement": 42,
"max_probability_error": 0.0,
"max_act_probability_error": 0.0,
"max_score_error": 0.0,
"max_confidence_error": 0.0
},
"timings": {
"short-choice": {
"p50_ms": 8.646207978017628,
"p95_ms": 9.25481078447774,
"samples_ms": [
9.07829194329679,
8.18304205313325,
9.08291689120233,
8.201583987101912,
8.889124961569905,
8.197375107556581,
8.84479214437306,
8.150875102728605,
8.756584022194147,
8.023166097700596,
8.873207960277796,
8.028666023164988,
8.933624951168895,
8.012040983885527,
8.973958902060986,
8.308249991387129,
9.084292221814394,
8.267541881650686,
9.461707901209593,
8.186208084225655,
8.84658400900662,
8.227124810218811,
8.559208130463958,
8.233916945755482,
8.653250057250261,
8.205916965380311,
8.639165898784995,
9.271292015910149,
8.67229187861085,
9.234667057171464
]
},
"short-three-types": {
"p50_ms": 27.439979137852788,
"p95_ms": 29.60230848984793,
"samples_ms": [
25.90083284303546,
26.88808413222432,
25.741792051121593,
27.01454097405076,
26.13366604782641,
26.60712506622076,
27.23487513139844,
26.556584052741528,
26.912541827186942,
26.535333832725883,
26.153957936912775,
27.079958003014326,
26.55725018121302,
28.237124904990196,
27.948707807809114,
28.098125010728836,
30.112708918750286,
29.620083048939705,
28.33245904184878,
28.19887502118945,
27.645083144307137,
26.446667034178972,
26.561542181298137,
28.03670894354582,
28.779208194464445,
28.47195789217949,
28.503790963441133,
29.580584028735757,
29.099459061399102,
28.65987503901124
]
},
"long-three-types": {
"p50_ms": 408.2140205428004,
"p95_ms": 420.70403959369287,
"samples_ms": [
395.5369577743113,
399.7525000013411,
402.282000053674,
413.9239580836147,
422.1989579964429,
418.8769171014428,
416.1644580308348,
436.78858294151723,
408.2535409834236,
408.4692499600351,
403.07912486605346,
406.7489579319954,
405.28624993748963,
407.0567497983575,
409.4870421104133,
407.96366706490517,
408.0827080179006,
405.73779097758234,
407.35787502489984,
407.85816707648337,
404.0020841639489,
408.17450010217726,
409.347582841292,
409.3512080144137,
409.3461660668254,
405.57579114101827,
410.8832080382854,
410.66658310592175,
413.0809591151774,
411.7653341963887
]
}
},
"peak_rss_bytes": 4395679744,
"mlx_peak_bytes": 2843246698
},
"mlx-f16": {
"load_s": 0.43084120796993375,
"first_request_ms": 19.334916956722736,
"parity": {
"questions": 42,
"argmax_agreement": 42,
"max_probability_error": 0.0030000000000000027,
"max_act_probability_error": 0.0020999999999999908,
"max_score_error": 0.0012000000000000899,
"max_confidence_error": 0.0030000000000000027
},
"timings": {
"short-choice": {
"p50_ms": 5.208916496485472,
"p95_ms": 6.072402570862323,
"samples_ms": [
5.075416993349791,
5.137792089954019,
6.072834134101868,
5.10474992915988,
6.321166874840856,
4.897790960967541,
5.235000047832727,
5.09270909242332,
5.874207941815257,
5.320167168974876,
5.103500094264746,
6.06404198333621,
5.257207900285721,
4.987166961655021,
5.835875170305371,
5.248166853561997,
4.975625080987811,
5.796832963824272,
5.181041080504656,
4.941333085298538,
5.786125082522631,
5.407500080764294,
5.114791914820671,
5.8659580536186695,
5.196749931201339,
4.893959034234285,
6.071875104680657,
5.221083061769605,
4.907374968752265,
5.124625051394105
]
},
"short-three-types": {
"p50_ms": 11.030271067284048,
"p95_ms": 12.32145627727732,
"samples_ms": [
9.481834014877677,
11.024374980479479,
9.665166959166527,
10.315499966964126,
10.323791997507215,
10.239499853923917,
10.601666057482362,
9.750708937644958,
11.057124938815832,
11.941166827455163,
10.765250073745847,
12.366999872028828,
12.512624962255359,
12.177583994343877,
11.05537498369813,
12.265791883692145,
12.0297078974545,
10.916166938841343,
11.379000032320619,
11.192667065188289,
10.09087497368455,
11.284457985311747,
11.039916891604662,
10.075125144794583,
11.041667079553008,
10.825832840055227,
10.167042026296258,
11.036167154088616,
11.068291962146759,
10.08412498049438
]
},
"long-three-types": {
"p50_ms": 125.08991651702672,
"p95_ms": 127.91901449672878,
"samples_ms": [
121.85625010170043,
122.14362481608987,
123.74708289280534,
123.17941710352898,
122.63866607099771,
124.6428748127073,
124.61724993772805,
123.61424998380244,
124.05583285726607,
125.16270810738206,
125.93666696920991,
127.71466700360179,
126.8857920076698,
123.60641593113542,
123.7822079565376,
124.26758301444352,
124.24008292146027,
125.06404099985957,
124.63741702958941,
127.16299993917346,
125.79516717232764,
125.11579203419387,
125.93049998395145,
125.45570801012218,
126.10699981451035,
125.36016595549881,
126.64404208771884,
127.44825007393956,
130.26875001378357,
128.0862079001963
]
}
},
"peak_rss_bytes": 4395679744,
"mlx_peak_bytes": 1421638915
},
"mlx-gguf-f16": {
"load_s": 14.322720540920272,
"first_request_ms": 26.145249838009477,
"parity": {
"questions": 42,
"argmax_agreement": 42,
"max_probability_error": 0.0030000000000000027,
"max_act_probability_error": 0.0020999999999999908,
"max_score_error": 0.0012000000000000899,
"max_confidence_error": 0.0030000000000000027
},
"timings": {
"short-choice": {
"p50_ms": 5.142416339367628,
"p95_ms": 5.902735039126128,
"samples_ms": [
5.306666949763894,
5.1711671985685825,
5.105291958898306,
5.774541990831494,
5.194957833737135,
5.091875093057752,
5.86091703735292,
5.198708036914468,
4.931250121444464,
5.810125032439828,
5.1814590115100145,
5.103625124320388,
5.80691616050899,
5.119749810546637,
4.867082927376032,
5.9305408503860235,
5.064791068434715,
4.862291039898992,
4.877207800745964,
6.1593749560415745,
4.819917026907206,
4.881040891632438,
5.868750158697367,
5.036500049754977,
4.845791961997747,
5.753041943535209,
5.16508286818862,
4.9448751378804445,
5.8095408603549,
5.098082823678851
]
},
"short-three-types": {
"p50_ms": 13.163916999474168,
"p95_ms": 20.25712855393067,
"samples_ms": [
10.799874784424901,
9.871625108644366,
14.080458087846637,
15.84208314307034,
20.250207977369428,
24.500958854332566,
20.063458010554314,
20.262790843844414,
18.66349996998906,
16.93916693329811,
16.21062494814396,
15.926875174045563,
15.474790940061212,
15.150333987548947,
14.243375044316053,
14.155917102470994,
13.539416948333383,
11.718417052179575,
12.788417050614953,
12.328125070780516,
10.87887492030859,
10.970166884362698,
11.961332987993956,
10.971874929964542,
10.524041950702667,
11.234583100304008,
10.418124962598085,
10.542208096012473,
10.890667093917727,
9.88187501206994
]
},
"long-three-types": {
"p50_ms": 124.07070794142783,
"p95_ms": 126.72857847064734,
"samples_ms": [
126.40779092907906,
125.8652089163661,
120.70324993692338,
125.4580831155181,
122.17758386395872,
125.37262495607138,
124.23625006340444,
124.41674992442131,
125.48016617074609,
120.07174990139902,
122.76983400806785,
124.17341582477093,
123.15416592173278,
126.99104100465775,
125.74133416637778,
120.3838330693543,
123.37704095989466,
122.06491688266397,
121.12629204057157,
125.58470806106925,
123.96800005808473,
123.88745788484812,
123.78191691823304,
124.87612501718104,
127.77420901693404,
126.26737495884299,
119.91520808078349,
121.14250008016825,
123.31841606646776,
125.73825009167194
]
}
},
"peak_rss_bytes": 6111084544,
"mlx_peak_bytes": 1421638915
},
"torch-gguf-q8": {
"load_s": 16.790536124957725,
"first_request_ms": 65.82020805217326,
"parity": {
"questions": 42,
"argmax_agreement": 42,
"max_probability_error": 0.015200000000000102,
"max_act_probability_error": 0.07369999999999999,
"max_score_error": 0.00550000000000006,
"max_confidence_error": 0.015199999999999991
},
"timings": {
"short-choice": {
"p50_ms": 26.122875045984983,
"p95_ms": 27.80325028579682,
"samples_ms": [
26.299958117306232,
26.505500078201294,
26.535833021625876,
26.517125079408288,
26.899375021457672,
27.410291135311127,
26.70149994082749,
26.932957815006375,
25.5129998549819,
26.386334095150232,
25.44220886193216,
26.027417043223977,
27.98812510445714,
25.280416943132877,
25.31812503002584,
24.913457920774817,
25.119666941463947,
24.88604187965393,
25.22508311085403,
24.90699989721179,
27.577292174100876,
26.21833304874599,
25.71058296598494,
25.68858303129673,
25.570583064109087,
25.84200003184378,
28.020832920446992,
27.051375014707446,
26.395124848932028,
25.549958925694227
]
},
"short-three-types": {
"p50_ms": 62.49914539512247,
"p95_ms": 125.7209476432763,
"samples_ms": [
60.92745787464082,
63.102582935243845,
62.82237498089671,
62.11025011725724,
63.319875160232186,
60.849040979519486,
61.882499838247895,
61.086750123649836,
61.313292011618614,
60.750583885237575,
60.62925001606345,
59.70504181459546,
61.28466711379588,
61.57308304682374,
62.57187481969595,
63.209500163793564,
60.82345801405609,
62.42641597054899,
61.910832999274135,
62.38312507048249,
89.54724995419383,
75.72958385571837,
134.2952500563115,
100.7456248626113,
100.9271249640733,
125.31062518246472,
96.37708310037851,
126.05666602030396,
94.56258313730359,
86.44129196181893
]
},
"long-three-types": {
"p50_ms": 1266.846957965754,
"p95_ms": 1403.54498779634,
"samples_ms": [
1213.3727078326046,
1228.3932918217033,
1249.0241250488907,
1225.9758750442415,
1237.6357920002192,
1254.133875016123,
1265.902667073533,
1230.628749821335,
1267.2083750367165,
1424.849750008434,
1289.3726250622422,
1251.7832079902291,
1246.0905001498759,
1266.4855408947915,
1262.3959579505026,
1337.7245420124382,
1272.63587503694,
1437.6328748185188,
1348.5361670609564,
1270.988459000364,
1268.2836661115289,
1268.9630831591785,
1273.3692498877645,
1377.5058339815587,
1309.6742080524564,
1262.4727908987552,
1260.940541047603,
1272.8466670960188,
1275.9640000294894,
1265.5512921046466
]
}
},
"peak_rss_bytes": 6148816896,
"mlx_peak_bytes": null
},
"mlx-gguf-q8": {
"load_s": 14.982421041931957,
"first_request_ms": 35.079791909083724,
"parity": {
"questions": 42,
"argmax_agreement": 42,
"max_probability_error": 0.0030000000000000027,
"max_act_probability_error": 0.0020999999999999908,
"max_score_error": 0.0007999999999999119,
"max_confidence_error": 0.0030000000000000027
},
"timings": {
"short-choice": {
"p50_ms": 6.608979078009725,
"p95_ms": 7.761526969261466,
"samples_ms": [
6.6324579529464245,
6.184333935379982,
7.972707971930504,
6.234582979232073,
7.354500005021691,
6.724750157445669,
7.503416854888201,
6.9359170738607645,
6.3974999357014894,
8.057290920987725,
6.2525831162929535,
7.459083804860711,
6.713459035381675,
6.4778749365359545,
7.396250031888485,
6.585500203073025,
7.316000061109662,
6.5283330623060465,
7.202249951660633,
6.45708292722702,
6.033665966242552,
7.2585418820381165,
6.132833892479539,
7.311125053092837,
6.536291912198067,
6.075833924114704,
7.21258413977921,
5.9309168718755245,
5.516042001545429,
6.445999955758452
]
},
"short-three-types": {
"p50_ms": 10.712645831517875,
"p95_ms": 12.019854364916682,
"samples_ms": [
10.53762505762279,
9.504124987870455,
10.739124845713377,
9.548000060021877,
10.872208047658205,
10.638416977599263,
9.546666871756315,
11.020832927897573,
9.458292042836547,
10.091457981616259,
10.686166817322373,
9.469332871958613,
10.616583051159978,
9.507999988272786,
9.960833005607128,
10.749541921541095,
10.348625015467405,
11.054417118430138,
10.762958088889718,
10.18533413298428,
10.920584201812744,
11.341291014105082,
9.913707850500941,
11.748791905120015,
12.019166955724359,
10.928499978035688,
12.020416790619493,
11.9377919472754,
11.339165968820453,
12.478082906454802
]
},
"long-three-types": {
"p50_ms": 186.60743755754083,
"p95_ms": 258.3231775206513,
"samples_ms": [
318.01679194904864,
265.0479590520263,
243.69887495413423,
250.10400009341538,
238.5930421296507,
219.68429116532207,
202.91625009849668,
186.29233306273818,
159.23545905388892,
165.35883396863937,
162.2020830400288,
162.32783300802112,
160.531209083274,
172.42029099725187,
169.07654097303748,
168.83791610598564,
182.5247078668326,
177.2719998843968,
177.06720903515816,
185.59987493790686,
184.45079098455608,
179.83149993233383,
192.80395796522498,
186.9225420523435,
189.82570921070874,
197.10833299905062,
192.07008415833116,
191.10641698352993,
190.38137514144182,
187.29425012134016
]
}
},
"peak_rss_bytes": 6148816896,
"mlx_peak_bytes": 1421638915
}
},
"mlx": "0.32.2"
}