Download evaluation/native_mlx.json from mccoysc/xDecision: direct link, hf CLI and curl.
- Browser
- Download file 27.1 kB
-
https://huggingface.co/mccoysc/xDecision/resolve/main/evaluation/native_mlx.json
- Command line
-
hf download hf://mccoysc/xDecision/evaluation/native_mlx.json
-
curl -L -o native_mlx.json https://huggingface.co/mccoysc/xDecision/resolve/main/evaluation/native_mlx.json
27.1 kB
| { | |
| "model": "xDecision", | |
| "processor": "Apple M5", | |
| "memory_bytes": 25769803776, | |
| "machine": "macOS-26.6.2-arm64-arm-64bit", | |
| "torch": "2.14.0", | |
| "threads": 4, | |
| "iterations": 30, | |
| "warmup": 5, | |
| "mlx_enable_tf32": "0", | |
| "input_sha256": "5475c43729d48eb4ef55a952a571dd40f0206890a49ef5b0f51f0bda3f88313e", | |
| "timing": "Sequential backends; synchronized end-to-end predict, excluding model load. No output cache.", | |
| "limits": "One machine/session; agreement is not accuracy. Load and first request are not fresh-process cold starts. Process RSS is cumulative across backends; MLX allocation is separate.", | |
| "results": { | |
| "torch-cpu": { | |
| "load_s": 1.8336434159427881, | |
| "first_request_ms": 65.17608417198062, | |
| "parity": { | |
| "questions": 42, | |
| "argmax_agreement": 42, | |
| "max_probability_error": 0.0, | |
| "max_act_probability_error": 0.0, | |
| "max_score_error": 0.0, | |
| "max_confidence_error": 0.0 | |
| }, | |
| "timings": { | |
| "short-choice": { | |
| "p50_ms": 25.009167031385005, | |
| "p95_ms": 26.35008773067966, | |
| "samples_ms": [ | |
| 25.580249959602952, | |
| 24.904084159061313, | |
| 25.626499904319644, | |
| 25.12637502513826, | |
| 25.341583183035254, | |
| 24.616667069494724, | |
| 24.470083182677627, | |
| 23.965499829500914, | |
| 24.464874994009733, | |
| 23.81699997931719, | |
| 25.715457974001765, | |
| 23.56974990107119, | |
| 26.23041719198227, | |
| 24.62316700257361, | |
| 25.330750038847327, | |
| 23.640000028535724, | |
| 24.533582851290703, | |
| 23.842666065320373, | |
| 23.983041988685727, | |
| 28.93995796330273, | |
| 25.99345799535513, | |
| 25.292209116742015, | |
| 25.987499859184027, | |
| 25.286666117608547, | |
| 25.114249903708696, | |
| 24.546624859794974, | |
| 26.44799998961389, | |
| 25.22704191505909, | |
| 24.410415906459093, | |
| 24.01049993932247 | |
| ] | |
| }, | |
| "short-three-types": { | |
| "p50_ms": 59.266437543556094, | |
| "p95_ms": 61.17129607591778, | |
| "samples_ms": [ | |
| 58.44441708177328, | |
| 59.365374967455864, | |
| 59.219166869297624, | |
| 57.941083097830415, | |
| 57.72687494754791, | |
| 57.723040925338864, | |
| 58.920334093272686, | |
| 58.38920804671943, | |
| 58.05695801973343, | |
| 58.266458101570606, | |
| 58.117832988500595, | |
| 58.81129205226898, | |
| 60.072166845202446, | |
| 59.27404202520847, | |
| 60.32779207453132, | |
| 57.795916916802526, | |
| 61.952083138749, | |
| 60.658416943624616, | |
| 59.784374898299575, | |
| 58.85491706430912, | |
| 59.515458066016436, | |
| 58.75099985860288, | |
| 59.258833061903715, | |
| 60.75645820237696, | |
| 59.79149998165667, | |
| 60.46687485650182, | |
| 60.36554090678692, | |
| 59.47137484326959, | |
| 61.51070888154209, | |
| 59.39895799383521 | |
| ] | |
| }, | |
| "long-three-types": { | |
| "p50_ms": 1033.5651044733822, | |
| "p95_ms": 1116.2748997681774, | |
| "samples_ms": [ | |
| 1128.8698751013726, | |
| 1024.5591660495847, | |
| 1014.6183748729527, | |
| 1012.2548341751099, | |
| 1023.1867081020027, | |
| 1024.2838331032544, | |
| 1019.5401250384748, | |
| 1014.0557079575956, | |
| 1027.6388749480247, | |
| 1023.492667125538, | |
| 1041.8245408218354, | |
| 1042.373958043754, | |
| 1064.232167089358, | |
| 1050.7710000965744, | |
| 1054.0782499592751, | |
| 1100.8810410276055, | |
| 1031.363375019282, | |
| 1036.6499170195311, | |
| 1039.8580830078572, | |
| 1027.7309999801219, | |
| 1052.3458749521524, | |
| 1034.1426669619977, | |
| 1032.9875419847667, | |
| 1031.6930410917848, | |
| 1030.512792058289, | |
| 1026.6135421115905, | |
| 1074.8492919374257, | |
| 1056.9510420318693, | |
| 1081.474999897182, | |
| 1160.2122080512345 | |
| ] | |
| } | |
| }, | |
| "peak_rss_bytes": 2708946944, | |
| "mlx_peak_bytes": null | |
| }, | |
| "torch-mps": { | |
| "load_s": 0.1871800001244992, | |
| "first_request_ms": 125.95162494108081, | |
| "parity": { | |
| "questions": 42, | |
| "argmax_agreement": 42, | |
| "max_probability_error": 0.0, | |
| "max_act_probability_error": 0.0, | |
| "max_score_error": 0.0, | |
| "max_confidence_error": 0.0 | |
| }, | |
| "timings": { | |
| "short-choice": { | |
| "p50_ms": 14.111166470684111, | |
| "p95_ms": 15.085989504586903, | |
| "samples_ms": [ | |
| 14.610416954383254, | |
| 13.652416877448559, | |
| 13.500791043043137, | |
| 14.008042169734836, | |
| 13.934707967564464, | |
| 14.189000008627772, | |
| 12.798999901860952, | |
| 14.285790966823697, | |
| 15.36308298818767, | |
| 14.334207866340876, | |
| 14.643166912719607, | |
| 13.611707836389542, | |
| 14.086582930758595, | |
| 14.164666179567575, | |
| 12.87633297033608, | |
| 14.085000148043036, | |
| 14.7192080039531, | |
| 13.887250097468495, | |
| 13.825541129335761, | |
| 12.815791182219982, | |
| 14.562875032424927, | |
| 13.600333128124475, | |
| 14.791625086218119, | |
| 14.135750010609627, | |
| 14.679709216579795, | |
| 14.545290963724256, | |
| 12.948499992489815, | |
| 13.461249880492687, | |
| 14.168542111292481, | |
| 15.326833119615912 | |
| ] | |
| }, | |
| "short-three-types": { | |
| "p50_ms": 28.994853957556188, | |
| "p95_ms": 32.37735977163538, | |
| "samples_ms": [ | |
| 27.80325012281537, | |
| 28.71349989436567, | |
| 28.994916938245296, | |
| 28.362750075757504, | |
| 29.32179206982255, | |
| 28.771250043064356, | |
| 29.572124825790524, | |
| 29.29999982006848, | |
| 29.796124901622534, | |
| 28.99479097686708, | |
| 30.674332985654473, | |
| 32.34566608443856, | |
| 33.36429107002914, | |
| 32.403290970250964, | |
| 32.118957955390215, | |
| 30.328291933983564, | |
| 29.698500176891685, | |
| 28.179042041301727, | |
| 29.035374987870455, | |
| 28.080582851544023, | |
| 28.442249866202474, | |
| 28.57420896179974, | |
| 27.672333177179098, | |
| 28.317665914073586, | |
| 29.480499913915992, | |
| 28.979500057175756, | |
| 27.944750152528286, | |
| 28.508624993264675, | |
| 29.175875009968877, | |
| 28.334291884675622 | |
| ] | |
| }, | |
| "long-three-types": { | |
| "p50_ms": 424.481229390949, | |
| "p95_ms": 447.1228920505382, | |
| "samples_ms": [ | |
| 416.01675003767014, | |
| 412.92400006204844, | |
| 417.44862496852875, | |
| 416.14104178734124, | |
| 416.77662497386336, | |
| 420.62212503515184, | |
| 416.1499999463558, | |
| 422.2178750205785, | |
| 417.6560000050813, | |
| 421.9182909000665, | |
| 421.87454109080136, | |
| 419.34454208239913, | |
| 424.60116697475314, | |
| 425.74237496592104, | |
| 444.5189170073718, | |
| 451.4110831078142, | |
| 449.2534170858562, | |
| 438.7376250233501, | |
| 437.65700003132224, | |
| 441.4112498052418, | |
| 442.23374989815056, | |
| 435.749749885872, | |
| 432.55587480962276, | |
| 427.5060829240829, | |
| 430.12141599319875, | |
| 429.78799995034933, | |
| 429.1073330678046, | |
| 419.95962522923946, | |
| 424.3612918071449, | |
| 421.29729106090963 | |
| ] | |
| } | |
| }, | |
| "peak_rss_bytes": 4395679744, | |
| "mlx_peak_bytes": null | |
| }, | |
| "mlx-f32": { | |
| "load_s": 0.8362782921176404, | |
| "first_request_ms": 62.71808291785419, | |
| "parity": { | |
| "questions": 42, | |
| "argmax_agreement": 42, | |
| "max_probability_error": 0.0, | |
| "max_act_probability_error": 0.0, | |
| "max_score_error": 0.0, | |
| "max_confidence_error": 0.0 | |
| }, | |
| "timings": { | |
| "short-choice": { | |
| "p50_ms": 8.646207978017628, | |
| "p95_ms": 9.25481078447774, | |
| "samples_ms": [ | |
| 9.07829194329679, | |
| 8.18304205313325, | |
| 9.08291689120233, | |
| 8.201583987101912, | |
| 8.889124961569905, | |
| 8.197375107556581, | |
| 8.84479214437306, | |
| 8.150875102728605, | |
| 8.756584022194147, | |
| 8.023166097700596, | |
| 8.873207960277796, | |
| 8.028666023164988, | |
| 8.933624951168895, | |
| 8.012040983885527, | |
| 8.973958902060986, | |
| 8.308249991387129, | |
| 9.084292221814394, | |
| 8.267541881650686, | |
| 9.461707901209593, | |
| 8.186208084225655, | |
| 8.84658400900662, | |
| 8.227124810218811, | |
| 8.559208130463958, | |
| 8.233916945755482, | |
| 8.653250057250261, | |
| 8.205916965380311, | |
| 8.639165898784995, | |
| 9.271292015910149, | |
| 8.67229187861085, | |
| 9.234667057171464 | |
| ] | |
| }, | |
| "short-three-types": { | |
| "p50_ms": 27.439979137852788, | |
| "p95_ms": 29.60230848984793, | |
| "samples_ms": [ | |
| 25.90083284303546, | |
| 26.88808413222432, | |
| 25.741792051121593, | |
| 27.01454097405076, | |
| 26.13366604782641, | |
| 26.60712506622076, | |
| 27.23487513139844, | |
| 26.556584052741528, | |
| 26.912541827186942, | |
| 26.535333832725883, | |
| 26.153957936912775, | |
| 27.079958003014326, | |
| 26.55725018121302, | |
| 28.237124904990196, | |
| 27.948707807809114, | |
| 28.098125010728836, | |
| 30.112708918750286, | |
| 29.620083048939705, | |
| 28.33245904184878, | |
| 28.19887502118945, | |
| 27.645083144307137, | |
| 26.446667034178972, | |
| 26.561542181298137, | |
| 28.03670894354582, | |
| 28.779208194464445, | |
| 28.47195789217949, | |
| 28.503790963441133, | |
| 29.580584028735757, | |
| 29.099459061399102, | |
| 28.65987503901124 | |
| ] | |
| }, | |
| "long-three-types": { | |
| "p50_ms": 408.2140205428004, | |
| "p95_ms": 420.70403959369287, | |
| "samples_ms": [ | |
| 395.5369577743113, | |
| 399.7525000013411, | |
| 402.282000053674, | |
| 413.9239580836147, | |
| 422.1989579964429, | |
| 418.8769171014428, | |
| 416.1644580308348, | |
| 436.78858294151723, | |
| 408.2535409834236, | |
| 408.4692499600351, | |
| 403.07912486605346, | |
| 406.7489579319954, | |
| 405.28624993748963, | |
| 407.0567497983575, | |
| 409.4870421104133, | |
| 407.96366706490517, | |
| 408.0827080179006, | |
| 405.73779097758234, | |
| 407.35787502489984, | |
| 407.85816707648337, | |
| 404.0020841639489, | |
| 408.17450010217726, | |
| 409.347582841292, | |
| 409.3512080144137, | |
| 409.3461660668254, | |
| 405.57579114101827, | |
| 410.8832080382854, | |
| 410.66658310592175, | |
| 413.0809591151774, | |
| 411.7653341963887 | |
| ] | |
| } | |
| }, | |
| "peak_rss_bytes": 4395679744, | |
| "mlx_peak_bytes": 2843246698 | |
| }, | |
| "mlx-f16": { | |
| "load_s": 0.43084120796993375, | |
| "first_request_ms": 19.334916956722736, | |
| "parity": { | |
| "questions": 42, | |
| "argmax_agreement": 42, | |
| "max_probability_error": 0.0030000000000000027, | |
| "max_act_probability_error": 0.0020999999999999908, | |
| "max_score_error": 0.0012000000000000899, | |
| "max_confidence_error": 0.0030000000000000027 | |
| }, | |
| "timings": { | |
| "short-choice": { | |
| "p50_ms": 5.208916496485472, | |
| "p95_ms": 6.072402570862323, | |
| "samples_ms": [ | |
| 5.075416993349791, | |
| 5.137792089954019, | |
| 6.072834134101868, | |
| 5.10474992915988, | |
| 6.321166874840856, | |
| 4.897790960967541, | |
| 5.235000047832727, | |
| 5.09270909242332, | |
| 5.874207941815257, | |
| 5.320167168974876, | |
| 5.103500094264746, | |
| 6.06404198333621, | |
| 5.257207900285721, | |
| 4.987166961655021, | |
| 5.835875170305371, | |
| 5.248166853561997, | |
| 4.975625080987811, | |
| 5.796832963824272, | |
| 5.181041080504656, | |
| 4.941333085298538, | |
| 5.786125082522631, | |
| 5.407500080764294, | |
| 5.114791914820671, | |
| 5.8659580536186695, | |
| 5.196749931201339, | |
| 4.893959034234285, | |
| 6.071875104680657, | |
| 5.221083061769605, | |
| 4.907374968752265, | |
| 5.124625051394105 | |
| ] | |
| }, | |
| "short-three-types": { | |
| "p50_ms": 11.030271067284048, | |
| "p95_ms": 12.32145627727732, | |
| "samples_ms": [ | |
| 9.481834014877677, | |
| 11.024374980479479, | |
| 9.665166959166527, | |
| 10.315499966964126, | |
| 10.323791997507215, | |
| 10.239499853923917, | |
| 10.601666057482362, | |
| 9.750708937644958, | |
| 11.057124938815832, | |
| 11.941166827455163, | |
| 10.765250073745847, | |
| 12.366999872028828, | |
| 12.512624962255359, | |
| 12.177583994343877, | |
| 11.05537498369813, | |
| 12.265791883692145, | |
| 12.0297078974545, | |
| 10.916166938841343, | |
| 11.379000032320619, | |
| 11.192667065188289, | |
| 10.09087497368455, | |
| 11.284457985311747, | |
| 11.039916891604662, | |
| 10.075125144794583, | |
| 11.041667079553008, | |
| 10.825832840055227, | |
| 10.167042026296258, | |
| 11.036167154088616, | |
| 11.068291962146759, | |
| 10.08412498049438 | |
| ] | |
| }, | |
| "long-three-types": { | |
| "p50_ms": 125.08991651702672, | |
| "p95_ms": 127.91901449672878, | |
| "samples_ms": [ | |
| 121.85625010170043, | |
| 122.14362481608987, | |
| 123.74708289280534, | |
| 123.17941710352898, | |
| 122.63866607099771, | |
| 124.6428748127073, | |
| 124.61724993772805, | |
| 123.61424998380244, | |
| 124.05583285726607, | |
| 125.16270810738206, | |
| 125.93666696920991, | |
| 127.71466700360179, | |
| 126.8857920076698, | |
| 123.60641593113542, | |
| 123.7822079565376, | |
| 124.26758301444352, | |
| 124.24008292146027, | |
| 125.06404099985957, | |
| 124.63741702958941, | |
| 127.16299993917346, | |
| 125.79516717232764, | |
| 125.11579203419387, | |
| 125.93049998395145, | |
| 125.45570801012218, | |
| 126.10699981451035, | |
| 125.36016595549881, | |
| 126.64404208771884, | |
| 127.44825007393956, | |
| 130.26875001378357, | |
| 128.0862079001963 | |
| ] | |
| } | |
| }, | |
| "peak_rss_bytes": 4395679744, | |
| "mlx_peak_bytes": 1421638915 | |
| }, | |
| "mlx-gguf-f16": { | |
| "load_s": 14.322720540920272, | |
| "first_request_ms": 26.145249838009477, | |
| "parity": { | |
| "questions": 42, | |
| "argmax_agreement": 42, | |
| "max_probability_error": 0.0030000000000000027, | |
| "max_act_probability_error": 0.0020999999999999908, | |
| "max_score_error": 0.0012000000000000899, | |
| "max_confidence_error": 0.0030000000000000027 | |
| }, | |
| "timings": { | |
| "short-choice": { | |
| "p50_ms": 5.142416339367628, | |
| "p95_ms": 5.902735039126128, | |
| "samples_ms": [ | |
| 5.306666949763894, | |
| 5.1711671985685825, | |
| 5.105291958898306, | |
| 5.774541990831494, | |
| 5.194957833737135, | |
| 5.091875093057752, | |
| 5.86091703735292, | |
| 5.198708036914468, | |
| 4.931250121444464, | |
| 5.810125032439828, | |
| 5.1814590115100145, | |
| 5.103625124320388, | |
| 5.80691616050899, | |
| 5.119749810546637, | |
| 4.867082927376032, | |
| 5.9305408503860235, | |
| 5.064791068434715, | |
| 4.862291039898992, | |
| 4.877207800745964, | |
| 6.1593749560415745, | |
| 4.819917026907206, | |
| 4.881040891632438, | |
| 5.868750158697367, | |
| 5.036500049754977, | |
| 4.845791961997747, | |
| 5.753041943535209, | |
| 5.16508286818862, | |
| 4.9448751378804445, | |
| 5.8095408603549, | |
| 5.098082823678851 | |
| ] | |
| }, | |
| "short-three-types": { | |
| "p50_ms": 13.163916999474168, | |
| "p95_ms": 20.25712855393067, | |
| "samples_ms": [ | |
| 10.799874784424901, | |
| 9.871625108644366, | |
| 14.080458087846637, | |
| 15.84208314307034, | |
| 20.250207977369428, | |
| 24.500958854332566, | |
| 20.063458010554314, | |
| 20.262790843844414, | |
| 18.66349996998906, | |
| 16.93916693329811, | |
| 16.21062494814396, | |
| 15.926875174045563, | |
| 15.474790940061212, | |
| 15.150333987548947, | |
| 14.243375044316053, | |
| 14.155917102470994, | |
| 13.539416948333383, | |
| 11.718417052179575, | |
| 12.788417050614953, | |
| 12.328125070780516, | |
| 10.87887492030859, | |
| 10.970166884362698, | |
| 11.961332987993956, | |
| 10.971874929964542, | |
| 10.524041950702667, | |
| 11.234583100304008, | |
| 10.418124962598085, | |
| 10.542208096012473, | |
| 10.890667093917727, | |
| 9.88187501206994 | |
| ] | |
| }, | |
| "long-three-types": { | |
| "p50_ms": 124.07070794142783, | |
| "p95_ms": 126.72857847064734, | |
| "samples_ms": [ | |
| 126.40779092907906, | |
| 125.8652089163661, | |
| 120.70324993692338, | |
| 125.4580831155181, | |
| 122.17758386395872, | |
| 125.37262495607138, | |
| 124.23625006340444, | |
| 124.41674992442131, | |
| 125.48016617074609, | |
| 120.07174990139902, | |
| 122.76983400806785, | |
| 124.17341582477093, | |
| 123.15416592173278, | |
| 126.99104100465775, | |
| 125.74133416637778, | |
| 120.3838330693543, | |
| 123.37704095989466, | |
| 122.06491688266397, | |
| 121.12629204057157, | |
| 125.58470806106925, | |
| 123.96800005808473, | |
| 123.88745788484812, | |
| 123.78191691823304, | |
| 124.87612501718104, | |
| 127.77420901693404, | |
| 126.26737495884299, | |
| 119.91520808078349, | |
| 121.14250008016825, | |
| 123.31841606646776, | |
| 125.73825009167194 | |
| ] | |
| } | |
| }, | |
| "peak_rss_bytes": 6111084544, | |
| "mlx_peak_bytes": 1421638915 | |
| }, | |
| "torch-gguf-q8": { | |
| "load_s": 16.790536124957725, | |
| "first_request_ms": 65.82020805217326, | |
| "parity": { | |
| "questions": 42, | |
| "argmax_agreement": 42, | |
| "max_probability_error": 0.015200000000000102, | |
| "max_act_probability_error": 0.07369999999999999, | |
| "max_score_error": 0.00550000000000006, | |
| "max_confidence_error": 0.015199999999999991 | |
| }, | |
| "timings": { | |
| "short-choice": { | |
| "p50_ms": 26.122875045984983, | |
| "p95_ms": 27.80325028579682, | |
| "samples_ms": [ | |
| 26.299958117306232, | |
| 26.505500078201294, | |
| 26.535833021625876, | |
| 26.517125079408288, | |
| 26.899375021457672, | |
| 27.410291135311127, | |
| 26.70149994082749, | |
| 26.932957815006375, | |
| 25.5129998549819, | |
| 26.386334095150232, | |
| 25.44220886193216, | |
| 26.027417043223977, | |
| 27.98812510445714, | |
| 25.280416943132877, | |
| 25.31812503002584, | |
| 24.913457920774817, | |
| 25.119666941463947, | |
| 24.88604187965393, | |
| 25.22508311085403, | |
| 24.90699989721179, | |
| 27.577292174100876, | |
| 26.21833304874599, | |
| 25.71058296598494, | |
| 25.68858303129673, | |
| 25.570583064109087, | |
| 25.84200003184378, | |
| 28.020832920446992, | |
| 27.051375014707446, | |
| 26.395124848932028, | |
| 25.549958925694227 | |
| ] | |
| }, | |
| "short-three-types": { | |
| "p50_ms": 62.49914539512247, | |
| "p95_ms": 125.7209476432763, | |
| "samples_ms": [ | |
| 60.92745787464082, | |
| 63.102582935243845, | |
| 62.82237498089671, | |
| 62.11025011725724, | |
| 63.319875160232186, | |
| 60.849040979519486, | |
| 61.882499838247895, | |
| 61.086750123649836, | |
| 61.313292011618614, | |
| 60.750583885237575, | |
| 60.62925001606345, | |
| 59.70504181459546, | |
| 61.28466711379588, | |
| 61.57308304682374, | |
| 62.57187481969595, | |
| 63.209500163793564, | |
| 60.82345801405609, | |
| 62.42641597054899, | |
| 61.910832999274135, | |
| 62.38312507048249, | |
| 89.54724995419383, | |
| 75.72958385571837, | |
| 134.2952500563115, | |
| 100.7456248626113, | |
| 100.9271249640733, | |
| 125.31062518246472, | |
| 96.37708310037851, | |
| 126.05666602030396, | |
| 94.56258313730359, | |
| 86.44129196181893 | |
| ] | |
| }, | |
| "long-three-types": { | |
| "p50_ms": 1266.846957965754, | |
| "p95_ms": 1403.54498779634, | |
| "samples_ms": [ | |
| 1213.3727078326046, | |
| 1228.3932918217033, | |
| 1249.0241250488907, | |
| 1225.9758750442415, | |
| 1237.6357920002192, | |
| 1254.133875016123, | |
| 1265.902667073533, | |
| 1230.628749821335, | |
| 1267.2083750367165, | |
| 1424.849750008434, | |
| 1289.3726250622422, | |
| 1251.7832079902291, | |
| 1246.0905001498759, | |
| 1266.4855408947915, | |
| 1262.3959579505026, | |
| 1337.7245420124382, | |
| 1272.63587503694, | |
| 1437.6328748185188, | |
| 1348.5361670609564, | |
| 1270.988459000364, | |
| 1268.2836661115289, | |
| 1268.9630831591785, | |
| 1273.3692498877645, | |
| 1377.5058339815587, | |
| 1309.6742080524564, | |
| 1262.4727908987552, | |
| 1260.940541047603, | |
| 1272.8466670960188, | |
| 1275.9640000294894, | |
| 1265.5512921046466 | |
| ] | |
| } | |
| }, | |
| "peak_rss_bytes": 6148816896, | |
| "mlx_peak_bytes": null | |
| }, | |
| "mlx-gguf-q8": { | |
| "load_s": 14.982421041931957, | |
| "first_request_ms": 35.079791909083724, | |
| "parity": { | |
| "questions": 42, | |
| "argmax_agreement": 42, | |
| "max_probability_error": 0.0030000000000000027, | |
| "max_act_probability_error": 0.0020999999999999908, | |
| "max_score_error": 0.0007999999999999119, | |
| "max_confidence_error": 0.0030000000000000027 | |
| }, | |
| "timings": { | |
| "short-choice": { | |
| "p50_ms": 6.608979078009725, | |
| "p95_ms": 7.761526969261466, | |
| "samples_ms": [ | |
| 6.6324579529464245, | |
| 6.184333935379982, | |
| 7.972707971930504, | |
| 6.234582979232073, | |
| 7.354500005021691, | |
| 6.724750157445669, | |
| 7.503416854888201, | |
| 6.9359170738607645, | |
| 6.3974999357014894, | |
| 8.057290920987725, | |
| 6.2525831162929535, | |
| 7.459083804860711, | |
| 6.713459035381675, | |
| 6.4778749365359545, | |
| 7.396250031888485, | |
| 6.585500203073025, | |
| 7.316000061109662, | |
| 6.5283330623060465, | |
| 7.202249951660633, | |
| 6.45708292722702, | |
| 6.033665966242552, | |
| 7.2585418820381165, | |
| 6.132833892479539, | |
| 7.311125053092837, | |
| 6.536291912198067, | |
| 6.075833924114704, | |
| 7.21258413977921, | |
| 5.9309168718755245, | |
| 5.516042001545429, | |
| 6.445999955758452 | |
| ] | |
| }, | |
| "short-three-types": { | |
| "p50_ms": 10.712645831517875, | |
| "p95_ms": 12.019854364916682, | |
| "samples_ms": [ | |
| 10.53762505762279, | |
| 9.504124987870455, | |
| 10.739124845713377, | |
| 9.548000060021877, | |
| 10.872208047658205, | |
| 10.638416977599263, | |
| 9.546666871756315, | |
| 11.020832927897573, | |
| 9.458292042836547, | |
| 10.091457981616259, | |
| 10.686166817322373, | |
| 9.469332871958613, | |
| 10.616583051159978, | |
| 9.507999988272786, | |
| 9.960833005607128, | |
| 10.749541921541095, | |
| 10.348625015467405, | |
| 11.054417118430138, | |
| 10.762958088889718, | |
| 10.18533413298428, | |
| 10.920584201812744, | |
| 11.341291014105082, | |
| 9.913707850500941, | |
| 11.748791905120015, | |
| 12.019166955724359, | |
| 10.928499978035688, | |
| 12.020416790619493, | |
| 11.9377919472754, | |
| 11.339165968820453, | |
| 12.478082906454802 | |
| ] | |
| }, | |
| "long-three-types": { | |
| "p50_ms": 186.60743755754083, | |
| "p95_ms": 258.3231775206513, | |
| "samples_ms": [ | |
| 318.01679194904864, | |
| 265.0479590520263, | |
| 243.69887495413423, | |
| 250.10400009341538, | |
| 238.5930421296507, | |
| 219.68429116532207, | |
| 202.91625009849668, | |
| 186.29233306273818, | |
| 159.23545905388892, | |
| 165.35883396863937, | |
| 162.2020830400288, | |
| 162.32783300802112, | |
| 160.531209083274, | |
| 172.42029099725187, | |
| 169.07654097303748, | |
| 168.83791610598564, | |
| 182.5247078668326, | |
| 177.2719998843968, | |
| 177.06720903515816, | |
| 185.59987493790686, | |
| 184.45079098455608, | |
| 179.83149993233383, | |
| 192.80395796522498, | |
| 186.9225420523435, | |
| 189.82570921070874, | |
| 197.10833299905062, | |
| 192.07008415833116, | |
| 191.10641698352993, | |
| 190.38137514144182, | |
| 187.29425012134016 | |
| ] | |
| } | |
| }, | |
| "peak_rss_bytes": 6148816896, | |
| "mlx_peak_bytes": 1421638915 | |
| } | |
| }, | |
| "mlx": "0.32.2" | |
| } | |