{
  "model": "xDecision",
  "processor": "Apple M5",
  "memory_bytes": 25769803776,
  "machine": "macOS-26.6.2-arm64-arm-64bit",
  "torch": "2.14.0",
  "threads": 4,
  "iterations": 30,
  "warmup": 5,
  "mlx_enable_tf32": "0",
  "input_sha256": "5475c43729d48eb4ef55a952a571dd40f0206890a49ef5b0f51f0bda3f88313e",
  "timing": "Sequential backends; synchronized end-to-end predict, excluding model load. No output cache.",
  "limits": "One machine/session; agreement is not accuracy. Load and first request are not fresh-process cold starts. Process RSS is cumulative across backends; MLX allocation is separate.",
  "results": {
    "torch-cpu": {
      "load_s": 1.8336434159427881,
      "first_request_ms": 65.17608417198062,
      "parity": {
        "questions": 42,
        "argmax_agreement": 42,
        "max_probability_error": 0.0,
        "max_act_probability_error": 0.0,
        "max_score_error": 0.0,
        "max_confidence_error": 0.0
      },
      "timings": {
        "short-choice": {
          "p50_ms": 25.009167031385005,
          "p95_ms": 26.35008773067966,
          "samples_ms": [
            25.580249959602952,
            24.904084159061313,
            25.626499904319644,
            25.12637502513826,
            25.341583183035254,
            24.616667069494724,
            24.470083182677627,
            23.965499829500914,
            24.464874994009733,
            23.81699997931719,
            25.715457974001765,
            23.56974990107119,
            26.23041719198227,
            24.62316700257361,
            25.330750038847327,
            23.640000028535724,
            24.533582851290703,
            23.842666065320373,
            23.983041988685727,
            28.93995796330273,
            25.99345799535513,
            25.292209116742015,
            25.987499859184027,
            25.286666117608547,
            25.114249903708696,
            24.546624859794974,
            26.44799998961389,
            25.22704191505909,
            24.410415906459093,
            24.01049993932247
          ]
        },
        "short-three-types": {
          "p50_ms": 59.266437543556094,
          "p95_ms": 61.17129607591778,
          "samples_ms": [
            58.44441708177328,
            59.365374967455864,
            59.219166869297624,
            57.941083097830415,
            57.72687494754791,
            57.723040925338864,
            58.920334093272686,
            58.38920804671943,
            58.05695801973343,
            58.266458101570606,
            58.117832988500595,
            58.81129205226898,
            60.072166845202446,
            59.27404202520847,
            60.32779207453132,
            57.795916916802526,
            61.952083138749,
            60.658416943624616,
            59.784374898299575,
            58.85491706430912,
            59.515458066016436,
            58.75099985860288,
            59.258833061903715,
            60.75645820237696,
            59.79149998165667,
            60.46687485650182,
            60.36554090678692,
            59.47137484326959,
            61.51070888154209,
            59.39895799383521
          ]
        },
        "long-three-types": {
          "p50_ms": 1033.5651044733822,
          "p95_ms": 1116.2748997681774,
          "samples_ms": [
            1128.8698751013726,
            1024.5591660495847,
            1014.6183748729527,
            1012.2548341751099,
            1023.1867081020027,
            1024.2838331032544,
            1019.5401250384748,
            1014.0557079575956,
            1027.6388749480247,
            1023.492667125538,
            1041.8245408218354,
            1042.373958043754,
            1064.232167089358,
            1050.7710000965744,
            1054.0782499592751,
            1100.8810410276055,
            1031.363375019282,
            1036.6499170195311,
            1039.8580830078572,
            1027.7309999801219,
            1052.3458749521524,
            1034.1426669619977,
            1032.9875419847667,
            1031.6930410917848,
            1030.512792058289,
            1026.6135421115905,
            1074.8492919374257,
            1056.9510420318693,
            1081.474999897182,
            1160.2122080512345
          ]
        }
      },
      "peak_rss_bytes": 2708946944,
      "mlx_peak_bytes": null
    },
    "torch-mps": {
      "load_s": 0.1871800001244992,
      "first_request_ms": 125.95162494108081,
      "parity": {
        "questions": 42,
        "argmax_agreement": 42,
        "max_probability_error": 0.0,
        "max_act_probability_error": 0.0,
        "max_score_error": 0.0,
        "max_confidence_error": 0.0
      },
      "timings": {
        "short-choice": {
          "p50_ms": 14.111166470684111,
          "p95_ms": 15.085989504586903,
          "samples_ms": [
            14.610416954383254,
            13.652416877448559,
            13.500791043043137,
            14.008042169734836,
            13.934707967564464,
            14.189000008627772,
            12.798999901860952,
            14.285790966823697,
            15.36308298818767,
            14.334207866340876,
            14.643166912719607,
            13.611707836389542,
            14.086582930758595,
            14.164666179567575,
            12.87633297033608,
            14.085000148043036,
            14.7192080039531,
            13.887250097468495,
            13.825541129335761,
            12.815791182219982,
            14.562875032424927,
            13.600333128124475,
            14.791625086218119,
            14.135750010609627,
            14.679709216579795,
            14.545290963724256,
            12.948499992489815,
            13.461249880492687,
            14.168542111292481,
            15.326833119615912
          ]
        },
        "short-three-types": {
          "p50_ms": 28.994853957556188,
          "p95_ms": 32.37735977163538,
          "samples_ms": [
            27.80325012281537,
            28.71349989436567,
            28.994916938245296,
            28.362750075757504,
            29.32179206982255,
            28.771250043064356,
            29.572124825790524,
            29.29999982006848,
            29.796124901622534,
            28.99479097686708,
            30.674332985654473,
            32.34566608443856,
            33.36429107002914,
            32.403290970250964,
            32.118957955390215,
            30.328291933983564,
            29.698500176891685,
            28.179042041301727,
            29.035374987870455,
            28.080582851544023,
            28.442249866202474,
            28.57420896179974,
            27.672333177179098,
            28.317665914073586,
            29.480499913915992,
            28.979500057175756,
            27.944750152528286,
            28.508624993264675,
            29.175875009968877,
            28.334291884675622
          ]
        },
        "long-three-types": {
          "p50_ms": 424.481229390949,
          "p95_ms": 447.1228920505382,
          "samples_ms": [
            416.01675003767014,
            412.92400006204844,
            417.44862496852875,
            416.14104178734124,
            416.77662497386336,
            420.62212503515184,
            416.1499999463558,
            422.2178750205785,
            417.6560000050813,
            421.9182909000665,
            421.87454109080136,
            419.34454208239913,
            424.60116697475314,
            425.74237496592104,
            444.5189170073718,
            451.4110831078142,
            449.2534170858562,
            438.7376250233501,
            437.65700003132224,
            441.4112498052418,
            442.23374989815056,
            435.749749885872,
            432.55587480962276,
            427.5060829240829,
            430.12141599319875,
            429.78799995034933,
            429.1073330678046,
            419.95962522923946,
            424.3612918071449,
            421.29729106090963
          ]
        }
      },
      "peak_rss_bytes": 4395679744,
      "mlx_peak_bytes": null
    },
    "mlx-f32": {
      "load_s": 0.8362782921176404,
      "first_request_ms": 62.71808291785419,
      "parity": {
        "questions": 42,
        "argmax_agreement": 42,
        "max_probability_error": 0.0,
        "max_act_probability_error": 0.0,
        "max_score_error": 0.0,
        "max_confidence_error": 0.0
      },
      "timings": {
        "short-choice": {
          "p50_ms": 8.646207978017628,
          "p95_ms": 9.25481078447774,
          "samples_ms": [
            9.07829194329679,
            8.18304205313325,
            9.08291689120233,
            8.201583987101912,
            8.889124961569905,
            8.197375107556581,
            8.84479214437306,
            8.150875102728605,
            8.756584022194147,
            8.023166097700596,
            8.873207960277796,
            8.028666023164988,
            8.933624951168895,
            8.012040983885527,
            8.973958902060986,
            8.308249991387129,
            9.084292221814394,
            8.267541881650686,
            9.461707901209593,
            8.186208084225655,
            8.84658400900662,
            8.227124810218811,
            8.559208130463958,
            8.233916945755482,
            8.653250057250261,
            8.205916965380311,
            8.639165898784995,
            9.271292015910149,
            8.67229187861085,
            9.234667057171464
          ]
        },
        "short-three-types": {
          "p50_ms": 27.439979137852788,
          "p95_ms": 29.60230848984793,
          "samples_ms": [
            25.90083284303546,
            26.88808413222432,
            25.741792051121593,
            27.01454097405076,
            26.13366604782641,
            26.60712506622076,
            27.23487513139844,
            26.556584052741528,
            26.912541827186942,
            26.535333832725883,
            26.153957936912775,
            27.079958003014326,
            26.55725018121302,
            28.237124904990196,
            27.948707807809114,
            28.098125010728836,
            30.112708918750286,
            29.620083048939705,
            28.33245904184878,
            28.19887502118945,
            27.645083144307137,
            26.446667034178972,
            26.561542181298137,
            28.03670894354582,
            28.779208194464445,
            28.47195789217949,
            28.503790963441133,
            29.580584028735757,
            29.099459061399102,
            28.65987503901124
          ]
        },
        "long-three-types": {
          "p50_ms": 408.2140205428004,
          "p95_ms": 420.70403959369287,
          "samples_ms": [
            395.5369577743113,
            399.7525000013411,
            402.282000053674,
            413.9239580836147,
            422.1989579964429,
            418.8769171014428,
            416.1644580308348,
            436.78858294151723,
            408.2535409834236,
            408.4692499600351,
            403.07912486605346,
            406.7489579319954,
            405.28624993748963,
            407.0567497983575,
            409.4870421104133,
            407.96366706490517,
            408.0827080179006,
            405.73779097758234,
            407.35787502489984,
            407.85816707648337,
            404.0020841639489,
            408.17450010217726,
            409.347582841292,
            409.3512080144137,
            409.3461660668254,
            405.57579114101827,
            410.8832080382854,
            410.66658310592175,
            413.0809591151774,
            411.7653341963887
          ]
        }
      },
      "peak_rss_bytes": 4395679744,
      "mlx_peak_bytes": 2843246698
    },
    "mlx-f16": {
      "load_s": 0.43084120796993375,
      "first_request_ms": 19.334916956722736,
      "parity": {
        "questions": 42,
        "argmax_agreement": 42,
        "max_probability_error": 0.0030000000000000027,
        "max_act_probability_error": 0.0020999999999999908,
        "max_score_error": 0.0012000000000000899,
        "max_confidence_error": 0.0030000000000000027
      },
      "timings": {
        "short-choice": {
          "p50_ms": 5.208916496485472,
          "p95_ms": 6.072402570862323,
          "samples_ms": [
            5.075416993349791,
            5.137792089954019,
            6.072834134101868,
            5.10474992915988,
            6.321166874840856,
            4.897790960967541,
            5.235000047832727,
            5.09270909242332,
            5.874207941815257,
            5.320167168974876,
            5.103500094264746,
            6.06404198333621,
            5.257207900285721,
            4.987166961655021,
            5.835875170305371,
            5.248166853561997,
            4.975625080987811,
            5.796832963824272,
            5.181041080504656,
            4.941333085298538,
            5.786125082522631,
            5.407500080764294,
            5.114791914820671,
            5.8659580536186695,
            5.196749931201339,
            4.893959034234285,
            6.071875104680657,
            5.221083061769605,
            4.907374968752265,
            5.124625051394105
          ]
        },
        "short-three-types": {
          "p50_ms": 11.030271067284048,
          "p95_ms": 12.32145627727732,
          "samples_ms": [
            9.481834014877677,
            11.024374980479479,
            9.665166959166527,
            10.315499966964126,
            10.323791997507215,
            10.239499853923917,
            10.601666057482362,
            9.750708937644958,
            11.057124938815832,
            11.941166827455163,
            10.765250073745847,
            12.366999872028828,
            12.512624962255359,
            12.177583994343877,
            11.05537498369813,
            12.265791883692145,
            12.0297078974545,
            10.916166938841343,
            11.379000032320619,
            11.192667065188289,
            10.09087497368455,
            11.284457985311747,
            11.039916891604662,
            10.075125144794583,
            11.041667079553008,
            10.825832840055227,
            10.167042026296258,
            11.036167154088616,
            11.068291962146759,
            10.08412498049438
          ]
        },
        "long-three-types": {
          "p50_ms": 125.08991651702672,
          "p95_ms": 127.91901449672878,
          "samples_ms": [
            121.85625010170043,
            122.14362481608987,
            123.74708289280534,
            123.17941710352898,
            122.63866607099771,
            124.6428748127073,
            124.61724993772805,
            123.61424998380244,
            124.05583285726607,
            125.16270810738206,
            125.93666696920991,
            127.71466700360179,
            126.8857920076698,
            123.60641593113542,
            123.7822079565376,
            124.26758301444352,
            124.24008292146027,
            125.06404099985957,
            124.63741702958941,
            127.16299993917346,
            125.79516717232764,
            125.11579203419387,
            125.93049998395145,
            125.45570801012218,
            126.10699981451035,
            125.36016595549881,
            126.64404208771884,
            127.44825007393956,
            130.26875001378357,
            128.0862079001963
          ]
        }
      },
      "peak_rss_bytes": 4395679744,
      "mlx_peak_bytes": 1421638915
    },
    "mlx-gguf-f16": {
      "load_s": 14.322720540920272,
      "first_request_ms": 26.145249838009477,
      "parity": {
        "questions": 42,
        "argmax_agreement": 42,
        "max_probability_error": 0.0030000000000000027,
        "max_act_probability_error": 0.0020999999999999908,
        "max_score_error": 0.0012000000000000899,
        "max_confidence_error": 0.0030000000000000027
      },
      "timings": {
        "short-choice": {
          "p50_ms": 5.142416339367628,
          "p95_ms": 5.902735039126128,
          "samples_ms": [
            5.306666949763894,
            5.1711671985685825,
            5.105291958898306,
            5.774541990831494,
            5.194957833737135,
            5.091875093057752,
            5.86091703735292,
            5.198708036914468,
            4.931250121444464,
            5.810125032439828,
            5.1814590115100145,
            5.103625124320388,
            5.80691616050899,
            5.119749810546637,
            4.867082927376032,
            5.9305408503860235,
            5.064791068434715,
            4.862291039898992,
            4.877207800745964,
            6.1593749560415745,
            4.819917026907206,
            4.881040891632438,
            5.868750158697367,
            5.036500049754977,
            4.845791961997747,
            5.753041943535209,
            5.16508286818862,
            4.9448751378804445,
            5.8095408603549,
            5.098082823678851
          ]
        },
        "short-three-types": {
          "p50_ms": 13.163916999474168,
          "p95_ms": 20.25712855393067,
          "samples_ms": [
            10.799874784424901,
            9.871625108644366,
            14.080458087846637,
            15.84208314307034,
            20.250207977369428,
            24.500958854332566,
            20.063458010554314,
            20.262790843844414,
            18.66349996998906,
            16.93916693329811,
            16.21062494814396,
            15.926875174045563,
            15.474790940061212,
            15.150333987548947,
            14.243375044316053,
            14.155917102470994,
            13.539416948333383,
            11.718417052179575,
            12.788417050614953,
            12.328125070780516,
            10.87887492030859,
            10.970166884362698,
            11.961332987993956,
            10.971874929964542,
            10.524041950702667,
            11.234583100304008,
            10.418124962598085,
            10.542208096012473,
            10.890667093917727,
            9.88187501206994
          ]
        },
        "long-three-types": {
          "p50_ms": 124.07070794142783,
          "p95_ms": 126.72857847064734,
          "samples_ms": [
            126.40779092907906,
            125.8652089163661,
            120.70324993692338,
            125.4580831155181,
            122.17758386395872,
            125.37262495607138,
            124.23625006340444,
            124.41674992442131,
            125.48016617074609,
            120.07174990139902,
            122.76983400806785,
            124.17341582477093,
            123.15416592173278,
            126.99104100465775,
            125.74133416637778,
            120.3838330693543,
            123.37704095989466,
            122.06491688266397,
            121.12629204057157,
            125.58470806106925,
            123.96800005808473,
            123.88745788484812,
            123.78191691823304,
            124.87612501718104,
            127.77420901693404,
            126.26737495884299,
            119.91520808078349,
            121.14250008016825,
            123.31841606646776,
            125.73825009167194
          ]
        }
      },
      "peak_rss_bytes": 6111084544,
      "mlx_peak_bytes": 1421638915
    },
    "torch-gguf-q8": {
      "load_s": 16.790536124957725,
      "first_request_ms": 65.82020805217326,
      "parity": {
        "questions": 42,
        "argmax_agreement": 42,
        "max_probability_error": 0.015200000000000102,
        "max_act_probability_error": 0.07369999999999999,
        "max_score_error": 0.00550000000000006,
        "max_confidence_error": 0.015199999999999991
      },
      "timings": {
        "short-choice": {
          "p50_ms": 26.122875045984983,
          "p95_ms": 27.80325028579682,
          "samples_ms": [
            26.299958117306232,
            26.505500078201294,
            26.535833021625876,
            26.517125079408288,
            26.899375021457672,
            27.410291135311127,
            26.70149994082749,
            26.932957815006375,
            25.5129998549819,
            26.386334095150232,
            25.44220886193216,
            26.027417043223977,
            27.98812510445714,
            25.280416943132877,
            25.31812503002584,
            24.913457920774817,
            25.119666941463947,
            24.88604187965393,
            25.22508311085403,
            24.90699989721179,
            27.577292174100876,
            26.21833304874599,
            25.71058296598494,
            25.68858303129673,
            25.570583064109087,
            25.84200003184378,
            28.020832920446992,
            27.051375014707446,
            26.395124848932028,
            25.549958925694227
          ]
        },
        "short-three-types": {
          "p50_ms": 62.49914539512247,
          "p95_ms": 125.7209476432763,
          "samples_ms": [
            60.92745787464082,
            63.102582935243845,
            62.82237498089671,
            62.11025011725724,
            63.319875160232186,
            60.849040979519486,
            61.882499838247895,
            61.086750123649836,
            61.313292011618614,
            60.750583885237575,
            60.62925001606345,
            59.70504181459546,
            61.28466711379588,
            61.57308304682374,
            62.57187481969595,
            63.209500163793564,
            60.82345801405609,
            62.42641597054899,
            61.910832999274135,
            62.38312507048249,
            89.54724995419383,
            75.72958385571837,
            134.2952500563115,
            100.7456248626113,
            100.9271249640733,
            125.31062518246472,
            96.37708310037851,
            126.05666602030396,
            94.56258313730359,
            86.44129196181893
          ]
        },
        "long-three-types": {
          "p50_ms": 1266.846957965754,
          "p95_ms": 1403.54498779634,
          "samples_ms": [
            1213.3727078326046,
            1228.3932918217033,
            1249.0241250488907,
            1225.9758750442415,
            1237.6357920002192,
            1254.133875016123,
            1265.902667073533,
            1230.628749821335,
            1267.2083750367165,
            1424.849750008434,
            1289.3726250622422,
            1251.7832079902291,
            1246.0905001498759,
            1266.4855408947915,
            1262.3959579505026,
            1337.7245420124382,
            1272.63587503694,
            1437.6328748185188,
            1348.5361670609564,
            1270.988459000364,
            1268.2836661115289,
            1268.9630831591785,
            1273.3692498877645,
            1377.5058339815587,
            1309.6742080524564,
            1262.4727908987552,
            1260.940541047603,
            1272.8466670960188,
            1275.9640000294894,
            1265.5512921046466
          ]
        }
      },
      "peak_rss_bytes": 6148816896,
      "mlx_peak_bytes": null
    },
    "mlx-gguf-q8": {
      "load_s": 14.982421041931957,
      "first_request_ms": 35.079791909083724,
      "parity": {
        "questions": 42,
        "argmax_agreement": 42,
        "max_probability_error": 0.0030000000000000027,
        "max_act_probability_error": 0.0020999999999999908,
        "max_score_error": 0.0007999999999999119,
        "max_confidence_error": 0.0030000000000000027
      },
      "timings": {
        "short-choice": {
          "p50_ms": 6.608979078009725,
          "p95_ms": 7.761526969261466,
          "samples_ms": [
            6.6324579529464245,
            6.184333935379982,
            7.972707971930504,
            6.234582979232073,
            7.354500005021691,
            6.724750157445669,
            7.503416854888201,
            6.9359170738607645,
            6.3974999357014894,
            8.057290920987725,
            6.2525831162929535,
            7.459083804860711,
            6.713459035381675,
            6.4778749365359545,
            7.396250031888485,
            6.585500203073025,
            7.316000061109662,
            6.5283330623060465,
            7.202249951660633,
            6.45708292722702,
            6.033665966242552,
            7.2585418820381165,
            6.132833892479539,
            7.311125053092837,
            6.536291912198067,
            6.075833924114704,
            7.21258413977921,
            5.9309168718755245,
            5.516042001545429,
            6.445999955758452
          ]
        },
        "short-three-types": {
          "p50_ms": 10.712645831517875,
          "p95_ms": 12.019854364916682,
          "samples_ms": [
            10.53762505762279,
            9.504124987870455,
            10.739124845713377,
            9.548000060021877,
            10.872208047658205,
            10.638416977599263,
            9.546666871756315,
            11.020832927897573,
            9.458292042836547,
            10.091457981616259,
            10.686166817322373,
            9.469332871958613,
            10.616583051159978,
            9.507999988272786,
            9.960833005607128,
            10.749541921541095,
            10.348625015467405,
            11.054417118430138,
            10.762958088889718,
            10.18533413298428,
            10.920584201812744,
            11.341291014105082,
            9.913707850500941,
            11.748791905120015,
            12.019166955724359,
            10.928499978035688,
            12.020416790619493,
            11.9377919472754,
            11.339165968820453,
            12.478082906454802
          ]
        },
        "long-three-types": {
          "p50_ms": 186.60743755754083,
          "p95_ms": 258.3231775206513,
          "samples_ms": [
            318.01679194904864,
            265.0479590520263,
            243.69887495413423,
            250.10400009341538,
            238.5930421296507,
            219.68429116532207,
            202.91625009849668,
            186.29233306273818,
            159.23545905388892,
            165.35883396863937,
            162.2020830400288,
            162.32783300802112,
            160.531209083274,
            172.42029099725187,
            169.07654097303748,
            168.83791610598564,
            182.5247078668326,
            177.2719998843968,
            177.06720903515816,
            185.59987493790686,
            184.45079098455608,
            179.83149993233383,
            192.80395796522498,
            186.9225420523435,
            189.82570921070874,
            197.10833299905062,
            192.07008415833116,
            191.10641698352993,
            190.38137514144182,
            187.29425012134016
          ]
        }
      },
      "peak_rss_bytes": 6148816896,
      "mlx_peak_bytes": 1421638915
    }
  },
  "mlx": "0.32.2"
}