{
 "subset": {
  "n": 1000,
  "warmup": 20,
  "seed": 0,
  "keys": [
   [
    2666,
    2
   ],
   [
    12648,
    4
   ],
   [
    18360,
    5
   ],
   [
    3715,
    0
   ],
   [
    13502,
    3
   ],
   [
    16562,
    4
   ],
   [
    12123,
    10
   ],
   [
    7509,
    1
   ],
   [
    18645,
    1
   ],
   [
    16951,
    3
   ],
   [
    10058,
    2
   ],
   [
    2269,
    11
   ],
   [
    548,
    0
   ],
   [
    3292,
    0
   ],
   [
    15225,
    2
   ],
   [
    54,
    1
   ],
   [
    19300,
    2
   ],
   [
    12754,
    1
   ],
   [
    9168,
    0
   ],
   [
    12499,
    3
   ],
   [
    2288,
    3
   ],
   [
    6903,
    0
   ],
   [
    8188,
    0
   ],
   [
    8917,
    3
   ],
   [
    5288,
    5
   ],
   [
    17180,
    0
   ],
   [
    3224,
    0
   ],
   [
    2856,
    3
   ],
   [
    12289,
    3
   ],
   [
    19833,
    1
   ],
   [
    19170,
    0
   ],
   [
    4158,
    1
   ],
   [
    11571,
    3
   ],
   [
    11274,
    8
   ],
   [
    4674,
    2
   ],
   [
    12743,
    4
   ],
   [
    7455,
    5
   ],
   [
    11187,
    6
   ],
   [
    17085,
    3
   ],
   [
    3238,
    0
   ],
   [
    14735,
    1
   ],
   [
    12176,
    4
   ],
   [
    9120,
    3
   ],
   [
    11242,
    3
   ],
   [
    6623,
    6
   ],
   [
    6792,
    6
   ],
   [
    6707,
    2
   ],
   [
    1082,
    10
   ],
   [
    10033,
    0
   ],
   [
    18619,
    3
   ],
   [
    2488,
    18
   ],
   [
    3152,
    0
   ],
   [
    4833,
    0
   ],
   [
    5563,
    3
   ],
   [
    1309,
    6
   ],
   [
    2836,
    3
   ],
   [
    14985,
    5
   ],
   [
    10726,
    0
   ],
   [
    8871,
    1
   ],
   [
    7866,
    1
   ],
   [
    15935,
    3
   ],
   [
    10724,
    6
   ],
   [
    17325,
    0
   ],
   [
    19269,
    5
   ],
   [
    13651,
    2
   ],
   [
    2916,
    1
   ],
   [
    12483,
    5
   ],
   [
    4406,
    7
   ],
   [
    19003,
    2
   ],
   [
    12885,
    1
   ],
   [
    6871,
    2
   ],
   [
    9131,
    0
   ],
   [
    586,
    4
   ],
   [
    10558,
    4
   ],
   [
    4451,
    3
   ],
   [
    8146,
    3
   ],
   [
    14230,
    3
   ],
   [
    6258,
    1
   ],
   [
    12743,
    1
   ],
   [
    16212,
    3
   ],
   [
    2269,
    9
   ],
   [
    3708,
    2
   ],
   [
    5407,
    3
   ],
   [
    7999,
    1
   ],
   [
    1581,
    3
   ],
   [
    2669,
    2
   ],
   [
    916,
    0
   ],
   [
    4670,
    0
   ],
   [
    6768,
    3
   ],
   [
    4560,
    1
   ],
   [
    14985,
    4
   ],
   [
    3224,
    5
   ],
   [
    14157,
    9
   ],
   [
    4414,
    5
   ],
   [
    1206,
    1
   ],
   [
    759,
    4
   ],
   [
    6996,
    6
   ],
   [
    6640,
    5
   ],
   [
    4640,
    1
   ],
   [
    18741,
    3
   ],
   [
    7753,
    1
   ],
   [
    2232,
    2
   ],
   [
    803,
    3
   ],
   [
    16208,
    2
   ],
   [
    3751,
    5
   ],
   [
    9963,
    3
   ],
   [
    2553,
    2
   ],
   [
    8158,
    1
   ],
   [
    2609,
    7
   ],
   [
    11551,
    7
   ],
   [
    13440,
    5
   ],
   [
    16614,
    0
   ],
   [
    6472,
    6
   ],
   [
    19676,
    9
   ],
   [
    18141,
    2
   ],
   [
    1330,
    2
   ],
   [
    3747,
    1
   ],
   [
    7249,
    1
   ],
   [
    10037,
    0
   ],
   [
    13669,
    3
   ],
   [
    18302,
    7
   ],
   [
    6236,
    3
   ],
   [
    7456,
    2
   ],
   [
    2093,
    5
   ],
   [
    5791,
    9
   ],
   [
    5926,
    4
   ],
   [
    13182,
    1
   ],
   [
    9628,
    3
   ],
   [
    4451,
    4
   ],
   [
    16924,
    0
   ],
   [
    6405,
    5
   ],
   [
    470,
    2
   ],
   [
    18351,
    0
   ],
   [
    15658,
    1
   ],
   [
    19880,
    2
   ],
   [
    11890,
    2
   ],
   [
    13619,
    5
   ],
   [
    14928,
    1
   ],
   [
    9634,
    2
   ],
   [
    5680,
    2
   ],
   [
    406,
    10
   ],
   [
    17629,
    4
   ],
   [
    2777,
    6
   ],
   [
    12904,
    0
   ],
   [
    1586,
    1
   ],
   [
    10966,
    0
   ],
   [
    5016,
    0
   ],
   [
    8982,
    11
   ],
   [
    18883,
    3
   ],
   [
    13470,
    1
   ],
   [
    11198,
    1
   ],
   [
    13674,
    3
   ],
   [
    4916,
    7
   ],
   [
    11870,
    2
   ],
   [
    14843,
    0
   ],
   [
    15775,
    3
   ],
   [
    2860,
    3
   ],
   [
    55,
    5
   ],
   [
    6919,
    0
   ],
   [
    12823,
    5
   ],
   [
    5852,
    2
   ],
   [
    8963,
    3
   ],
   [
    8202,
    2
   ],
   [
    17180,
    3
   ],
   [
    14454,
    1
   ],
   [
    1049,
    5
   ],
   [
    15405,
    0
   ],
   [
    15890,
    4
   ],
   [
    1632,
    5
   ],
   [
    6119,
    1
   ],
   [
    17098,
    4
   ],
   [
    2302,
    12
   ],
   [
    9935,
    9
   ],
   [
    5791,
    0
   ],
   [
    17114,
    0
   ],
   [
    19019,
    4
   ],
   [
    27,
    1
   ],
   [
    1311,
    0
   ],
   [
    19315,
    1
   ],
   [
    12514,
    4
   ],
   [
    11912,
    0
   ],
   [
    18124,
    2
   ],
   [
    1741,
    6
   ],
   [
    15802,
    0
   ],
   [
    6760,
    1
   ],
   [
    2991,
    0
   ],
   [
    4833,
    5
   ],
   [
    569,
    4
   ],
   [
    15386,
    1
   ],
   [
    15865,
    2
   ],
   [
    122,
    5
   ],
   [
    7826,
    8
   ],
   [
    548,
    4
   ],
   [
    62,
    10
   ],
   [
    3507,
    4
   ],
   [
    6876,
    0
   ],
   [
    4545,
    3
   ],
   [
    7204,
    3
   ],
   [
    11622,
    8
   ],
   [
    10925,
    4
   ],
   [
    6574,
    0
   ],
   [
    3700,
    2
   ],
   [
    18577,
    0
   ],
   [
    15100,
    8
   ],
   [
    2886,
    1
   ],
   [
    777,
    2
   ],
   [
    10711,
    6
   ],
   [
    17452,
    0
   ],
   [
    4414,
    0
   ],
   [
    9823,
    8
   ],
   [
    4976,
    3
   ],
   [
    13355,
    1
   ],
   [
    4406,
    3
   ],
   [
    5707,
    3
   ],
   [
    10844,
    7
   ],
   [
    662,
    0
   ],
   [
    1464,
    2
   ],
   [
    1364,
    0
   ],
   [
    7565,
    0
   ],
   [
    9961,
    2
   ],
   [
    12045,
    6
   ],
   [
    13970,
    3
   ],
   [
    1438,
    4
   ],
   [
    19300,
    17
   ],
   [
    17661,
    3
   ],
   [
    16593,
    2
   ],
   [
    14251,
    2
   ],
   [
    6443,
    4
   ],
   [
    7608,
    8
   ],
   [
    14307,
    2
   ],
   [
    11274,
    7
   ],
   [
    270,
    0
   ],
   [
    5199,
    1
   ],
   [
    5611,
    4
   ],
   [
    10559,
    4
   ],
   [
    12758,
    0
   ],
   [
    12926,
    2
   ],
   [
    13975,
    3
   ],
   [
    3315,
    2
   ],
   [
    12933,
    3
   ],
   [
    1198,
    0
   ],
   [
    1378,
    1
   ],
   [
    10501,
    0
   ],
   [
    5990,
    4
   ],
   [
    5563,
    1
   ],
   [
    11234,
    4
   ],
   [
    13731,
    2
   ],
   [
    15063,
    0
   ],
   [
    4812,
    1
   ],
   [
    11326,
    1
   ],
   [
    4406,
    0
   ],
   [
    18679,
    0
   ],
   [
    8982,
    4
   ],
   [
    1704,
    0
   ],
   [
    11831,
    4
   ],
   [
    6463,
    0
   ],
   [
    2582,
    6
   ],
   [
    11623,
    3
   ],
   [
    15425,
    2
   ],
   [
    12624,
    5
   ],
   [
    11501,
    1
   ],
   [
    15775,
    8
   ],
   [
    4114,
    3
   ],
   [
    3592,
    2
   ],
   [
    18828,
    5
   ],
   [
    18521,
    3
   ],
   [
    12904,
    19
   ],
   [
    13247,
    2
   ],
   [
    4662,
    2
   ],
   [
    18741,
    1
   ],
   [
    4414,
    4
   ],
   [
    19453,
    4
   ],
   [
    16259,
    0
   ],
   [
    1302,
    0
   ],
   [
    11622,
    0
   ],
   [
    12863,
    0
   ],
   [
    5741,
    6
   ],
   [
    6153,
    0
   ],
   [
    3031,
    2
   ],
   [
    2405,
    3
   ],
   [
    3000,
    10
   ],
   [
    7195,
    3
   ],
   [
    8158,
    4
   ],
   [
    2232,
    3
   ],
   [
    14723,
    8
   ],
   [
    229,
    3
   ],
   [
    3519,
    3
   ],
   [
    15048,
    1
   ],
   [
    11234,
    9
   ],
   [
    17181,
    3
   ],
   [
    19038,
    5
   ],
   [
    7935,
    6
   ],
   [
    16072,
    3
   ],
   [
    2991,
    3
   ],
   [
    14103,
    2
   ],
   [
    8084,
    1
   ],
   [
    10058,
    9
   ],
   [
    6153,
    1
   ],
   [
    16464,
    21
   ],
   [
    6904,
    12
   ],
   [
    13669,
    2
   ],
   [
    4406,
    2
   ],
   [
    937,
    6
   ],
   [
    17402,
    8
   ],
   [
    7379,
    1
   ],
   [
    19382,
    2
   ],
   [
    15204,
    1
   ],
   [
    7607,
    2
   ],
   [
    1458,
    0
   ],
   [
    7884,
    1
   ],
   [
    3833,
    0
   ],
   [
    7195,
    2
   ],
   [
    17647,
    4
   ],
   [
    14390,
    3
   ],
   [
    13745,
    5
   ],
   [
    5636,
    3
   ],
   [
    3833,
    2
   ],
   [
    19033,
    1
   ],
   [
    5459,
    5
   ],
   [
    15527,
    4
   ],
   [
    16101,
    2
   ],
   [
    19347,
    0
   ],
   [
    12429,
    0
   ],
   [
    19510,
    1
   ],
   [
    19537,
    0
   ],
   [
    7382,
    2
   ],
   [
    296,
    0
   ],
   [
    13070,
    1
   ],
   [
    12223,
    0
   ],
   [
    12364,
    1
   ],
   [
    1194,
    2
   ],
   [
    5459,
    4
   ],
   [
    9823,
    14
   ],
   [
    5742,
    2
   ],
   [
    14454,
    7
   ],
   [
    11379,
    1
   ],
   [
    18302,
    10
   ],
   [
    2406,
    3
   ],
   [
    3000,
    9
   ],
   [
    8282,
    2
   ],
   [
    1360,
    6
   ],
   [
    11550,
    0
   ],
   [
    569,
    13
   ],
   [
    17181,
    4
   ],
   [
    12649,
    1
   ],
   [
    5899,
    0
   ],
   [
    5497,
    4
   ],
   [
    17842,
    1
   ],
   [
    14212,
    4
   ],
   [
    19719,
    4
   ],
   [
    14607,
    1
   ],
   [
    19657,
    5
   ],
   [
    1089,
    6
   ],
   [
    3195,
    1
   ],
   [
    2748,
    2
   ],
   [
    16228,
    1
   ],
   [
    7585,
    0
   ],
   [
    11234,
    6
   ],
   [
    15886,
    1
   ],
   [
    18894,
    0
   ],
   [
    14928,
    0
   ],
   [
    8770,
    1
   ],
   [
    723,
    0
   ],
   [
    27,
    3
   ],
   [
    6562,
    9
   ],
   [
    11623,
    0
   ],
   [
    19812,
    4
   ],
   [
    9793,
    2
   ],
   [
    12743,
    3
   ],
   [
    2405,
    0
   ],
   [
    19300,
    1
   ],
   [
    10081,
    2
   ],
   [
    11632,
    2
   ],
   [
    15613,
    1
   ],
   [
    14681,
    2
   ],
   [
    14654,
    1
   ],
   [
    5990,
    3
   ],
   [
    4782,
    0
   ],
   [
    8939,
    1
   ],
   [
    11185,
    10
   ],
   [
    12816,
    4
   ],
   [
    1969,
    3
   ],
   [
    1198,
    4
   ],
   [
    18875,
    2
   ],
   [
    15886,
    2
   ],
   [
    5251,
    6
   ],
   [
    19268,
    2
   ],
   [
    2889,
    1
   ],
   [
    5636,
    4
   ],
   [
    13490,
    4
   ],
   [
    15718,
    1
   ],
   [
    1193,
    2
   ],
   [
    18087,
    4
   ],
   [
    14774,
    4
   ],
   [
    17677,
    0
   ],
   [
    1654,
    0
   ],
   [
    18306,
    4
   ],
   [
    719,
    4
   ],
   [
    1082,
    1
   ],
   [
    4943,
    0
   ],
   [
    12479,
    0
   ],
   [
    3845,
    2
   ],
   [
    13349,
    8
   ],
   [
    7020,
    0
   ],
   [
    14651,
    1
   ],
   [
    19197,
    1
   ],
   [
    4195,
    4
   ],
   [
    2195,
    4
   ],
   [
    18141,
    3
   ],
   [
    12926,
    8
   ],
   [
    11417,
    3
   ],
   [
    4745,
    4
   ],
   [
    14804,
    0
   ],
   [
    11332,
    2
   ],
   [
    4608,
    3
   ],
   [
    6792,
    2
   ],
   [
    1306,
    3
   ],
   [
    15017,
    1
   ],
   [
    17065,
    2
   ],
   [
    14220,
    0
   ],
   [
    17533,
    2
   ],
   [
    13605,
    4
   ],
   [
    2685,
    0
   ],
   [
    1542,
    0
   ],
   [
    1359,
    1
   ],
   [
    18991,
    1
   ],
   [
    9804,
    8
   ],
   [
    907,
    11
   ],
   [
    7884,
    2
   ],
   [
    8537,
    17
   ],
   [
    3297,
    0
   ],
   [
    15966,
    9
   ],
   [
    19813,
    2
   ],
   [
    11726,
    10
   ],
   [
    4297,
    5
   ],
   [
    5373,
    6
   ],
   [
    16212,
    1
   ],
   [
    16002,
    2
   ],
   [
    2991,
    9
   ],
   [
    15832,
    1
   ],
   [
    2288,
    2
   ],
   [
    3591,
    4
   ],
   [
    15802,
    1
   ],
   [
    5742,
    6
   ],
   [
    1045,
    8
   ],
   [
    17127,
    2
   ],
   [
    16464,
    18
   ],
   [
    15857,
    7
   ],
   [
    1027,
    2
   ],
   [
    19375,
    1
   ],
   [
    12483,
    8
   ],
   [
    9753,
    0
   ],
   [
    2775,
    3
   ],
   [
    13490,
    0
   ],
   [
    2553,
    8
   ],
   [
    4608,
    0
   ],
   [
    13674,
    4
   ],
   [
    1024,
    3
   ],
   [
    13314,
    5
   ],
   [
    13390,
    2
   ],
   [
    6430,
    0
   ],
   [
    300,
    0
   ],
   [
    8658,
    3
   ],
   [
    13886,
    1
   ],
   [
    2582,
    0
   ],
   [
    5342,
    0
   ],
   [
    7609,
    0
   ],
   [
    122,
    2
   ],
   [
    7509,
    6
   ],
   [
    4635,
    6
   ],
   [
    228,
    1
   ],
   [
    11306,
    3
   ],
   [
    14114,
    4
   ],
   [
    898,
    1
   ],
   [
    8738,
    5
   ],
   [
    5276,
    7
   ],
   [
    6710,
    1
   ],
   [
    17479,
    3
   ],
   [
    4244,
    1
   ],
   [
    18619,
    6
   ],
   [
    13274,
    4
   ],
   [
    9921,
    3
   ],
   [
    4812,
    9
   ],
   [
    937,
    11
   ],
   [
    7614,
    0
   ],
   [
    13752,
    2
   ],
   [
    12856,
    4
   ],
   [
    18488,
    0
   ],
   [
    11304,
    4
   ],
   [
    12571,
    3
   ],
   [
    6638,
    1
   ],
   [
    2860,
    4
   ],
   [
    3770,
    0
   ],
   [
    11831,
    2
   ],
   [
    5773,
    3
   ],
   [
    14321,
    5
   ],
   [
    4680,
    4
   ],
   [
    8197,
    0
   ],
   [
    12123,
    5
   ],
   [
    19877,
    2
   ],
   [
    9128,
    1
   ],
   [
    8885,
    2
   ],
   [
    6623,
    7
   ],
   [
    15966,
    3
   ],
   [
    1624,
    0
   ],
   [
    4916,
    5
   ],
   [
    723,
    1
   ],
   [
    2771,
    8
   ],
   [
    2630,
    1
   ],
   [
    4916,
    2
   ],
   [
    15966,
    11
   ],
   [
    11501,
    4
   ],
   [
    15857,
    3
   ],
   [
    5288,
    2
   ],
   [
    16004,
    0
   ],
   [
    11488,
    1
   ],
   [
    13526,
    1
   ],
   [
    3000,
    8
   ],
   [
    9394,
    4
   ],
   [
    17085,
    0
   ],
   [
    14129,
    0
   ],
   [
    2093,
    1
   ],
   [
    14321,
    2
   ],
   [
    15613,
    10
   ],
   [
    231,
    5
   ],
   [
    15857,
    10
   ],
   [
    12303,
    3
   ],
   [
    16837,
    2
   ],
   [
    7472,
    4
   ],
   [
    14220,
    1
   ],
   [
    11306,
    4
   ],
   [
    3198,
    2
   ],
   [
    6666,
    1
   ],
   [
    4114,
    2
   ],
   [
    10786,
    2
   ],
   [
    4226,
    6
   ],
   [
    5684,
    7
   ],
   [
    17111,
    1
   ],
   [
    15225,
    1
   ],
   [
    6666,
    0
   ],
   [
    15999,
    0
   ],
   [
    16467,
    5
   ],
   [
    6405,
    3
   ],
   [
    9394,
    2
   ],
   [
    17457,
    0
   ],
   [
    13072,
    0
   ],
   [
    5288,
    6
   ],
   [
    13554,
    1
   ],
   [
    17900,
    6
   ],
   [
    3026,
    8
   ],
   [
    18938,
    1
   ],
   [
    7456,
    6
   ],
   [
    11390,
    4
   ],
   [
    62,
    2
   ],
   [
    17237,
    0
   ],
   [
    17880,
    7
   ],
   [
    229,
    2
   ],
   [
    7999,
    0
   ],
   [
    11490,
    5
   ],
   [
    4358,
    1
   ],
   [
    11559,
    0
   ],
   [
    5773,
    0
   ],
   [
    16140,
    0
   ],
   [
    18335,
    4
   ],
   [
    3242,
    5
   ],
   [
    19453,
    3
   ],
   [
    8730,
    3
   ],
   [
    15476,
    8
   ],
   [
    10925,
    5
   ],
   [
    4602,
    2
   ],
   [
    10531,
    0
   ],
   [
    1360,
    5
   ],
   [
    27,
    4
   ],
   [
    9823,
    9
   ],
   [
    15204,
    8
   ],
   [
    15100,
    0
   ],
   [
    17065,
    3
   ],
   [
    3761,
    0
   ],
   [
    13502,
    4
   ],
   [
    11077,
    0
   ],
   [
    7045,
    1
   ],
   [
    3010,
    0
   ],
   [
    1194,
    1
   ],
   [
    2553,
    10
   ],
   [
    10137,
    0
   ],
   [
    11728,
    1
   ],
   [
    13061,
    0
   ],
   [
    4535,
    1
   ],
   [
    9464,
    0
   ],
   [
    5973,
    2
   ],
   [
    2488,
    2
   ],
   [
    15778,
    0
   ],
   [
    11234,
    7
   ],
   [
    11040,
    9
   ],
   [
    4979,
    2
   ],
   [
    7733,
    5
   ],
   [
    15703,
    2
   ],
   [
    15438,
    2
   ],
   [
    10844,
    10
   ],
   [
    11296,
    10
   ],
   [
    16924,
    1
   ],
   [
    14220,
    4
   ],
   [
    5154,
    0
   ],
   [
    5776,
    5
   ],
   [
    4635,
    8
   ],
   [
    4581,
    3
   ],
   [
    14556,
    3
   ],
   [
    15349,
    2
   ],
   [
    18182,
    0
   ],
   [
    5215,
    1
   ],
   [
    11494,
    2
   ],
   [
    13512,
    2
   ],
   [
    18438,
    1
   ],
   [
    7969,
    1
   ],
   [
    19061,
    3
   ],
   [
    19636,
    1
   ],
   [
    19269,
    6
   ],
   [
    2200,
    0
   ],
   [
    17037,
    1
   ],
   [
    11549,
    2
   ],
   [
    5312,
    2
   ],
   [
    19347,
    2
   ],
   [
    1842,
    2
   ],
   [
    7878,
    3
   ],
   [
    904,
    7
   ],
   [
    18340,
    1
   ],
   [
    14985,
    0
   ],
   [
    300,
    2
   ],
   [
    2412,
    0
   ],
   [
    2886,
    2
   ],
   [
    15070,
    0
   ],
   [
    219,
    0
   ],
   [
    13735,
    2
   ],
   [
    1378,
    0
   ],
   [
    4422,
    1
   ],
   [
    136,
    0
   ],
   [
    10531,
    5
   ],
   [
    11304,
    2
   ],
   [
    8455,
    2
   ],
   [
    5251,
    2
   ],
   [
    11198,
    2
   ],
   [
    6904,
    0
   ],
   [
    3845,
    4
   ],
   [
    16562,
    1
   ],
   [
    17842,
    4
   ],
   [
    14681,
    1
   ],
   [
    6207,
    8
   ],
   [
    12649,
    3
   ],
   [
    16588,
    4
   ],
   [
    5455,
    5
   ],
   [
    17154,
    2
   ],
   [
    5446,
    0
   ],
   [
    12123,
    8
   ],
   [
    4806,
    6
   ],
   [
    7633,
    6
   ],
   [
    6707,
    3
   ],
   [
    17037,
    3
   ],
   [
    13428,
    7
   ],
   [
    14928,
    5
   ],
   [
    16275,
    0
   ],
   [
    19268,
    4
   ],
   [
    14942,
    3
   ],
   [
    8084,
    2
   ],
   [
    7045,
    4
   ],
   [
    16760,
    1
   ],
   [
    7499,
    3
   ],
   [
    1741,
    5
   ],
   [
    14843,
    5
   ],
   [
    1089,
    1
   ],
   [
    8780,
    3
   ],
   [
    13805,
    3
   ],
   [
    2078,
    0
   ],
   [
    6353,
    2
   ],
   [
    8740,
    1
   ],
   [
    11482,
    2
   ],
   [
    3026,
    10
   ],
   [
    19979,
    5
   ],
   [
    11128,
    2
   ],
   [
    13497,
    3
   ],
   [
    15746,
    0
   ],
   [
    17647,
    2
   ],
   [
    1908,
    1
   ],
   [
    16464,
    11
   ],
   [
    17510,
    0
   ],
   [
    19190,
    3
   ],
   [
    9793,
    6
   ],
   [
    18577,
    1
   ],
   [
    7869,
    1
   ],
   [
    12904,
    21
   ],
   [
    10255,
    4
   ],
   [
    1464,
    0
   ],
   [
    1501,
    0
   ],
   [
    1842,
    5
   ],
   [
    5968,
    0
   ],
   [
    13437,
    0
   ],
   [
    136,
    2
   ],
   [
    228,
    0
   ],
   [
    5241,
    4
   ],
   [
    2302,
    9
   ],
   [
    16324,
    0
   ],
   [
    8193,
    0
   ],
   [
    15100,
    5
   ],
   [
    8158,
    3
   ],
   [
    17457,
    4
   ],
   [
    6930,
    4
   ],
   [
    13019,
    1
   ],
   [
    3761,
    5
   ],
   [
    3023,
    2
   ],
   [
    12245,
    0
   ],
   [
    12434,
    2
   ],
   [
    17538,
    2
   ],
   [
    12499,
    2
   ],
   [
    9806,
    1
   ],
   [
    971,
    0
   ],
   [
    1512,
    0
   ],
   [
    6844,
    3
   ],
   [
    14107,
    3
   ],
   [
    2836,
    1
   ],
   [
    7682,
    2
   ],
   [
    13327,
    1
   ],
   [
    6759,
    2
   ],
   [
    7379,
    0
   ],
   [
    9683,
    0
   ],
   [
    11571,
    1
   ],
   [
    11912,
    3
   ],
   [
    14723,
    6
   ],
   [
    9793,
    5
   ],
   [
    18883,
    7
   ],
   [
    13248,
    1
   ],
   [
    8982,
    8
   ],
   [
    1512,
    3
   ],
   [
    11728,
    4
   ],
   [
    2612,
    4
   ],
   [
    270,
    5
   ],
   [
    19339,
    5
   ],
   [
    16696,
    4
   ],
   [
    1672,
    2
   ],
   [
    15749,
    9
   ],
   [
    17841,
    5
   ],
   [
    16768,
    2
   ],
   [
    4535,
    0
   ],
   [
    2886,
    3
   ],
   [
    8998,
    5
   ],
   [
    3559,
    6
   ],
   [
    5684,
    5
   ],
   [
    15768,
    14
   ],
   [
    7826,
    7
   ],
   [
    16829,
    0
   ],
   [
    2757,
    1
   ],
   [
    16259,
    6
   ],
   [
    15055,
    5
   ],
   [
    1330,
    1
   ],
   [
    6505,
    0
   ],
   [
    9467,
    0
   ],
   [
    19190,
    0
   ],
   [
    8084,
    0
   ],
   [
    4796,
    1
   ],
   [
    10911,
    0
   ],
   [
    13468,
    12
   ],
   [
    12248,
    3
   ],
   [
    16584,
    4
   ],
   [
    4045,
    0
   ],
   [
    11132,
    4
   ],
   [
    7337,
    0
   ],
   [
    11429,
    3
   ],
   [
    16860,
    0
   ],
   [
    20097,
    2
   ],
   [
    17900,
    0
   ],
   [
    10058,
    11
   ],
   [
    10608,
    2
   ],
   [
    8725,
    3
   ],
   [
    591,
    0
   ],
   [
    4545,
    0
   ],
   [
    3566,
    2
   ],
   [
    6338,
    4
   ],
   [
    15775,
    7
   ],
   [
    9431,
    1
   ],
   [
    7969,
    2
   ],
   [
    11128,
    4
   ],
   [
    20189,
    6
   ],
   [
    16353,
    2
   ],
   [
    4618,
    0
   ],
   [
    10608,
    1
   ],
   [
    4535,
    2
   ],
   [
    13669,
    9
   ],
   [
    8537,
    16
   ],
   [
    11040,
    0
   ],
   [
    8165,
    6
   ],
   [
    2387,
    2
   ],
   [
    11829,
    2
   ],
   [
    12581,
    2
   ],
   [
    8770,
    2
   ],
   [
    14265,
    4
   ],
   [
    18800,
    0
   ],
   [
    11185,
    6
   ],
   [
    6304,
    5
   ],
   [
    5147,
    1
   ],
   [
    569,
    16
   ],
   [
    19719,
    2
   ],
   [
    12614,
    2
   ],
   [
    13970,
    4
   ],
   [
    904,
    3
   ],
   [
    4812,
    4
   ],
   [
    15076,
    0
   ],
   [
    5732,
    5
   ],
   [
    6418,
    11
   ],
   [
    19939,
    4
   ],
   [
    2748,
    3
   ],
   [
    5025,
    0
   ],
   [
    7587,
    2
   ],
   [
    19375,
    3
   ],
   [
    8826,
    9
   ],
   [
    4916,
    3
   ],
   [
    8770,
    3
   ],
   [
    14723,
    5
   ],
   [
    13498,
    3
   ],
   [
    4936,
    2
   ],
   [
    4090,
    1
   ],
   [
    907,
    1
   ],
   [
    13669,
    4
   ],
   [
    19088,
    1
   ],
   [
    17538,
    0
   ],
   [
    11843,
    1
   ],
   [
    407,
    0
   ],
   [
    5968,
    3
   ],
   [
    19315,
    3
   ],
   [
    18828,
    1
   ],
   [
    11972,
    0
   ],
   [
    2777,
    1
   ],
   [
    9935,
    8
   ],
   [
    5147,
    3
   ],
   [
    12163,
    2
   ],
   [
    11296,
    1
   ],
   [
    14681,
    0
   ],
   [
    2146,
    0
   ],
   [
    7633,
    2
   ],
   [
    1309,
    0
   ],
   [
    12076,
    0
   ],
   [
    9466,
    1
   ],
   [
    13217,
    4
   ],
   [
    16837,
    0
   ],
   [
    8333,
    2
   ],
   [
    10033,
    1
   ],
   [
    13256,
    0
   ],
   [
    5956,
    2
   ],
   [
    11726,
    3
   ],
   [
    591,
    7
   ],
   [
    13605,
    1
   ],
   [
    2098,
    1
   ],
   [
    5636,
    1
   ],
   [
    13498,
    2
   ],
   [
    777,
    5
   ],
   [
    19242,
    2
   ],
   [
    2250,
    1
   ],
   [
    898,
    0
   ],
   [
    9072,
    1
   ],
   [
    1569,
    0
   ],
   [
    8333,
    0
   ],
   [
    12546,
    2
   ],
   [
    2426,
    0
   ],
   [
    2269,
    13
   ],
   [
    13307,
    5
   ],
   [
    16035,
    1
   ],
   [
    5029,
    0
   ],
   [
    7903,
    4
   ],
   [
    17207,
    2
   ],
   [
    16588,
    3
   ],
   [
    5278,
    2
   ],
   [
    11900,
    4
   ],
   [
    6418,
    13
   ],
   [
    12624,
    10
   ],
   [
    15613,
    11
   ],
   [
    14620,
    3
   ],
   [
    282,
    4
   ],
   [
    15646,
    0
   ],
   [
    10167,
    6
   ],
   [
    17953,
    0
   ],
   [
    1438,
    3
   ],
   [
    4618,
    13
   ],
   [
    15613,
    15
   ],
   [
    14942,
    15
   ],
   [
    6304,
    4
   ],
   [
    122,
    0
   ],
   [
    19572,
    4
   ],
   [
    5185,
    2
   ],
   [
    20097,
    1
   ],
   [
    5459,
    1
   ],
   [
    2814,
    5
   ],
   [
    12648,
    2
   ],
   [
    8898,
    3
   ],
   [
    6418,
    3
   ],
   [
    9318,
    5
   ],
   [
    6207,
    18
   ],
   [
    6207,
    9
   ],
   [
    2775,
    2
   ],
   [
    16259,
    10
   ],
   [
    3793,
    6
   ],
   [
    17628,
    2
   ],
   [
    5564,
    1
   ],
   [
    1359,
    2
   ],
   [
    9747,
    3
   ],
   [
    14321,
    3
   ],
   [
    937,
    0
   ],
   [
    1206,
    0
   ],
   [
    19366,
    3
   ],
   [
    3148,
    2
   ],
   [
    13669,
    0
   ],
   [
    11304,
    6
   ],
   [
    5611,
    0
   ],
   [
    17629,
    6
   ],
   [
    8871,
    12
   ],
   [
    19813,
    7
   ],
   [
    13560,
    0
   ],
   [
    5956,
    1
   ],
   [
    15457,
    4
   ],
   [
    12926,
    4
   ],
   [
    10527,
    2
   ],
   [
    19269,
    0
   ],
   [
    15020,
    5
   ],
   [
    569,
    5
   ],
   [
    11885,
    2
   ],
   [
    11198,
    7
   ],
   [
    18231,
    1
   ],
   [
    1170,
    4
   ],
   [
    10098,
    0
   ],
   [
    17563,
    0
   ],
   [
    15063,
    4
   ],
   [
    4579,
    0
   ],
   [
    15442,
    3
   ],
   [
    13349,
    7
   ],
   [
    19349,
    5
   ],
   [
    1819,
    5
   ],
   [
    723,
    3
   ],
   [
    10672,
    1
   ],
   [
    1089,
    17
   ],
   [
    9783,
    4
   ],
   [
    11234,
    8
   ],
   [
    7607,
    0
   ],
   [
    13070,
    2
   ],
   [
    14768,
    10
   ],
   [
    9628,
    2
   ],
   [
    7643,
    0
   ],
   [
    12633,
    3
   ],
   [
    9120,
    12
   ],
   [
    13498,
    1
   ],
   [
    5961,
    2
   ],
   [
    5676,
    11
   ],
   [
    12649,
    2
   ],
   [
    270,
    3
   ],
   [
    1819,
    6
   ],
   [
    13274,
    6
   ],
   [
    13800,
    2
   ],
   [
    11274,
    9
   ],
   [
    12434,
    4
   ],
   [
    19315,
    5
   ],
   [
    15442,
    1
   ],
   [
    16464,
    17
   ],
   [
    6239,
    3
   ],
   [
    53,
    5
   ],
   [
    5251,
    4
   ],
   [
    1489,
    6
   ],
   [
    16860,
    1
   ],
   [
    4699,
    3
   ],
   [
    13126,
    3
   ],
   [
    270,
    7
   ],
   [
    18800,
    3
   ],
   [
    9893,
    2
   ],
   [
    6759,
    0
   ],
   [
    2553,
    6
   ],
   [
    16128,
    2
   ],
   [
    10844,
    12
   ],
   [
    6385,
    1
   ],
   [
    6239,
    0
   ],
   [
    2302,
    2
   ],
   [
    5779,
    4
   ],
   [
    4190,
    1
   ],
   [
    19680,
    2
   ],
   [
    16584,
    1
   ],
   [
    10255,
    2
   ],
   [
    11890,
    1
   ],
   [
    11128,
    10
   ],
   [
    480,
    6
   ],
   [
    16379,
    4
   ],
   [
    10928,
    4
   ],
   [
    12182,
    12
   ],
   [
    13182,
    0
   ],
   [
    6844,
    0
   ],
   [
    16467,
    4
   ],
   [
    5276,
    4
   ],
   [
    190,
    10
   ],
   [
    19956,
    1
   ],
   [
    5741,
    0
   ],
   [
    18046,
    0
   ],
   [
    15749,
    1
   ],
   [
    8501,
    5
   ],
   [
    575,
    1
   ],
   [
    5794,
    3
   ],
   [
    6996,
    4
   ],
   [
    13522,
    5
   ],
   [
    19382,
    0
   ]
  ]
 },
 "gpu": "NVIDIA GeForce RTX 4090",
 "protocol": "Same-input generative baselines vs Laya-Android: latency and AndroidControl-High Type / Grounding.\n\npython scripts/bench_llm_baselines.py --out results/baselines/same_input_1000.json\nSubset: 1,000 steps sampled (seed 0) from the 8,444 official test steps; identical for every model.\nInput for every model: goal + last 3 gold actions + the same accessibility-derived candidates (src/serialization.py).\nLLMs (zero-shot, greedy, batch 1) must answer {\"operation\": ..., \"target\": index}. Laya-Android answers both questions\nin one forward pass. Latency = tokenization + model call (+ generation), CUDA-synchronized, after 20 warm-up steps.\nScoring: unmodified InfiGUI-R1 judge via scripts/evaluate_androidcontrol.to_reference.",
 "models": {
  "Laya-Android (322M)": {
   "params_b": 0.32,
   "type": 77.2,
   "grounding": 62.8,
   "unparseable": 0,
   "latency_ms": {
    "median": 38.7,
    "p95": 47.5,
    "mean": 39.8
   },
   "mean_generated_tokens": 0,
   "peak_vram_gb": 1.73
  },
  "ollama:qwen3.5:9b-fit": {
   "serving": "Ollama 0.34.0 (GGUF)",
   "params": "9.7B",
   "quantization": "Q4_K_M",
   "family": "qwen35",
   "type": 63.6,
   "grounding": 51.6,
   "unparseable": 9,
   "latency_ms": {
    "median": 228.6,
    "p95": 273.2,
    "mean": 231.2
   },
   "mean_generated_tokens": 14.0,
   "mean_input_tokens": 407.5
  },
  "ollama:gemma4:fit": {
   "serving": "Ollama 0.34.0 (GGUF)",
   "params": "8.0B",
   "quantization": "Q4_K_M",
   "family": "gemma4",
   "type": 57.4,
   "grounding": 45.0,
   "unparseable": 2,
   "latency_ms": {
    "median": 171.1,
    "p95": 202.2,
    "mean": 172.4
   },
   "mean_generated_tokens": 14.1,
   "mean_input_tokens": 408.8
  },
  "ollama:ministral-3:14b": {
   "serving": "Ollama 0.34.0 (GGUF)",
   "params": "13.9B",
   "quantization": "Q4_K_M",
   "family": "mistral3",
   "type": 53.2,
   "grounding": 42.0,
   "unparseable": 4,
   "latency_ms": {
    "median": 285.3,
    "p95": 424.2,
    "mean": 301.2
   },
   "mean_generated_tokens": 19.8,
   "mean_input_tokens": 401.3
  },
  "ollama:gemma4:26b": {
   "serving": "Ollama 0.34.0 (GGUF)",
   "params": "25.2B",
   "quantization": "Q4_K_M",
   "family": "gemma4",
   "type": 64.8,
   "grounding": 56.6,
   "unparseable": 2,
   "latency_ms": {
    "median": 201.1,
    "p95": 256.3,
    "mean": 211.9
   },
   "mean_generated_tokens": 14.1,
   "mean_input_tokens": 412.8
  },
  "ollama:qwen3.8:27b": {
   "serving": "Ollama 0.34.0 (GGUF)",
   "params": "27.3B",
   "quantization": "Q4_K_M",
   "family": "qwen35",
   "type": 76.8,
   "grounding": 64.5,
   "unparseable": 0,
   "latency_ms": {
    "median": 574.7,
    "p95": 693.9,
    "mean": 585.3
   },
   "mean_generated_tokens": 13.4,
   "mean_input_tokens": 407.5
  },
  "ollama:qwen3.5:0.8b": {
   "serving": "Ollama 0.34.0 (GGUF)",
   "params": "752.39M",
   "quantization": "Q8_0",
   "family": "qwen35",
   "type": 41.3,
   "grounding": 0.0,
   "unparseable": 5,
   "latency_ms": {
    "median": 124.5,
    "p95": 158.4,
    "mean": 127.6
   },
   "mean_generated_tokens": 15.0,
   "mean_input_tokens": 407.5,
   "think": false,
   "num_predict": 32
  },
  "ollama:qwen3.5:2b": {
   "serving": "Ollama 0.34.0 (GGUF)",
   "params": "1.9B",
   "quantization": "Q8_0",
   "family": "qwen35",
   "type": 57.4,
   "grounding": 24.5,
   "unparseable": 3,
   "latency_ms": {
    "median": 126.1,
    "p95": 143.8,
    "mean": 125.8
   },
   "mean_generated_tokens": 13.5,
   "mean_input_tokens": 407.5,
   "think": false,
   "num_predict": 32
  },
  "ollama:qwen3.5:4b": {
   "serving": "Ollama 0.34.0 (GGUF)",
   "params": "4.2B",
   "quantization": "Q4_K_M",
   "family": "qwen35",
   "type": 67.3,
   "grounding": 52.8,
   "unparseable": 0,
   "latency_ms": {
    "median": 192.9,
    "p95": 251.5,
    "mean": 197.8
   },
   "mean_generated_tokens": 15.6,
   "mean_input_tokens": 407.5,
   "think": false,
   "num_predict": 32
  },
  "ollama:gemma4:e2b": {
   "serving": "Ollama 0.34.0 (GGUF)",
   "params": "4.6B",
   "quantization": "Q4_K_M",
   "family": "gemma4",
   "type": 56.9,
   "grounding": 22.7,
   "unparseable": 2,
   "latency_ms": {
    "median": 115.6,
    "p95": 139.4,
    "mean": 116.9
   },
   "mean_generated_tokens": 15.4,
   "mean_input_tokens": 408.8,
   "think": false,
   "num_predict": 32
  },
  "ollama:gemma4:e4b": {
   "serving": "Ollama 0.34.0 (GGUF)",
   "params": "7.5B",
   "quantization": "Q4_K_M",
   "family": "gemma4",
   "type": 57.1,
   "grounding": 44.1,
   "unparseable": 3,
   "latency_ms": {
    "median": 145.7,
    "p95": 174.9,
    "mean": 146.7
   },
   "mean_generated_tokens": 14.1,
   "mean_input_tokens": 408.8,
   "think": false,
   "num_predict": 32
  },
  "Qwen/Qwen3.5-0.8B": {
   "serving": "vLLM 0.31.0 (Docker)",
   "type": 40.9,
   "grounding": 0.4,
   "unparseable": 4,
   "latency_ms": {
    "median": 125.3,
    "p95": 178.2,
    "mean": 131.4
   },
   "mean_generated_tokens": 14.6,
   "mean_input_tokens": 407.5,
   "gpu_memory_used_mb_incl_kv_cache_reservation": 21355
  },
  "Qwen/Qwen3.5-2B": {
   "serving": "vLLM 0.31.0 (Docker)",
   "type": 57.5,
   "grounding": 23.0,
   "unparseable": 4,
   "latency_ms": {
    "median": 149.4,
    "p95": 201.9,
    "mean": 154.6
   },
   "mean_generated_tokens": 13.2,
   "mean_input_tokens": 407.5,
   "gpu_memory_used_mb_incl_kv_cache_reservation": 20999
  },
  "Qwen/Qwen3.5-4B": {
   "serving": "vLLM 0.31.0 (Docker)",
   "type": 67.1,
   "grounding": 50.5,
   "unparseable": 0,
   "latency_ms": {
    "median": 242.5,
    "p95": 304.8,
    "mean": 250.2
   },
   "mean_generated_tokens": 15.4,
   "mean_input_tokens": 407.5,
   "gpu_memory_used_mb_incl_kv_cache_reservation": 21087
  },
  "google/gemma-4-E2B-it": {
   "serving": "vLLM 0.31.0 (Docker)",
   "type": 58.4,
   "grounding": 23.6,
   "unparseable": 4,
   "latency_ms": {
    "median": 154.7,
    "p95": 209.8,
    "mean": 159.3
   },
   "mean_generated_tokens": 15.4,
   "mean_input_tokens": 409.8,
   "gpu_memory_used_mb_incl_kv_cache_reservation": 22463
  }
 }
}