Hugging Face's logo

ByunByun
/
laya-android

Text Classification
Transformers
Safetensors
Laya
English
system-one
decision-model
android
mobile-agent
gui-agent
accessibility
androidcontrol
mmbert
Eval Results (legacy)
Model card Files
xet
 Community
laya-android / eval /same_input_1000.json
ByunByun's picture
ByunByun
v0.3.0: same-input comparison with generative LLMs and Jev
4b2c25b verified
Raw History Blame Contribute Delete
37.9 kB
{
"subset": {
"n": 1000,
"warmup": 20,
"seed": 0,
"keys": [
[
2666,
2
],
[
12648,
4
],
[
18360,
5
],
[
3715,
0
],
[
13502,
3
],
[
16562,
4
],
[
12123,
10
],
[
7509,
1
],
[
18645,
1
],
[
16951,
3
],
[
10058,
2
],
[
2269,
11
],
[
548,
0
],
[
3292,
0
],
[
15225,
2
],
[
54,
1
],
[
19300,
2
],
[
12754,
1
],
[
9168,
0
],
[
12499,
3
],
[
2288,
3
],
[
6903,
0
],
[
8188,
0
],
[
8917,
3
],
[
5288,
5
],
[
17180,
0
],
[
3224,
0
],
[
2856,
3
],
[
12289,
3
],
[
19833,
1
],
[
19170,
0
],
[
4158,
1
],
[
11571,
3
],
[
11274,
8
],
[
4674,
2
],
[
12743,
4
],
[
7455,
5
],
[
11187,
6
],
[
17085,
3
],
[
3238,
0
],
[
14735,
1
],
[
12176,
4
],
[
9120,
3
],
[
11242,
3
],
[
6623,
6
],
[
6792,
6
],
[
6707,
2
],
[
1082,
10
],
[
10033,
0
],
[
18619,
3
],
[
2488,
18
],
[
3152,
0
],
[
4833,
0
],
[
5563,
3
],
[
1309,
6
],
[
2836,
3
],
[
14985,
5
],
[
10726,
0
],
[
8871,
1
],
[
7866,
1
],
[
15935,
3
],
[
10724,
6
],
[
17325,
0
],
[
19269,
5
],
[
13651,
2
],
[
2916,
1
],
[
12483,
5
],
[
4406,
7
],
[
19003,
2
],
[
12885,
1
],
[
6871,
2
],
[
9131,
0
],
[
586,
4
],
[
10558,
4
],
[
4451,
3
],
[
8146,
3
],
[
14230,
3
],
[
6258,
1
],
[
12743,
1
],
[
16212,
3
],
[
2269,
9
],
[
3708,
2
],
[
5407,
3
],
[
7999,
1
],
[
1581,
3
],
[
2669,
2
],
[
916,
0
],
[
4670,
0
],
[
6768,
3
],
[
4560,
1
],
[
14985,
4
],
[
3224,
5
],
[
14157,
9
],
[
4414,
5
],
[
1206,
1
],
[
759,
4
],
[
6996,
6
],
[
6640,
5
],
[
4640,
1
],
[
18741,
3
],
[
7753,
1
],
[
2232,
2
],
[
803,
3
],
[
16208,
2
],
[
3751,
5
],
[
9963,
3
],
[
2553,
2
],
[
8158,
1
],
[
2609,
7
],
[
11551,
7
],
[
13440,
5
],
[
16614,
0
],
[
6472,
6
],
[
19676,
9
],
[
18141,
2
],
[
1330,
2
],
[
3747,
1
],
[
7249,
1
],
[
10037,
0
],
[
13669,
3
],
[
18302,
7
],
[
6236,
3
],
[
7456,
2
],
[
2093,
5
],
[
5791,
9
],
[
5926,
4
],
[
13182,
1
],
[
9628,
3
],
[
4451,
4
],
[
16924,
0
],
[
6405,
5
],
[
470,
2
],
[
18351,
0
],
[
15658,
1
],
[
19880,
2
],
[
11890,
2
],
[
13619,
5
],
[
14928,
1
],
[
9634,
2
],
[
5680,
2
],
[
406,
10
],
[
17629,
4
],
[
2777,
6
],
[
12904,
0
],
[
1586,
1
],
[
10966,
0
],
[
5016,
0
],
[
8982,
11
],
[
18883,
3
],
[
13470,
1
],
[
11198,
1
],
[
13674,
3
],
[
4916,
7
],
[
11870,
2
],
[
14843,
0
],
[
15775,
3
],
[
2860,
3
],
[
55,
5
],
[
6919,
0
],
[
12823,
5
],
[
5852,
2
],
[
8963,
3
],
[
8202,
2
],
[
17180,
3
],
[
14454,
1
],
[
1049,
5
],
[
15405,
0
],
[
15890,
4
],
[
1632,
5
],
[
6119,
1
],
[
17098,
4
],
[
2302,
12
],
[
9935,
9
],
[
5791,
0
],
[
17114,
0
],
[
19019,
4
],
[
27,
1
],
[
1311,
0
],
[
19315,
1
],
[
12514,
4
],
[
11912,
0
],
[
18124,
2
],
[
1741,
6
],
[
15802,
0
],
[
6760,
1
],
[
2991,
0
],
[
4833,
5
],
[
569,
4
],
[
15386,
1
],
[
15865,
2
],
[
122,
5
],
[
7826,
8
],
[
548,
4
],
[
62,
10
],
[
3507,
4
],
[
6876,
0
],
[
4545,
3
],
[
7204,
3
],
[
11622,
8
],
[
10925,
4
],
[
6574,
0
],
[
3700,
2
],
[
18577,
0
],
[
15100,
8
],
[
2886,
1
],
[
777,
2
],
[
10711,
6
],
[
17452,
0
],
[
4414,
0
],
[
9823,
8
],
[
4976,
3
],
[
13355,
1
],
[
4406,
3
],
[
5707,
3
],
[
10844,
7
],
[
662,
0
],
[
1464,
2
],
[
1364,
0
],
[
7565,
0
],
[
9961,
2
],
[
12045,
6
],
[
13970,
3
],
[
1438,
4
],
[
19300,
17
],
[
17661,
3
],
[
16593,
2
],
[
14251,
2
],
[
6443,
4
],
[
7608,
8
],
[
14307,
2
],
[
11274,
7
],
[
270,
0
],
[
5199,
1
],
[
5611,
4
],
[
10559,
4
],
[
12758,
0
],
[
12926,
2
],
[
13975,
3
],
[
3315,
2
],
[
12933,
3
],
[
1198,
0
],
[
1378,
1
],
[
10501,
0
],
[
5990,
4
],
[
5563,
1
],
[
11234,
4
],
[
13731,
2
],
[
15063,
0
],
[
4812,
1
],
[
11326,
1
],
[
4406,
0
],
[
18679,
0
],
[
8982,
4
],
[
1704,
0
],
[
11831,
4
],
[
6463,
0
],
[
2582,
6
],
[
11623,
3
],
[
15425,
2
],
[
12624,
5
],
[
11501,
1
],
[
15775,
8
],
[
4114,
3
],
[
3592,
2
],
[
18828,
5
],
[
18521,
3
],
[
12904,
19
],
[
13247,
2
],
[
4662,
2
],
[
18741,
1
],
[
4414,
4
],
[
19453,
4
],
[
16259,
0
],
[
1302,
0
],
[
11622,
0
],
[
12863,
0
],
[
5741,
6
],
[
6153,
0
],
[
3031,
2
],
[
2405,
3
],
[
3000,
10
],
[
7195,
3
],
[
8158,
4
],
[
2232,
3
],
[
14723,
8
],
[
229,
3
],
[
3519,
3
],
[
15048,
1
],
[
11234,
9
],
[
17181,
3
],
[
19038,
5
],
[
7935,
6
],
[
16072,
3
],
[
2991,
3
],
[
14103,
2
],
[
8084,
1
],
[
10058,
9
],
[
6153,
1
],
[
16464,
21
],
[
6904,
12
],
[
13669,
2
],
[
4406,
2
],
[
937,
6
],
[
17402,
8
],
[
7379,
1
],
[
19382,
2
],
[
15204,
1
],
[
7607,
2
],
[
1458,
0
],
[
7884,
1
],
[
3833,
0
],
[
7195,
2
],
[
17647,
4
],
[
14390,
3
],
[
13745,
5
],
[
5636,
3
],
[
3833,
2
],
[
19033,
1
],
[
5459,
5
],
[
15527,
4
],
[
16101,
2
],
[
19347,
0
],
[
12429,
0
],
[
19510,
1
],
[
19537,
0
],
[
7382,
2
],
[
296,
0
],
[
13070,
1
],
[
12223,
0
],
[
12364,
1
],
[
1194,
2
],
[
5459,
4
],
[
9823,
14
],
[
5742,
2
],
[
14454,
7
],
[
11379,
1
],
[
18302,
10
],
[
2406,
3
],
[
3000,
9
],
[
8282,
2
],
[
1360,
6
],
[
11550,
0
],
[
569,
13
],
[
17181,
4
],
[
12649,
1
],
[
5899,
0
],
[
5497,
4
],
[
17842,
1
],
[
14212,
4
],
[
19719,
4
],
[
14607,
1
],
[
19657,
5
],
[
1089,
6
],
[
3195,
1
],
[
2748,
2
],
[
16228,
1
],
[
7585,
0
],
[
11234,
6
],
[
15886,
1
],
[
18894,
0
],
[
14928,
0
],
[
8770,
1
],
[
723,
0
],
[
27,
3
],
[
6562,
9
],
[
11623,
0
],
[
19812,
4
],
[
9793,
2
],
[
12743,
3
],
[
2405,
0
],
[
19300,
1
],
[
10081,
2
],
[
11632,
2
],
[
15613,
1
],
[
14681,
2
],
[
14654,
1
],
[
5990,
3
],
[
4782,
0
],
[
8939,
1
],
[
11185,
10
],
[
12816,
4
],
[
1969,
3
],
[
1198,
4
],
[
18875,
2
],
[
15886,
2
],
[
5251,
6
],
[
19268,
2
],
[
2889,
1
],
[
5636,
4
],
[
13490,
4
],
[
15718,
1
],
[
1193,
2
],
[
18087,
4
],
[
14774,
4
],
[
17677,
0
],
[
1654,
0
],
[
18306,
4
],
[
719,
4
],
[
1082,
1
],
[
4943,
0
],
[
12479,
0
],
[
3845,
2
],
[
13349,
8
],
[
7020,
0
],
[
14651,
1
],
[
19197,
1
],
[
4195,
4
],
[
2195,
4
],
[
18141,
3
],
[
12926,
8
],
[
11417,
3
],
[
4745,
4
],
[
14804,
0
],
[
11332,
2
],
[
4608,
3
],
[
6792,
2
],
[
1306,
3
],
[
15017,
1
],
[
17065,
2
],
[
14220,
0
],
[
17533,
2
],
[
13605,
4
],
[
2685,
0
],
[
1542,
0
],
[
1359,
1
],
[
18991,
1
],
[
9804,
8
],
[
907,
11
],
[
7884,
2
],
[
8537,
17
],
[
3297,
0
],
[
15966,
9
],
[
19813,
2
],
[
11726,
10
],
[
4297,
5
],
[
5373,
6
],
[
16212,
1
],
[
16002,
2
],
[
2991,
9
],
[
15832,
1
],
[
2288,
2
],
[
3591,
4
],
[
15802,
1
],
[
5742,
6
],
[
1045,
8
],
[
17127,
2
],
[
16464,
18
],
[
15857,
7
],
[
1027,
2
],
[
19375,
1
],
[
12483,
8
],
[
9753,
0
],
[
2775,
3
],
[
13490,
0
],
[
2553,
8
],
[
4608,
0
],
[
13674,
4
],
[
1024,
3
],
[
13314,
5
],
[
13390,
2
],
[
6430,
0
],
[
300,
0
],
[
8658,
3
],
[
13886,
1
],
[
2582,
0
],
[
5342,
0
],
[
7609,
0
],
[
122,
2
],
[
7509,
6
],
[
4635,
6
],
[
228,
1
],
[
11306,
3
],
[
14114,
4
],
[
898,
1
],
[
8738,
5
],
[
5276,
7
],
[
6710,
1
],
[
17479,
3
],
[
4244,
1
],
[
18619,
6
],
[
13274,
4
],
[
9921,
3
],
[
4812,
9
],
[
937,
11
],
[
7614,
0
],
[
13752,
2
],
[
12856,
4
],
[
18488,
0
],
[
11304,
4
],
[
12571,
3
],
[
6638,
1
],
[
2860,
4
],
[
3770,
0
],
[
11831,
2
],
[
5773,
3
],
[
14321,
5
],
[
4680,
4
],
[
8197,
0
],
[
12123,
5
],
[
19877,
2
],
[
9128,
1
],
[
8885,
2
],
[
6623,
7
],
[
15966,
3
],
[
1624,
0
],
[
4916,
5
],
[
723,
1
],
[
2771,
8
],
[
2630,
1
],
[
4916,
2
],
[
15966,
11
],
[
11501,
4
],
[
15857,
3
],
[
5288,
2
],
[
16004,
0
],
[
11488,
1
],
[
13526,
1
],
[
3000,
8
],
[
9394,
4
],
[
17085,
0
],
[
14129,
0
],
[
2093,
1
],
[
14321,
2
],
[
15613,
10
],
[
231,
5
],
[
15857,
10
],
[
12303,
3
],
[
16837,
2
],
[
7472,
4
],
[
14220,
1
],
[
11306,
4
],
[
3198,
2
],
[
6666,
1
],
[
4114,
2
],
[
10786,
2
],
[
4226,
6
],
[
5684,
7
],
[
17111,
1
],
[
15225,
1
],
[
6666,
0
],
[
15999,
0
],
[
16467,
5
],
[
6405,
3
],
[
9394,
2
],
[
17457,
0
],
[
13072,
0
],
[
5288,
6
],
[
13554,
1
],
[
17900,
6
],
[
3026,
8
],
[
18938,
1
],
[
7456,
6
],
[
11390,
4
],
[
62,
2
],
[
17237,
0
],
[
17880,
7
],
[
229,
2
],
[
7999,
0
],
[
11490,
5
],
[
4358,
1
],
[
11559,
0
],
[
5773,
0
],
[
16140,
0
],
[
18335,
4
],
[
3242,
5
],
[
19453,
3
],
[
8730,
3
],
[
15476,
8
],
[
10925,
5
],
[
4602,
2
],
[
10531,
0
],
[
1360,
5
],
[
27,
4
],
[
9823,
9
],
[
15204,
8
],
[
15100,
0
],
[
17065,
3
],
[
3761,
0
],
[
13502,
4
],
[
11077,
0
],
[
7045,
1
],
[
3010,
0
],
[
1194,
1
],
[
2553,
10
],
[
10137,
0
],
[
11728,
1
],
[
13061,
0
],
[
4535,
1
],
[
9464,
0
],
[
5973,
2
],
[
2488,
2
],
[
15778,
0
],
[
11234,
7
],
[
11040,
9
],
[
4979,
2
],
[
7733,
5
],
[
15703,
2
],
[
15438,
2
],
[
10844,
10
],
[
11296,
10
],
[
16924,
1
],
[
14220,
4
],
[
5154,
0
],
[
5776,
5
],
[
4635,
8
],
[
4581,
3
],
[
14556,
3
],
[
15349,
2
],
[
18182,
0
],
[
5215,
1
],
[
11494,
2
],
[
13512,
2
],
[
18438,
1
],
[
7969,
1
],
[
19061,
3
],
[
19636,
1
],
[
19269,
6
],
[
2200,
0
],
[
17037,
1
],
[
11549,
2
],
[
5312,
2
],
[
19347,
2
],
[
1842,
2
],
[
7878,
3
],
[
904,
7
],
[
18340,
1
],
[
14985,
0
],
[
300,
2
],
[
2412,
0
],
[
2886,
2
],
[
15070,
0
],
[
219,
0
],
[
13735,
2
],
[
1378,
0
],
[
4422,
1
],
[
136,
0
],
[
10531,
5
],
[
11304,
2
],
[
8455,
2
],
[
5251,
2
],
[
11198,
2
],
[
6904,
0
],
[
3845,
4
],
[
16562,
1
],
[
17842,
4
],
[
14681,
1
],
[
6207,
8
],
[
12649,
3
],
[
16588,
4
],
[
5455,
5
],
[
17154,
2
],
[
5446,
0
],
[
12123,
8
],
[
4806,
6
],
[
7633,
6
],
[
6707,
3
],
[
17037,
3
],
[
13428,
7
],
[
14928,
5
],
[
16275,
0
],
[
19268,
4
],
[
14942,
3
],
[
8084,
2
],
[
7045,
4
],
[
16760,
1
],
[
7499,
3
],
[
1741,
5
],
[
14843,
5
],
[
1089,
1
],
[
8780,
3
],
[
13805,
3
],
[
2078,
0
],
[
6353,
2
],
[
8740,
1
],
[
11482,
2
],
[
3026,
10
],
[
19979,
5
],
[
11128,
2
],
[
13497,
3
],
[
15746,
0
],
[
17647,
2
],
[
1908,
1
],
[
16464,
11
],
[
17510,
0
],
[
19190,
3
],
[
9793,
6
],
[
18577,
1
],
[
7869,
1
],
[
12904,
21
],
[
10255,
4
],
[
1464,
0
],
[
1501,
0
],
[
1842,
5
],
[
5968,
0
],
[
13437,
0
],
[
136,
2
],
[
228,
0
],
[
5241,
4
],
[
2302,
9
],
[
16324,
0
],
[
8193,
0
],
[
15100,
5
],
[
8158,
3
],
[
17457,
4
],
[
6930,
4
],
[
13019,
1
],
[
3761,
5
],
[
3023,
2
],
[
12245,
0
],
[
12434,
2
],
[
17538,
2
],
[
12499,
2
],
[
9806,
1
],
[
971,
0
],
[
1512,
0
],
[
6844,
3
],
[
14107,
3
],
[
2836,
1
],
[
7682,
2
],
[
13327,
1
],
[
6759,
2
],
[
7379,
0
],
[
9683,
0
],
[
11571,
1
],
[
11912,
3
],
[
14723,
6
],
[
9793,
5
],
[
18883,
7
],
[
13248,
1
],
[
8982,
8
],
[
1512,
3
],
[
11728,
4
],
[
2612,
4
],
[
270,
5
],
[
19339,
5
],
[
16696,
4
],
[
1672,
2
],
[
15749,
9
],
[
17841,
5
],
[
16768,
2
],
[
4535,
0
],
[
2886,
3
],
[
8998,
5
],
[
3559,
6
],
[
5684,
5
],
[
15768,
14
],
[
7826,
7
],
[
16829,
0
],
[
2757,
1
],
[
16259,
6
],
[
15055,
5
],
[
1330,
1
],
[
6505,
0
],
[
9467,
0
],
[
19190,
0
],
[
8084,
0
],
[
4796,
1
],
[
10911,
0
],
[
13468,
12
],
[
12248,
3
],
[
16584,
4
],
[
4045,
0
],
[
11132,
4
],
[
7337,
0
],
[
11429,
3
],
[
16860,
0
],
[
20097,
2
],
[
17900,
0
],
[
10058,
11
],
[
10608,
2
],
[
8725,
3
],
[
591,
0
],
[
4545,
0
],
[
3566,
2
],
[
6338,
4
],
[
15775,
7
],
[
9431,
1
],
[
7969,
2
],
[
11128,
4
],
[
20189,
6
],
[
16353,
2
],
[
4618,
0
],
[
10608,
1
],
[
4535,
2
],
[
13669,
9
],
[
8537,
16
],
[
11040,
0
],
[
8165,
6
],
[
2387,
2
],
[
11829,
2
],
[
12581,
2
],
[
8770,
2
],
[
14265,
4
],
[
18800,
0
],
[
11185,
6
],
[
6304,
5
],
[
5147,
1
],
[
569,
16
],
[
19719,
2
],
[
12614,
2
],
[
13970,
4
],
[
904,
3
],
[
4812,
4
],
[
15076,
0
],
[
5732,
5
],
[
6418,
11
],
[
19939,
4
],
[
2748,
3
],
[
5025,
0
],
[
7587,
2
],
[
19375,
3
],
[
8826,
9
],
[
4916,
3
],
[
8770,
3
],
[
14723,
5
],
[
13498,
3
],
[
4936,
2
],
[
4090,
1
],
[
907,
1
],
[
13669,
4
],
[
19088,
1
],
[
17538,
0
],
[
11843,
1
],
[
407,
0
],
[
5968,
3
],
[
19315,
3
],
[
18828,
1
],
[
11972,
0
],
[
2777,
1
],
[
9935,
8
],
[
5147,
3
],
[
12163,
2
],
[
11296,
1
],
[
14681,
0
],
[
2146,
0
],
[
7633,
2
],
[
1309,
0
],
[
12076,
0
],
[
9466,
1
],
[
13217,
4
],
[
16837,
0
],
[
8333,
2
],
[
10033,
1
],
[
13256,
0
],
[
5956,
2
],
[
11726,
3
],
[
591,
7
],
[
13605,
1
],
[
2098,
1
],
[
5636,
1
],
[
13498,
2
],
[
777,
5
],
[
19242,
2
],
[
2250,
1
],
[
898,
0
],
[
9072,
1
],
[
1569,
0
],
[
8333,
0
],
[
12546,
2
],
[
2426,
0
],
[
2269,
13
],
[
13307,
5
],
[
16035,
1
],
[
5029,
0
],
[
7903,
4
],
[
17207,
2
],
[
16588,
3
],
[
5278,
2
],
[
11900,
4
],
[
6418,
13
],
[
12624,
10
],
[
15613,
11
],
[
14620,
3
],
[
282,
4
],
[
15646,
0
],
[
10167,
6
],
[
17953,
0
],
[
1438,
3
],
[
4618,
13
],
[
15613,
15
],
[
14942,
15
],
[
6304,
4
],
[
122,
0
],
[
19572,
4
],
[
5185,
2
],
[
20097,
1
],
[
5459,
1
],
[
2814,
5
],
[
12648,
2
],
[
8898,
3
],
[
6418,
3
],
[
9318,
5
],
[
6207,
18
],
[
6207,
9
],
[
2775,
2
],
[
16259,
10
],
[
3793,
6
],
[
17628,
2
],
[
5564,
1
],
[
1359,
2
],
[
9747,
3
],
[
14321,
3
],
[
937,
0
],
[
1206,
0
],
[
19366,
3
],
[
3148,
2
],
[
13669,
0
],
[
11304,
6
],
[
5611,
0
],
[
17629,
6
],
[
8871,
12
],
[
19813,
7
],
[
13560,
0
],
[
5956,
1
],
[
15457,
4
],
[
12926,
4
],
[
10527,
2
],
[
19269,
0
],
[
15020,
5
],
[
569,
5
],
[
11885,
2
],
[
11198,
7
],
[
18231,
1
],
[
1170,
4
],
[
10098,
0
],
[
17563,
0
],
[
15063,
4
],
[
4579,
0
],
[
15442,
3
],
[
13349,
7
],
[
19349,
5
],
[
1819,
5
],
[
723,
3
],
[
10672,
1
],
[
1089,
17
],
[
9783,
4
],
[
11234,
8
],
[
7607,
0
],
[
13070,
2
],
[
14768,
10
],
[
9628,
2
],
[
7643,
0
],
[
12633,
3
],
[
9120,
12
],
[
13498,
1
],
[
5961,
2
],
[
5676,
11
],
[
12649,
2
],
[
270,
3
],
[
1819,
6
],
[
13274,
6
],
[
13800,
2
],
[
11274,
9
],
[
12434,
4
],
[
19315,
5
],
[
15442,
1
],
[
16464,
17
],
[
6239,
3
],
[
53,
5
],
[
5251,
4
],
[
1489,
6
],
[
16860,
1
],
[
4699,
3
],
[
13126,
3
],
[
270,
7
],
[
18800,
3
],
[
9893,
2
],
[
6759,
0
],
[
2553,
6
],
[
16128,
2
],
[
10844,
12
],
[
6385,
1
],
[
6239,
0
],
[
2302,
2
],
[
5779,
4
],
[
4190,
1
],
[
19680,
2
],
[
16584,
1
],
[
10255,
2
],
[
11890,
1
],
[
11128,
10
],
[
480,
6
],
[
16379,
4
],
[
10928,
4
],
[
12182,
12
],
[
13182,
0
],
[
6844,
0
],
[
16467,
4
],
[
5276,
4
],
[
190,
10
],
[
19956,
1
],
[
5741,
0
],
[
18046,
0
],
[
15749,
1
],
[
8501,
5
],
[
575,
1
],
[
5794,
3
],
[
6996,
4
],
[
13522,
5
],
[
19382,
0
]
]
},
"gpu": "NVIDIA GeForce RTX 4090",
"protocol": "Same-input generative baselines vs Laya-Android: latency and AndroidControl-High Type / Grounding.\n\npython scripts/bench_llm_baselines.py --out results/baselines/same_input_1000.json\nSubset: 1,000 steps sampled (seed 0) from the 8,444 official test steps; identical for every model.\nInput for every model: goal + last 3 gold actions + the same accessibility-derived candidates (src/serialization.py).\nLLMs (zero-shot, greedy, batch 1) must answer {\"operation\": ..., \"target\": index}. Laya-Android answers both questions\nin one forward pass. Latency = tokenization + model call (+ generation), CUDA-synchronized, after 20 warm-up steps.\nScoring: unmodified InfiGUI-R1 judge via scripts/evaluate_androidcontrol.to_reference.",
"models": {
"Laya-Android (322M)": {
"params_b": 0.32,
"type": 77.2,
"grounding": 62.8,
"unparseable": 0,
"latency_ms": {
"median": 38.7,
"p95": 47.5,
"mean": 39.8
},
"mean_generated_tokens": 0,
"peak_vram_gb": 1.73
},
"ollama:qwen3.5:9b-fit": {
"serving": "Ollama 0.34.0 (GGUF)",
"params": "9.7B",
"quantization": "Q4_K_M",
"family": "qwen35",
"type": 63.6,
"grounding": 51.6,
"unparseable": 9,
"latency_ms": {
"median": 228.6,
"p95": 273.2,
"mean": 231.2
},
"mean_generated_tokens": 14.0,
"mean_input_tokens": 407.5
},
"ollama:gemma4:fit": {
"serving": "Ollama 0.34.0 (GGUF)",
"params": "8.0B",
"quantization": "Q4_K_M",
"family": "gemma4",
"type": 57.4,
"grounding": 45.0,
"unparseable": 2,
"latency_ms": {
"median": 171.1,
"p95": 202.2,
"mean": 172.4
},
"mean_generated_tokens": 14.1,
"mean_input_tokens": 408.8
},
"ollama:ministral-3:14b": {
"serving": "Ollama 0.34.0 (GGUF)",
"params": "13.9B",
"quantization": "Q4_K_M",
"family": "mistral3",
"type": 53.2,
"grounding": 42.0,
"unparseable": 4,
"latency_ms": {
"median": 285.3,
"p95": 424.2,
"mean": 301.2
},
"mean_generated_tokens": 19.8,
"mean_input_tokens": 401.3
},
"ollama:gemma4:26b": {
"serving": "Ollama 0.34.0 (GGUF)",
"params": "25.2B",
"quantization": "Q4_K_M",
"family": "gemma4",
"type": 64.8,
"grounding": 56.6,
"unparseable": 2,
"latency_ms": {
"median": 201.1,
"p95": 256.3,
"mean": 211.9
},
"mean_generated_tokens": 14.1,
"mean_input_tokens": 412.8
},
"ollama:qwen3.8:27b": {
"serving": "Ollama 0.34.0 (GGUF)",
"params": "27.3B",
"quantization": "Q4_K_M",
"family": "qwen35",
"type": 76.8,
"grounding": 64.5,
"unparseable": 0,
"latency_ms": {
"median": 574.7,
"p95": 693.9,
"mean": 585.3
},
"mean_generated_tokens": 13.4,
"mean_input_tokens": 407.5
},
"ollama:qwen3.5:0.8b": {
"serving": "Ollama 0.34.0 (GGUF)",
"params": "752.39M",
"quantization": "Q8_0",
"family": "qwen35",
"type": 41.3,
"grounding": 0.0,
"unparseable": 5,
"latency_ms": {
"median": 124.5,
"p95": 158.4,
"mean": 127.6
},
"mean_generated_tokens": 15.0,
"mean_input_tokens": 407.5,
"think": false,
"num_predict": 32
},
"ollama:qwen3.5:2b": {
"serving": "Ollama 0.34.0 (GGUF)",
"params": "1.9B",
"quantization": "Q8_0",
"family": "qwen35",
"type": 57.4,
"grounding": 24.5,
"unparseable": 3,
"latency_ms": {
"median": 126.1,
"p95": 143.8,
"mean": 125.8
},
"mean_generated_tokens": 13.5,
"mean_input_tokens": 407.5,
"think": false,
"num_predict": 32
},
"ollama:qwen3.5:4b": {
"serving": "Ollama 0.34.0 (GGUF)",
"params": "4.2B",
"quantization": "Q4_K_M",
"family": "qwen35",
"type": 67.3,
"grounding": 52.8,
"unparseable": 0,
"latency_ms": {
"median": 192.9,
"p95": 251.5,
"mean": 197.8
},
"mean_generated_tokens": 15.6,
"mean_input_tokens": 407.5,
"think": false,
"num_predict": 32
},
"ollama:gemma4:e2b": {
"serving": "Ollama 0.34.0 (GGUF)",
"params": "4.6B",
"quantization": "Q4_K_M",
"family": "gemma4",
"type": 56.9,
"grounding": 22.7,
"unparseable": 2,
"latency_ms": {
"median": 115.6,
"p95": 139.4,
"mean": 116.9
},
"mean_generated_tokens": 15.4,
"mean_input_tokens": 408.8,
"think": false,
"num_predict": 32
},
"ollama:gemma4:e4b": {
"serving": "Ollama 0.34.0 (GGUF)",
"params": "7.5B",
"quantization": "Q4_K_M",
"family": "gemma4",
"type": 57.1,
"grounding": 44.1,
"unparseable": 3,
"latency_ms": {
"median": 145.7,
"p95": 174.9,
"mean": 146.7
},
"mean_generated_tokens": 14.1,
"mean_input_tokens": 408.8,
"think": false,
"num_predict": 32
},
"Qwen/Qwen3.5-0.8B": {
"serving": "vLLM 0.31.0 (Docker)",
"type": 40.9,
"grounding": 0.4,
"unparseable": 4,
"latency_ms": {
"median": 125.3,
"p95": 178.2,
"mean": 131.4
},
"mean_generated_tokens": 14.6,
"mean_input_tokens": 407.5,
"gpu_memory_used_mb_incl_kv_cache_reservation": 21355
},
"Qwen/Qwen3.5-2B": {
"serving": "vLLM 0.31.0 (Docker)",
"type": 57.5,
"grounding": 23.0,
"unparseable": 4,
"latency_ms": {
"median": 149.4,
"p95": 201.9,
"mean": 154.6
},
"mean_generated_tokens": 13.2,
"mean_input_tokens": 407.5,
"gpu_memory_used_mb_incl_kv_cache_reservation": 20999
},
"Qwen/Qwen3.5-4B": {
"serving": "vLLM 0.31.0 (Docker)",
"type": 67.1,
"grounding": 50.5,
"unparseable": 0,
"latency_ms": {
"median": 242.5,
"p95": 304.8,
"mean": 250.2
},
"mean_generated_tokens": 15.4,
"mean_input_tokens": 407.5,
"gpu_memory_used_mb_incl_kv_cache_reservation": 21087
},
"google/gemma-4-E2B-it": {
"serving": "vLLM 0.31.0 (Docker)",
"type": 58.4,
"grounding": 23.6,
"unparseable": 4,
"latency_ms": {
"median": 154.7,
"p95": 209.8,
"mean": 159.3
},
"mean_generated_tokens": 15.4,
"mean_input_tokens": 409.8,
"gpu_memory_used_mb_incl_kv_cache_reservation": 22463
}
}
}