{
 "model": "ByunByun/laya-android",
 "protocol": "docs/eval_protocol.md",
 "protocol_commit": "41f0a3391045bd513f02b59fd467d39f43887981",
 "code_commit": "41f0a3391045bd513f02b59fd467d39f43887981",
 "reference_evaluator": "InfiGUI-R1@a4fca17 evaluate_android_control_action",
 "full_sr_note": "N/A: v0 does not generate INPUT_TEXT text or OPEN_APP app names",
 "androidcontrol_low": "not evaluated (v0 trained with high-level goal only)",
 "androidcontrol_high": {
  "all": {
   "steps": 8444,
   "episodes": 1543,
   "type": 29.04,
   "grounding": 11.81,
   "gold_clicks": 5074,
   "full_sr": null
  },
  "test_idd": {
   "steps": 3897,
   "episodes": 721,
   "type": 31.15,
   "grounding": 11.82,
   "gold_clicks": 2445,
   "full_sr": null
  },
  "test_app_unseen": {
   "steps": 3475,
   "episodes": 631,
   "type": 29.7,
   "grounding": 11.86,
   "gold_clicks": 2024,
   "full_sr": null
  },
  "test_task_unseen": {
   "steps": 4464,
   "episodes": 803,
   "type": 27.31,
   "grounding": 11.77,
   "gold_clicks": 2583,
   "full_sr": null
  },
  "test_category_unseen": {
   "steps": 3891,
   "episodes": 700,
   "type": 28.58,
   "grounding": 11.78,
   "gold_clicks": 2241,
   "full_sr": null
  }
 },
 "internal": {
  "temperature": [
   1.0,
   1.0,
   1.0
  ],
  "splits": {
   "test_idd": {
    "steps": 4618,
    "operation_accuracy": 0.2737,
    "target_top1_accuracy": 0.122,
    "joint_action_accuracy": 0.1153,
    "grounding_coverage": 0.9523,
    "ece_operation": 0.5465,
    "brier_operation": 1.2337,
    "ece_target": 0.3893,
    "brier_target": 1.1383,
    "latency_ms_median": 40.1,
    "latency_ms_p95": 44.07,
    "target_top1_by_candidates": {
     "1-5": {
      "acc": 0.3346,
      "n": 254
     },
     "6-10": {
      "acc": 0.1496,
      "n": 421
     },
     "11-20": {
      "acc": 0.1065,
      "n": 901
     },
     "21-50": {
      "acc": 0.0563,
      "n": 693
     },
     ">50": {
      "acc": 0.0294,
      "n": 68
     }
    },
    "operation_macro_f1": 0.1647,
    "op_per_class": {
     "CLICK": {
      "precision": 0.5315,
      "recall": 0.3481,
      "f1": 0.4207,
      "n_gold": 2445,
      "n_pred": 1601
     },
     "LONG_PRESS": {
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "n_gold": 9,
      "n_pred": 18
     },
     "SCROLL_UP": {
      "precision": 0.2083,
      "recall": 0.4615,
      "f1": 0.2871,
      "n_gold": 65,
      "n_pred": 144
     },
     "SCROLL_DOWN": {
      "precision": 0.506,
      "recall": 0.3058,
      "f1": 0.3812,
      "n_gold": 412,
      "n_pred": 249
     },
     "SCROLL_LEFT": {
      "precision": 0.1429,
      "recall": 0.0667,
      "f1": 0.0909,
      "n_gold": 15,
      "n_pred": 7
     },
     "SCROLL_RIGHT": {
      "precision": 0.375,
      "recall": 0.1667,
      "f1": 0.2308,
      "n_gold": 18,
      "n_pred": 8
     },
     "OPEN_APP": {
      "precision": 0.0923,
      "recall": 0.4766,
      "f1": 0.1546,
      "n_gold": 256,
      "n_pred": 1322
     },
     "INPUT_TEXT": {
      "precision": 0.0647,
      "recall": 0.1045,
      "f1": 0.0799,
      "n_gold": 287,
      "n_pred": 464
     },
     "BACK": {
      "precision": 0.1596,
      "recall": 0.2564,
      "f1": 0.1967,
      "n_gold": 117,
      "n_pred": 188
     },
     "HOME": {
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "n_gold": 0,
      "n_pred": 16
     },
     "WAIT": {
      "precision": 0.1515,
      "recall": 0.0183,
      "f1": 0.0327,
      "n_gold": 273,
      "n_pred": 33
     },
     "DONE": {
      "precision": 0.1162,
      "recall": 0.0915,
      "f1": 0.1024,
      "n_gold": 721,
      "n_pred": 568
     }
    },
    "unfit_items": {}
   },
   "test_app_unseen": {
    "steps": 4106,
    "operation_accuracy": 0.2504,
    "target_top1_accuracy": 0.143,
    "joint_action_accuracy": 0.1116,
    "grounding_coverage": 0.9328,
    "ece_operation": 0.572,
    "brier_operation": 1.2737,
    "ece_target": 0.3491,
    "brier_target": 1.0841,
    "latency_ms_median": 39.67,
    "latency_ms_p95": 43.64,
    "target_top1_by_candidates": {
     "1-5": {
      "acc": 0.5024,
      "n": 205
     },
     "6-10": {
      "acc": 0.1053,
      "n": 361
     },
     "11-20": {
      "acc": 0.1016,
      "n": 886
     },
     "21-50": {
      "acc": 0.0951,
      "n": 410
     },
     ">50": {
      "acc": 0.0,
      "n": 26
     }
    },
    "operation_macro_f1": 0.1521,
    "op_per_class": {
     "CLICK": {
      "precision": 0.5088,
      "recall": 0.3434,
      "f1": 0.41,
      "n_gold": 2024,
      "n_pred": 1366
     },
     "LONG_PRESS": {
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "n_gold": 0,
      "n_pred": 19
     },
     "SCROLL_UP": {
      "precision": 0.0179,
      "recall": 0.0909,
      "f1": 0.0299,
      "n_gold": 22,
      "n_pred": 112
     },
     "SCROLL_DOWN": {
      "precision": 0.3576,
      "recall": 0.1425,
      "f1": 0.2038,
      "n_gold": 414,
      "n_pred": 165
     },
     "SCROLL_LEFT": {
      "precision": 0.1333,
      "recall": 0.0769,
      "f1": 0.0976,
      "n_gold": 26,
      "n_pred": 15
     },
     "SCROLL_RIGHT": {
      "precision": 0.6379,
      "recall": 0.5606,
      "f1": 0.5968,
      "n_gold": 66,
      "n_pred": 58
     },
     "OPEN_APP": {
      "precision": 0.1074,
      "recall": 0.5761,
      "f1": 0.1811,
      "n_gold": 243,
      "n_pred": 1303
     },
     "INPUT_TEXT": {
      "precision": 0.0086,
      "recall": 0.0123,
      "f1": 0.0101,
      "n_gold": 243,
      "n_pred": 350
     },
     "BACK": {
      "precision": 0.1827,
      "recall": 0.2796,
      "f1": 0.221,
      "n_gold": 211,
      "n_pred": 323
     },
     "HOME": {
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "n_gold": 0,
      "n_pred": 27
     },
     "WAIT": {
      "precision": 0.0909,
      "recall": 0.0088,
      "f1": 0.0161,
      "n_gold": 226,
      "n_pred": 22
     },
     "DONE": {
      "precision": 0.0838,
      "recall": 0.046,
      "f1": 0.0594,
      "n_gold": 631,
      "n_pred": 346
     }
    },
    "unfit_items": {}
   },
   "test_task_unseen": {
    "steps": 5267,
    "operation_accuracy": 0.2326,
    "target_top1_accuracy": 0.1439,
    "joint_action_accuracy": 0.1076,
    "grounding_coverage": 0.9226,
    "ece_operation": 0.589,
    "brier_operation": 1.3054,
    "ece_target": 0.3585,
    "brier_target": 1.088,
    "latency_ms_median": 39.79,
    "latency_ms_p95": 42.93,
    "target_top1_by_candidates": {
     "1-5": {
      "acc": 0.4437,
      "n": 293
     },
     "6-10": {
      "acc": 0.127,
      "n": 441
     },
     "11-20": {
      "acc": 0.1045,
      "n": 1101
     },
     "21-50": {
      "acc": 0.08,
      "n": 525
     },
     ">50": {
      "acc": 0.0,
      "n": 23
     }
    },
    "operation_macro_f1": 0.1503,
    "op_per_class": {
     "CLICK": {
      "precision": 0.5065,
      "recall": 0.3148,
      "f1": 0.3883,
      "n_gold": 2583,
      "n_pred": 1605
     },
     "LONG_PRESS": {
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "n_gold": 0,
      "n_pred": 19
     },
     "SCROLL_UP": {
      "precision": 0.0365,
      "recall": 0.1613,
      "f1": 0.0595,
      "n_gold": 31,
      "n_pred": 137
     },
     "SCROLL_DOWN": {
      "precision": 0.3781,
      "recall": 0.1442,
      "f1": 0.2088,
      "n_gold": 527,
      "n_pred": 201
     },
     "SCROLL_LEFT": {
      "precision": 0.1786,
      "recall": 0.0877,
      "f1": 0.1176,
      "n_gold": 57,
      "n_pred": 28
     },
     "SCROLL_RIGHT": {
      "precision": 0.6167,
      "recall": 0.5,
      "f1": 0.5522,
      "n_gold": 74,
      "n_pred": 60
     },
     "OPEN_APP": {
      "precision": 0.0964,
      "recall": 0.5101,
      "f1": 0.1621,
      "n_gold": 347,
      "n_pred": 1837
     },
     "INPUT_TEXT": {
      "precision": 0.0133,
      "recall": 0.0207,
      "f1": 0.0162,
      "n_gold": 338,
      "n_pred": 528
     },
     "BACK": {
      "precision": 0.184,
      "recall": 0.278,
      "f1": 0.2214,
      "n_gold": 223,
      "n_pred": 337
     },
     "HOME": {
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "n_gold": 0,
      "n_pred": 31
     },
     "WAIT": {
      "precision": 0.0625,
      "recall": 0.007,
      "f1": 0.0127,
      "n_gold": 284,
      "n_pred": 32
     },
     "DONE": {
      "precision": 0.0907,
      "recall": 0.0511,
      "f1": 0.0653,
      "n_gold": 803,
      "n_pred": 452
     }
    },
    "unfit_items": {}
   },
   "test_category_unseen": {
    "steps": 4591,
    "operation_accuracy": 0.2413,
    "target_top1_accuracy": 0.1327,
    "joint_action_accuracy": 0.1083,
    "grounding_coverage": 0.938,
    "ece_operation": 0.5806,
    "brier_operation": 1.2867,
    "ece_target": 0.3585,
    "brier_target": 1.0978,
    "latency_ms_median": 39.61,
    "latency_ms_p95": 43.79,
    "target_top1_by_candidates": {
     "1-5": {
      "acc": 0.4615,
      "n": 221
     },
     "6-10": {
      "acc": 0.1047,
      "n": 382
     },
     "11-20": {
      "acc": 0.0986,
      "n": 984
     },
     "21-50": {
      "acc": 0.0813,
      "n": 492
     },
     ">50": {
      "acc": 0.0,
      "n": 23
     }
    },
    "operation_macro_f1": 0.1477,
    "op_per_class": {
     "CLICK": {
      "precision": 0.502,
      "recall": 0.3298,
      "f1": 0.3981,
      "n_gold": 2241,
      "n_pred": 1472
     },
     "LONG_PRESS": {
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "n_gold": 0,
      "n_pred": 19
     },
     "SCROLL_UP": {
      "precision": 0.0407,
      "recall": 0.1667,
      "f1": 0.0654,
      "n_gold": 30,
      "n_pred": 123
     },
     "SCROLL_DOWN": {
      "precision": 0.3842,
      "recall": 0.1363,
      "f1": 0.2012,
      "n_gold": 499,
      "n_pred": 177
     },
     "SCROLL_LEFT": {
      "precision": 0.1176,
      "recall": 0.0426,
      "f1": 0.0625,
      "n_gold": 47,
      "n_pred": 17
     },
     "SCROLL_RIGHT": {
      "precision": 0.6379,
      "recall": 0.5211,
      "f1": 0.5736,
      "n_gold": 71,
      "n_pred": 58
     },
     "OPEN_APP": {
      "precision": 0.1018,
      "recall": 0.5821,
      "f1": 0.1733,
      "n_gold": 280,
      "n_pred": 1601
     },
     "INPUT_TEXT": {
      "precision": 0.0053,
      "recall": 0.0075,
      "f1": 0.0062,
      "n_gold": 268,
      "n_pred": 376
     },
     "BACK": {
      "precision": 0.1824,
      "recall": 0.2791,
      "f1": 0.2206,
      "n_gold": 215,
      "n_pred": 329
     },
     "HOME": {
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "n_gold": 0,
      "n_pred": 29
     },
     "WAIT": {
      "precision": 0.08,
      "recall": 0.0083,
      "f1": 0.0151,
      "n_gold": 240,
      "n_pred": 25
     },
     "DONE": {
      "precision": 0.0822,
      "recall": 0.0429,
      "f1": 0.0563,
      "n_gold": 700,
      "n_pred": 365
     }
    },
    "unfit_items": {}
   }
  }
 }
}