{
 "model": "ByunByun/laya-android",
 "protocol": "docs/eval_protocol.md",
 "protocol_commit": "41f0a3391045bd513f02b59fd467d39f43887981",
 "code_commit": "41f0a3391045bd513f02b59fd467d39f43887981",
 "reference_evaluator": "InfiGUI-R1@a4fca17 evaluate_android_control_action",
 "full_sr_note": "N/A: v0 does not generate INPUT_TEXT text or OPEN_APP app names",
 "androidcontrol_low": "not evaluated (v0 trained with high-level goal only)",
 "androidcontrol_high": {
  "all": {
   "steps": 8444,
   "episodes": 1543,
   "type": 76.66,
   "grounding": 61.71,
   "gold_clicks": 5074,
   "full_sr": null
  },
  "test_idd": {
   "steps": 3897,
   "episodes": 721,
   "type": 81.4,
   "grounding": 67.69,
   "gold_clicks": 2445,
   "full_sr": null
  },
  "test_app_unseen": {
   "steps": 3475,
   "episodes": 631,
   "type": 70.76,
   "grounding": 53.16,
   "gold_clicks": 2024,
   "full_sr": null
  },
  "test_task_unseen": {
   "steps": 4464,
   "episodes": 803,
   "type": 72.76,
   "grounding": 56.48,
   "gold_clicks": 2583,
   "full_sr": null
  },
  "test_category_unseen": {
   "steps": 3891,
   "episodes": 700,
   "type": 71.45,
   "grounding": 54.93,
   "gold_clicks": 2241,
   "full_sr": null
  }
 },
 "internal": {
  "temperature": [
   1.1609286069869995,
   1.0,
   1.0
  ],
  "splits": {
   "test_idd": {
    "steps": 4618,
    "operation_accuracy": 0.8012,
    "target_top1_accuracy": 0.6808,
    "joint_action_accuracy": 0.659,
    "grounding_coverage": 0.9523,
    "ece_operation": 0.0124,
    "brier_operation": 0.2892,
    "ece_target": 0.0316,
    "brier_target": 0.4301,
    "latency_ms_median": 40.38,
    "latency_ms_p95": 44.81,
    "target_top1_by_candidates": {
     "1-5": {
      "acc": 0.8386,
      "n": 254
     },
     "6-10": {
      "acc": 0.6888,
      "n": 421
     },
     "11-20": {
      "acc": 0.6848,
      "n": 901
     },
     "21-50": {
      "acc": 0.6219,
      "n": 693
     },
     ">50": {
      "acc": 0.5882,
      "n": 68
     }
    },
    "operation_macro_f1": 0.6069,
    "op_per_class": {
     "CLICK": {
      "precision": 0.8471,
      "recall": 0.8928,
      "f1": 0.8694,
      "n_gold": 2445,
      "n_pred": 2577
     },
     "LONG_PRESS": {
      "precision": 0.6667,
      "recall": 0.2222,
      "f1": 0.3333,
      "n_gold": 9,
      "n_pred": 3
     },
     "SCROLL_UP": {
      "precision": 0.7059,
      "recall": 0.5538,
      "f1": 0.6207,
      "n_gold": 65,
      "n_pred": 51
     },
     "SCROLL_DOWN": {
      "precision": 0.6478,
      "recall": 0.6383,
      "f1": 0.643,
      "n_gold": 412,
      "n_pred": 406
     },
     "SCROLL_LEFT": {
      "precision": 1.0,
      "recall": 0.1333,
      "f1": 0.2353,
      "n_gold": 15,
      "n_pred": 2
     },
     "SCROLL_RIGHT": {
      "precision": 0.4545,
      "recall": 0.2778,
      "f1": 0.3448,
      "n_gold": 18,
      "n_pred": 11
     },
     "OPEN_APP": {
      "precision": 0.9091,
      "recall": 0.9375,
      "f1": 0.9231,
      "n_gold": 256,
      "n_pred": 264
     },
     "INPUT_TEXT": {
      "precision": 0.8203,
      "recall": 0.8746,
      "f1": 0.8465,
      "n_gold": 287,
      "n_pred": 306
     },
     "BACK": {
      "precision": 0.6566,
      "recall": 0.5556,
      "f1": 0.6019,
      "n_gold": 117,
      "n_pred": 99
     },
     "WAIT": {
      "precision": 0.7115,
      "recall": 0.4066,
      "f1": 0.5175,
      "n_gold": 273,
      "n_pred": 156
     },
     "DONE": {
      "precision": 0.7295,
      "recall": 0.7517,
      "f1": 0.7404,
      "n_gold": 721,
      "n_pred": 743
     }
    },
    "unfit_items": {}
   },
   "test_app_unseen": {
    "steps": 4106,
    "operation_accuracy": 0.681,
    "target_top1_accuracy": 0.599,
    "joint_action_accuracy": 0.5398,
    "grounding_coverage": 0.9328,
    "ece_operation": 0.0549,
    "brier_operation": 0.4391,
    "ece_target": 0.0639,
    "brier_target": 0.5265,
    "latency_ms_median": 36.61,
    "latency_ms_p95": 43.43,
    "target_top1_by_candidates": {
     "1-5": {
      "acc": 0.7854,
      "n": 205
     },
     "6-10": {
      "acc": 0.6288,
      "n": 361
     },
     "11-20": {
      "acc": 0.6061,
      "n": 886
     },
     "21-50": {
      "acc": 0.4756,
      "n": 410
     },
     ">50": {
      "acc": 0.4231,
      "n": 26
     }
    },
    "operation_macro_f1": 0.502,
    "op_per_class": {
     "CLICK": {
      "precision": 0.7871,
      "recall": 0.789,
      "f1": 0.7881,
      "n_gold": 2024,
      "n_pred": 2029
     },
     "SCROLL_UP": {
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "n_gold": 22,
      "n_pred": 8
     },
     "SCROLL_DOWN": {
      "precision": 0.4938,
      "recall": 0.5725,
      "f1": 0.5302,
      "n_gold": 414,
      "n_pred": 480
     },
     "SCROLL_LEFT": {
      "precision": 0.2857,
      "recall": 0.0769,
      "f1": 0.1212,
      "n_gold": 26,
      "n_pred": 7
     },
     "SCROLL_RIGHT": {
      "precision": 0.7258,
      "recall": 0.6818,
      "f1": 0.7031,
      "n_gold": 66,
      "n_pred": 62
     },
     "OPEN_APP": {
      "precision": 0.7103,
      "recall": 0.8477,
      "f1": 0.773,
      "n_gold": 243,
      "n_pred": 290
     },
     "INPUT_TEXT": {
      "precision": 0.7647,
      "recall": 0.9095,
      "f1": 0.8308,
      "n_gold": 243,
      "n_pred": 289
     },
     "BACK": {
      "precision": 0.5285,
      "recall": 0.3081,
      "f1": 0.3892,
      "n_gold": 211,
      "n_pred": 123
     },
     "WAIT": {
      "precision": 0.3333,
      "recall": 0.2965,
      "f1": 0.3138,
      "n_gold": 226,
      "n_pred": 201
     },
     "DONE": {
      "precision": 0.577,
      "recall": 0.5642,
      "f1": 0.5705,
      "n_gold": 631,
      "n_pred": 617
     }
    },
    "unfit_items": {}
   },
   "test_task_unseen": {
    "steps": 5267,
    "operation_accuracy": 0.6996,
    "target_top1_accuracy": 0.6286,
    "joint_action_accuracy": 0.5668,
    "grounding_coverage": 0.9226,
    "ece_operation": 0.045,
    "brier_operation": 0.4164,
    "ece_target": 0.0549,
    "brier_target": 0.485,
    "latency_ms_median": 39.62,
    "latency_ms_p95": 41.45,
    "target_top1_by_candidates": {
     "1-5": {
      "acc": 0.8396,
      "n": 293
     },
     "6-10": {
      "acc": 0.6599,
      "n": 441
     },
     "11-20": {
      "acc": 0.653,
      "n": 1101
     },
     "21-50": {
      "acc": 0.44,
      "n": 525
     },
     ">50": {
      "acc": 0.4783,
      "n": 23
     }
    },
    "operation_macro_f1": 0.5153,
    "op_per_class": {
     "CLICK": {
      "precision": 0.7896,
      "recall": 0.808,
      "f1": 0.7987,
      "n_gold": 2583,
      "n_pred": 2643
     },
     "SCROLL_UP": {
      "precision": 0.1538,
      "recall": 0.0645,
      "f1": 0.0909,
      "n_gold": 31,
      "n_pred": 13
     },
     "SCROLL_DOWN": {
      "precision": 0.5378,
      "recall": 0.6072,
      "f1": 0.5704,
      "n_gold": 527,
      "n_pred": 595
     },
     "SCROLL_LEFT": {
      "precision": 0.2857,
      "recall": 0.0351,
      "f1": 0.0625,
      "n_gold": 57,
      "n_pred": 7
     },
     "SCROLL_RIGHT": {
      "precision": 0.697,
      "recall": 0.6216,
      "f1": 0.6571,
      "n_gold": 74,
      "n_pred": 66
     },
     "OPEN_APP": {
      "precision": 0.7469,
      "recall": 0.8761,
      "f1": 0.8064,
      "n_gold": 347,
      "n_pred": 407
     },
     "INPUT_TEXT": {
      "precision": 0.7922,
      "recall": 0.9024,
      "f1": 0.8437,
      "n_gold": 338,
      "n_pred": 385
     },
     "BACK": {
      "precision": 0.542,
      "recall": 0.3184,
      "f1": 0.4011,
      "n_gold": 223,
      "n_pred": 131
     },
     "WAIT": {
      "precision": 0.3734,
      "recall": 0.3169,
      "f1": 0.3429,
      "n_gold": 284,
      "n_pred": 241
     },
     "DONE": {
      "precision": 0.5879,
      "recall": 0.5704,
      "f1": 0.579,
      "n_gold": 803,
      "n_pred": 779
     }
    },
    "unfit_items": {}
   },
   "test_category_unseen": {
    "steps": 4591,
    "operation_accuracy": 0.685,
    "target_top1_accuracy": 0.6018,
    "joint_action_accuracy": 0.5454,
    "grounding_coverage": 0.938,
    "ece_operation": 0.049,
    "brier_operation": 0.4346,
    "ece_target": 0.0659,
    "brier_target": 0.5161,
    "latency_ms_median": 35.97,
    "latency_ms_p95": 43.44,
    "target_top1_by_candidates": {
     "1-5": {
      "acc": 0.8009,
      "n": 221
     },
     "6-10": {
      "acc": 0.6387,
      "n": 382
     },
     "11-20": {
      "acc": 0.6321,
      "n": 984
     },
     "21-50": {
      "acc": 0.4289,
      "n": 492
     },
     ">50": {
      "acc": 0.4783,
      "n": 23
     }
    },
    "operation_macro_f1": 0.5092,
    "op_per_class": {
     "CLICK": {
      "precision": 0.7912,
      "recall": 0.7929,
      "f1": 0.7921,
      "n_gold": 2241,
      "n_pred": 2246
     },
     "SCROLL_UP": {
      "precision": 0.1538,
      "recall": 0.0667,
      "f1": 0.093,
      "n_gold": 30,
      "n_pred": 13
     },
     "SCROLL_DOWN": {
      "precision": 0.5337,
      "recall": 0.6192,
      "f1": 0.5733,
      "n_gold": 499,
      "n_pred": 579
     },
     "SCROLL_LEFT": {
      "precision": 0.2857,
      "recall": 0.0426,
      "f1": 0.0741,
      "n_gold": 47,
      "n_pred": 7
     },
     "SCROLL_RIGHT": {
      "precision": 0.7077,
      "recall": 0.6479,
      "f1": 0.6765,
      "n_gold": 71,
      "n_pred": 65
     },
     "OPEN_APP": {
      "precision": 0.7026,
      "recall": 0.8607,
      "f1": 0.7737,
      "n_gold": 280,
      "n_pred": 343
     },
     "INPUT_TEXT": {
      "precision": 0.7825,
      "recall": 0.8993,
      "f1": 0.8368,
      "n_gold": 268,
      "n_pred": 308
     },
     "BACK": {
      "precision": 0.5323,
      "recall": 0.307,
      "f1": 0.3894,
      "n_gold": 215,
      "n_pred": 124
     },
     "WAIT": {
      "precision": 0.3409,
      "recall": 0.3125,
      "f1": 0.3261,
      "n_gold": 240,
      "n_pred": 220
     },
     "DONE": {
      "precision": 0.5627,
      "recall": 0.5514,
      "f1": 0.557,
      "n_gold": 700,
      "n_pred": 686
     }
    },
    "unfit_items": {}
   }
  }
 }
}