{
  "scope": "offline grounding and held-out step prediction; not live task success",
  "training_validation_metrics": {
    "action": 669,
    "coordinate20": 240,
    "coordinate50": 310,
    "coordinate_n": 421,
    "exact": 209,
    "invalid": 0,
    "n": 690,
    "text_exact": 43,
    "text_n": 98,
    "loss": 4.06466895114374,
    "seconds": 235.57891338691115,
    "prediction_sha256": {
      "predictions-rank0.jsonl": "2924b3fc656f070b79336e870aa37ade29f010fa307f9bec81258ca74460b96c",
      "predictions-rank1.jsonl": "43db561d66fd975658886e27c9e4bed5475aa29287c2d3eab8991ffd15852e53"
    }
  },
  "benchmark_results": {
    "screenspot": {
      "n": 501,
      "matched": 349,
      "rate": 0.6966067864271457,
      "action_correct": 451,
      "strict_matches": 0,
      "invalid": 0,
      "complete": true,
      "groups": {
        "icon": {
          "n": 211,
          "matched": 116,
          "rate": 0.5497630331753555,
          "action_correct": 187,
          "strict_matches": 0,
          "invalid": 0
        },
        "text": {
          "n": 290,
          "matched": 233,
          "rate": 0.803448275862069,
          "action_correct": 264,
          "strict_matches": 0,
          "invalid": 0
        }
      },
      "sources": {
        "ios": {
          "n": 238,
          "matched": 163,
          "rate": 0.6848739495798319,
          "action_correct": 213,
          "strict_matches": 0,
          "invalid": 0
        },
        "android": {
          "n": 211,
          "matched": 156,
          "rate": 0.7393364928909952,
          "action_correct": 190,
          "strict_matches": 0,
          "invalid": 0
        },
        "shop": {
          "n": 52,
          "matched": 30,
          "rate": 0.5769230769230769,
          "action_correct": 48,
          "strict_matches": 0,
          "invalid": 0
        }
      }
    },
    "androidcontrol": {
      "n": 8423,
      "matched": 6065,
      "rate": 0.720052237919981,
      "action_correct": 8077,
      "strict_matches": 2438,
      "invalid": 0,
      "complete": true,
      "groups": {
        "open_app": {
          "n": 600,
          "matched": 313,
          "rate": 0.5216666666666666,
          "action_correct": 597,
          "strict_matches": 303,
          "invalid": 0
        },
        "long_press": {
          "n": 9,
          "matched": 3,
          "rate": 0.3333333333333333,
          "action_correct": 8,
          "strict_matches": 1,
          "invalid": 0
        },
        "tap": {
          "n": 5070,
          "matched": 3716,
          "rate": 0.7329388560157791,
          "action_correct": 4956,
          "strict_matches": 112,
          "invalid": 0
        },
        "type_text": {
          "n": 632,
          "matched": 135,
          "rate": 0.21360759493670886,
          "action_correct": 618,
          "strict_matches": 124,
          "invalid": 0
        },
        "wait": {
          "n": 560,
          "matched": 474,
          "rate": 0.8464285714285714,
          "action_correct": 474,
          "strict_matches": 474,
          "invalid": 0
        },
        "back": {
          "n": 342,
          "matched": 337,
          "rate": 0.9853801169590644,
          "action_correct": 337,
          "strict_matches": 337,
          "invalid": 0
        },
        "scroll_down": {
          "n": 946,
          "matched": 937,
          "rate": 0.9904862579281184,
          "action_correct": 937,
          "strict_matches": 937,
          "invalid": 0
        },
        "scroll_right": {
          "n": 94,
          "matched": 86,
          "rate": 0.9148936170212766,
          "action_correct": 86,
          "strict_matches": 86,
          "invalid": 0
        },
        "scroll_up": {
          "n": 96,
          "matched": 52,
          "rate": 0.5416666666666666,
          "action_correct": 52,
          "strict_matches": 52,
          "invalid": 0
        },
        "scroll_left": {
          "n": 74,
          "matched": 12,
          "rate": 0.16216216216216217,
          "action_correct": 12,
          "strict_matches": 12,
          "invalid": 0
        }
      },
      "text_exact": {
        "correct": 124,
        "n": 632
      },
      "coordinate_rows": {
        "n": 5079,
        "matched": 3719,
        "rate": 0.7322307540854499,
        "action_correct": 4964,
        "strict_matches": 113,
        "invalid": 0
      },
      "merged_sha256": "7b640d279ca61c1804cc3bae524f7a2a2a1f0b5e8c909b01aa2e515688ac2ef2"
    }
  },
  "benchmark_results_sha256": "ecf774753a6fbccd1ea0725f3244d26b273fff803edb9029e4d44386cb5fcefa",
  "test_evaluated": true,
  "androidcontrol_metric": {
    "coordinate_per_axis": 50,
    "text_similarity": 0.9,
    "text_rule": "1 - codepoint Levenshtein distance / max lengths; case and whitespace preserved",
    "other_fields": "exact",
    "denominator": "8423 admitted low-level test steps; local protocol, not official leaderboard"
  },
  "screenspot_metric": "Inclusive point in original box; decoded tap required; 501 full mobile subset including Android, iOS, shop"
}