{
  "historical_results": true,
  "new_evaluation_for_release": false,
  "checkpoint_sha256": "bd82d3c508a7b591b7c1578614231b142689982c842c6864c80baaa213e5dde5",
  "osworld_selected_suite": {
    "official_tasks": 369,
    "selected_tasks": 361,
    "scored": 356,
    "successes": 58,
    "success_rate_scored": 0.16292134831460675,
    "unscored": 5,
    "mean_reward_scored": 0.17120773098371703
  },
  "development_validation": {
    "step": 3368,
    "n": 1967,
    "loss": 5.176659643919462,
    "raw_action_selector_valid": 0.9415353329944077,
    "constrained_schema_valid": 0.9994916115912558,
    "action_accuracy": 0.7559735638027453,
    "canonical_exact_match": 0.24046771733604475,
    "coordinate_bin_accuracy": 0.19882246376811594,
    "action_and_point_within_20_of_1000": 0.41304347826086957,
    "coordinate_rows": 1104,
    "single_forward_per_example": true,
    "by_action": {
      "double_click": {
        "n": 199,
        "exact": 0.005025125628140704,
        "action_accuracy": 0.5326633165829145
      },
      "key": {
        "n": 301,
        "exact": 0.6744186046511628,
        "action_accuracy": 0.7740863787375415
      },
      "left_click": {
        "n": 584,
        "exact": 0.0017123287671232876,
        "action_accuracy": 0.7465753424657534
      },
      "left_click_drag": {
        "n": 24,
        "exact": 0.0,
        "action_accuracy": 0.7916666666666666
      },
      "middle_click": {
        "n": 3,
        "exact": 0.0,
        "action_accuracy": 0.3333333333333333
      },
      "mouse_move": {
        "n": 36,
        "exact": 0.0,
        "action_accuracy": 0.16666666666666666
      },
      "right_click": {
        "n": 160,
        "exact": 0.00625,
        "action_accuracy": 0.6625
      },
      "scroll": {
        "n": 83,
        "exact": 0.04819277108433735,
        "action_accuracy": 0.8192771084337349
      },
      "terminate": {
        "n": 163,
        "exact": 0.8773006134969326,
        "action_accuracy": 0.8773006134969326
      },
      "triple_click": {
        "n": 15,
        "exact": 0.0,
        "action_accuracy": 0.6
      },
      "type": {
        "n": 399,
        "exact": 0.3007518796992481,
        "action_accuracy": 0.9022556390977443
      }
    }
  },
  "scope": "Five OSWorld tasks unscored; development validation repeatedly observed; not an official leaderboard submission."
}