| { |
| "historical_results": true, |
| "new_evaluation_for_release": false, |
| "checkpoint_sha256": "bd82d3c508a7b591b7c1578614231b142689982c842c6864c80baaa213e5dde5", |
| "osworld_selected_suite": { |
| "official_tasks": 369, |
| "selected_tasks": 361, |
| "scored": 356, |
| "successes": 58, |
| "success_rate_scored": 0.16292134831460675, |
| "unscored": 5, |
| "mean_reward_scored": 0.17120773098371703 |
| }, |
| "development_validation": { |
| "step": 3368, |
| "n": 1967, |
| "loss": 5.176659643919462, |
| "raw_action_selector_valid": 0.9415353329944077, |
| "constrained_schema_valid": 0.9994916115912558, |
| "action_accuracy": 0.7559735638027453, |
| "canonical_exact_match": 0.24046771733604475, |
| "coordinate_bin_accuracy": 0.19882246376811594, |
| "action_and_point_within_20_of_1000": 0.41304347826086957, |
| "coordinate_rows": 1104, |
| "single_forward_per_example": true, |
| "by_action": { |
| "double_click": { |
| "n": 199, |
| "exact": 0.005025125628140704, |
| "action_accuracy": 0.5326633165829145 |
| }, |
| "key": { |
| "n": 301, |
| "exact": 0.6744186046511628, |
| "action_accuracy": 0.7740863787375415 |
| }, |
| "left_click": { |
| "n": 584, |
| "exact": 0.0017123287671232876, |
| "action_accuracy": 0.7465753424657534 |
| }, |
| "left_click_drag": { |
| "n": 24, |
| "exact": 0.0, |
| "action_accuracy": 0.7916666666666666 |
| }, |
| "middle_click": { |
| "n": 3, |
| "exact": 0.0, |
| "action_accuracy": 0.3333333333333333 |
| }, |
| "mouse_move": { |
| "n": 36, |
| "exact": 0.0, |
| "action_accuracy": 0.16666666666666666 |
| }, |
| "right_click": { |
| "n": 160, |
| "exact": 0.00625, |
| "action_accuracy": 0.6625 |
| }, |
| "scroll": { |
| "n": 83, |
| "exact": 0.04819277108433735, |
| "action_accuracy": 0.8192771084337349 |
| }, |
| "terminate": { |
| "n": 163, |
| "exact": 0.8773006134969326, |
| "action_accuracy": 0.8773006134969326 |
| }, |
| "triple_click": { |
| "n": 15, |
| "exact": 0.0, |
| "action_accuracy": 0.6 |
| }, |
| "type": { |
| "n": 399, |
| "exact": 0.3007518796992481, |
| "action_accuracy": 0.9022556390977443 |
| } |
| } |
| }, |
| "scope": "Five OSWorld tasks unscored; development validation repeatedly observed; not an official leaderboard submission." |
| } |
|
|
