| { |
| "scope": "offline grounding and held-out step prediction; not live task success", |
| "training_validation_metrics": { |
| "action": 669, |
| "coordinate20": 240, |
| "coordinate50": 310, |
| "coordinate_n": 421, |
| "exact": 209, |
| "invalid": 0, |
| "n": 690, |
| "text_exact": 43, |
| "text_n": 98, |
| "loss": 4.06466895114374, |
| "seconds": 235.57891338691115, |
| "prediction_sha256": { |
| "predictions-rank0.jsonl": "2924b3fc656f070b79336e870aa37ade29f010fa307f9bec81258ca74460b96c", |
| "predictions-rank1.jsonl": "43db561d66fd975658886e27c9e4bed5475aa29287c2d3eab8991ffd15852e53" |
| } |
| }, |
| "benchmark_results": { |
| "screenspot": { |
| "n": 501, |
| "matched": 349, |
| "rate": 0.6966067864271457, |
| "action_correct": 451, |
| "strict_matches": 0, |
| "invalid": 0, |
| "complete": true, |
| "groups": { |
| "icon": { |
| "n": 211, |
| "matched": 116, |
| "rate": 0.5497630331753555, |
| "action_correct": 187, |
| "strict_matches": 0, |
| "invalid": 0 |
| }, |
| "text": { |
| "n": 290, |
| "matched": 233, |
| "rate": 0.803448275862069, |
| "action_correct": 264, |
| "strict_matches": 0, |
| "invalid": 0 |
| } |
| }, |
| "sources": { |
| "ios": { |
| "n": 238, |
| "matched": 163, |
| "rate": 0.6848739495798319, |
| "action_correct": 213, |
| "strict_matches": 0, |
| "invalid": 0 |
| }, |
| "android": { |
| "n": 211, |
| "matched": 156, |
| "rate": 0.7393364928909952, |
| "action_correct": 190, |
| "strict_matches": 0, |
| "invalid": 0 |
| }, |
| "shop": { |
| "n": 52, |
| "matched": 30, |
| "rate": 0.5769230769230769, |
| "action_correct": 48, |
| "strict_matches": 0, |
| "invalid": 0 |
| } |
| } |
| }, |
| "androidcontrol": { |
| "n": 8423, |
| "matched": 6065, |
| "rate": 0.720052237919981, |
| "action_correct": 8077, |
| "strict_matches": 2438, |
| "invalid": 0, |
| "complete": true, |
| "groups": { |
| "open_app": { |
| "n": 600, |
| "matched": 313, |
| "rate": 0.5216666666666666, |
| "action_correct": 597, |
| "strict_matches": 303, |
| "invalid": 0 |
| }, |
| "long_press": { |
| "n": 9, |
| "matched": 3, |
| "rate": 0.3333333333333333, |
| "action_correct": 8, |
| "strict_matches": 1, |
| "invalid": 0 |
| }, |
| "tap": { |
| "n": 5070, |
| "matched": 3716, |
| "rate": 0.7329388560157791, |
| "action_correct": 4956, |
| "strict_matches": 112, |
| "invalid": 0 |
| }, |
| "type_text": { |
| "n": 632, |
| "matched": 135, |
| "rate": 0.21360759493670886, |
| "action_correct": 618, |
| "strict_matches": 124, |
| "invalid": 0 |
| }, |
| "wait": { |
| "n": 560, |
| "matched": 474, |
| "rate": 0.8464285714285714, |
| "action_correct": 474, |
| "strict_matches": 474, |
| "invalid": 0 |
| }, |
| "back": { |
| "n": 342, |
| "matched": 337, |
| "rate": 0.9853801169590644, |
| "action_correct": 337, |
| "strict_matches": 337, |
| "invalid": 0 |
| }, |
| "scroll_down": { |
| "n": 946, |
| "matched": 937, |
| "rate": 0.9904862579281184, |
| "action_correct": 937, |
| "strict_matches": 937, |
| "invalid": 0 |
| }, |
| "scroll_right": { |
| "n": 94, |
| "matched": 86, |
| "rate": 0.9148936170212766, |
| "action_correct": 86, |
| "strict_matches": 86, |
| "invalid": 0 |
| }, |
| "scroll_up": { |
| "n": 96, |
| "matched": 52, |
| "rate": 0.5416666666666666, |
| "action_correct": 52, |
| "strict_matches": 52, |
| "invalid": 0 |
| }, |
| "scroll_left": { |
| "n": 74, |
| "matched": 12, |
| "rate": 0.16216216216216217, |
| "action_correct": 12, |
| "strict_matches": 12, |
| "invalid": 0 |
| } |
| }, |
| "text_exact": { |
| "correct": 124, |
| "n": 632 |
| }, |
| "coordinate_rows": { |
| "n": 5079, |
| "matched": 3719, |
| "rate": 0.7322307540854499, |
| "action_correct": 4964, |
| "strict_matches": 113, |
| "invalid": 0 |
| }, |
| "merged_sha256": "7b640d279ca61c1804cc3bae524f7a2a2a1f0b5e8c909b01aa2e515688ac2ef2" |
| } |
| }, |
| "benchmark_results_sha256": "ecf774753a6fbccd1ea0725f3244d26b273fff803edb9029e4d44386cb5fcefa", |
| "test_evaluated": true, |
| "androidcontrol_metric": { |
| "coordinate_per_axis": 50, |
| "text_similarity": 0.9, |
| "text_rule": "1 - codepoint Levenshtein distance / max lengths; case and whitespace preserved", |
| "other_fields": "exact", |
| "denominator": "8423 admitted low-level test steps; local protocol, not official leaderboard" |
| }, |
| "screenspot_metric": "Inclusive point in original box; decoded tap required; 501 full mobile subset including Android, iOS, shop" |
| } |
|
|
