File size: 1,975 Bytes
a9f9804
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
{
 "version": 2,
 "kind": "step",
 "confidence": "raw step confidence: P(op) x P(target of that op), temperature 1",
 "error_target": 0.02,
 "fitted_to_error": 0.01,
 "fitted": "2026-10-06",
 "method": "two-fold cross-fit by task on held-out test sets; per class and class|modality, the strictest set at the fitted error; a class whose pooled out-of-fold error exceeds 2% fails closed (1.01: nothing runs directly, MT-C02)",
 "fitted_on": {
  "vcua_clean": 600,
  "vcua_cl_clean": 600,
  "omniact_clean": 600,
  "gui360_real": 600,
  "ac_low": 600,
  "omniact_cl_clean": 600,
  "gui360_cl": 399,
  "m2w_task_cl": 513
 },
 "excluded": {
  "anbench_plan": "multi-pass scoring artifact: 18 of 20 misses at >= 0.98 have the top-ranked element in the gold set"
 },
 "cuts": {
  "browser": 1.01,
  "browser|full": 1.01,
  "desktop": 0.99453,
  "desktop|vision": 0.99453,
  "mobile": 1.01,
  "mobile|full": 1.01,
  "office": 1.01,
  "office|full": 1.01
 },
 "downgraded": {
  "browser": "pooled out-of-fold error 25.0% over 4 executed steps",
  "mobile": "pooled out-of-fold error 3.6% over 28 executed steps",
  "office": "pooled out-of-fold error 3.3% over 60 executed steps"
 },
 "out_of_fold": {
  "VideoCUA": {
   "set": "vcua_cl_clean",
   "coverage": 0.14666666666666667,
   "error": 0.011363636363636354
  },
  "OmniACT": {
   "set": "omniact_cl_clean",
   "coverage": 0.17833333333333334,
   "error": 0.009345794392523366
  },
  "GUI-360": {
   "set": "gui360_cl",
   "coverage": 0.10275689223057644,
   "error": 0.04878048780487809
  },
  "Mind2Web": {
   "set": "m2w_task_cl",
   "coverage": 0.007797270955165692,
   "error": 0.25
  },
  "AndroidControl": {
   "set": "ac_low",
   "coverage": 0.04666666666666667,
   "error": 0.0357142857142857
  }
 },
 "cascade_routes": {
  "browser": {
   "ask_below": 0.4,
   "take_at": 0.7
  },
  "desktop": {
   "ask_below": 0.8,
   "take_at": 0.5
  },
  "mobile": {
   "ask_below": 0.6,
   "take_at": 0.3
  },
  "office": null
 }
}