File size: 1,975 Bytes
a9f9804 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | {
"version": 2,
"kind": "step",
"confidence": "raw step confidence: P(op) x P(target of that op), temperature 1",
"error_target": 0.02,
"fitted_to_error": 0.01,
"fitted": "2026-10-06",
"method": "two-fold cross-fit by task on held-out test sets; per class and class|modality, the strictest set at the fitted error; a class whose pooled out-of-fold error exceeds 2% fails closed (1.01: nothing runs directly, MT-C02)",
"fitted_on": {
"vcua_clean": 600,
"vcua_cl_clean": 600,
"omniact_clean": 600,
"gui360_real": 600,
"ac_low": 600,
"omniact_cl_clean": 600,
"gui360_cl": 399,
"m2w_task_cl": 513
},
"excluded": {
"anbench_plan": "multi-pass scoring artifact: 18 of 20 misses at >= 0.98 have the top-ranked element in the gold set"
},
"cuts": {
"browser": 1.01,
"browser|full": 1.01,
"desktop": 0.99453,
"desktop|vision": 0.99453,
"mobile": 1.01,
"mobile|full": 1.01,
"office": 1.01,
"office|full": 1.01
},
"downgraded": {
"browser": "pooled out-of-fold error 25.0% over 4 executed steps",
"mobile": "pooled out-of-fold error 3.6% over 28 executed steps",
"office": "pooled out-of-fold error 3.3% over 60 executed steps"
},
"out_of_fold": {
"VideoCUA": {
"set": "vcua_cl_clean",
"coverage": 0.14666666666666667,
"error": 0.011363636363636354
},
"OmniACT": {
"set": "omniact_cl_clean",
"coverage": 0.17833333333333334,
"error": 0.009345794392523366
},
"GUI-360": {
"set": "gui360_cl",
"coverage": 0.10275689223057644,
"error": 0.04878048780487809
},
"Mind2Web": {
"set": "m2w_task_cl",
"coverage": 0.007797270955165692,
"error": 0.25
},
"AndroidControl": {
"set": "ac_low",
"coverage": 0.04666666666666667,
"error": 0.0357142857142857
}
},
"cascade_routes": {
"browser": {
"ask_below": 0.4,
"take_at": 0.7
},
"desktop": {
"ask_below": 0.8,
"take_at": 0.5
},
"mobile": {
"ask_below": 0.6,
"take_at": 0.3
},
"office": null
}
} |