{ "raw": { "n_questions": 18854, "n_steps": 9000, "questions": { "all": { "n": 18854, "acc": 0.8255542590431738, "nll": 0.5790347412414116, "brier": 0.24178541787764835, "ece": 0.015306594949512841, "auto@1%": 0.261111700434921, "auto@2%": 0.505887344860507 }, "mod:full": { "n": 8860, "acc": 0.8258465011286682, "nll": 0.5460403093428967, "brier": 0.2353815103078951, "ece": 0.02396489798709713, "auto@1%": 0.3975169300225734, "auto@2%": 0.5248306997742663 }, "mod:vision": { "n": 9994, "acc": 0.8252951771062638, "nll": 0.6082853582737151, "brier": 0.24746268634553045, "ece": 0.014838912165115925, "auto@1%": 0.23334000400240143, "auto@2%": 0.4729837902741645 }, "q:click_target": { "n": 6769, "acc": 0.7315703944452652, "nll": 0.9211588503065505, "brier": 0.35088032487150467, "ece": 0.024903446149637493, "auto@1%": 0.1920520017727877, "auto@2%": 0.35721672329738513 }, "q:click_target|mod:full": { "n": 2982, "acc": 0.6894701542588867, "nll": 0.9940949638772786, "brier": 0.3890555680254928, "ece": 0.029380558689749914, "auto@1%": 0.30818242790073774, "auto@2%": 0.358148893360161 }, "q:click_target|mod:vision": { "n": 3787, "acc": 0.7647214153683655, "nll": 0.8637267165151825, "brier": 0.3208199670459983, "ece": 0.028248248277880146, "auto@1%": 0.2072880908370742, "auto@2%": 0.35780301029838923 }, "q:op": { "n": 9000, "acc": 0.8747777777777778, "nll": 0.4105662676475764, "brier": 0.18736618582303657, "ece": 0.022732439812579774, "auto@1%": 0.478, "auto@2%": 0.6192222222222222 }, "q:op|mod:full": { "n": 4006, "acc": 0.8984023964053919, "nll": 0.33737300407069964, "brier": 0.15619840828495632, "ece": 0.03404717479862959, "auto@1%": 0.5304543185222167, "auto@2%": 0.6635047428856715 }, "q:op|mod:vision": { "n": 4994, "acc": 0.855826992390869, "nll": 0.46927916590327695, "brier": 0.21236781113692316, "ece": 0.018079401005932097, "auto@1%": 0.4677613135762915, "auto@2%": 0.5989187024429315 }, "q:scroll_target": { "n": 284, "acc": 0.7640845070422535, "nll": 0.6463162961051102, "brier": 0.29124844724738846, "ece": 0.057572235068107845, "auto@1%": 0.528169014084507, "auto@2%": 0.5633802816901409 }, "q:scroll_target|mod:full": { "n": 183, "acc": 0.8087431693989071, "nll": 0.3304048752836481, "brier": 0.2195146899241896, "ece": 0.06336668038983957, "auto@1%": 0.5956284153005464, "auto@2%": 0.6065573770491803 }, "q:scroll_target|mod:vision": { "n": 101, "acc": 0.6831683168316832, "nll": 1.2187102566034032, "brier": 0.4212214926943726, "ece": 0.11197083875428544, "auto@1%": 0.43564356435643564, "auto@2%": 0.43564356435643564 }, "q:select_target": { "n": 31, "acc": 0.9032258064516129, "nll": 0.14361699906863964, "brier": 0.10132333183097682, "ece": 0.08140645009870662, "auto@1%": 0.8387096774193549, "auto@2%": 0.8387096774193549 }, "q:select_target|mod:full": { "n": 31, "acc": 0.9032258064516129, "nll": 0.14361699906863964, "brier": 0.10132333183097682, "ece": 0.08140645009870662, "auto@1%": 0.8387096774193549, "auto@2%": 0.8387096774193549 }, "q:set_value_target": { "n": 793, "acc": 0.8978562421185372, "nll": 0.34646736053673494, "brier": 0.15054134477454564, "ece": 0.03312025252472526, "auto@1%": 0.6658259773013872, "auto@2%": 0.7679697351828499 }, "q:set_value_target|mod:full": { "n": 435, "acc": 0.903448275862069, "nll": 0.26054433734160354, "brier": 0.1409436269607447, "ece": 0.03741812265298925, "auto@1%": 0.6919540229885057, "auto@2%": 0.8091954022988506 }, "q:set_value_target|mod:vision": { "n": 358, "acc": 0.8910614525139665, "nll": 0.45087103397215994, "brier": 0.16220337619634287, "ece": 0.03798207489591525, "auto@1%": 0.6564245810055865, "auto@2%": 0.7262569832402235 }, "q:subgoal_done": { "n": 1977, "acc": 0.9018715225088518, "nll": 0.2650206235921775, "brier": 0.14768983737552052, "ece": 0.05846293989826484, "auto@1%": 0.4496712190187152, "auto@2%": 0.6332827516439049 }, "q:subgoal_done|mod:full": { "n": 1223, "acc": 0.8937040065412919, "nll": 0.281076613489096, "brier": 0.15941370778052744, "ece": 0.06024373789737829, "auto@1%": 0.18887980376124286, "auto@2%": 0.6230580539656582 }, "q:subgoal_done|mod:vision": { "n": 754, "acc": 0.9151193633952255, "nll": 0.23897755244637994, "brier": 0.1286735329918024, "ece": 0.05731703782999492, "auto@1%": 0.5848806366047745, "auto@2%": 0.7228116710875332 } }, "steps": { "all": { "n": 9000, "op_acc": 0.8747777777777778, "element_top1": 0.7501586898565443, "step_success": 0.694, "auto@1%": 0.1801111111111111, "auto@2%": 0.2767777777777778, "hotkey_key_acc": 0.7934782608695652, "hotkey_key_n": 644, "value_acc": 0.8441890166028098, "value_n": 783, "step_success_strict": 0.6778888888888889 }, "mod:full": { "n": 4006, "op_acc": 0.8984023964053919, "element_top1": 0.7229413384742496, "step_success": 0.6749875187219171, "auto@1%": 0.21742386420369444, "auto@2%": 0.2825761357963055, "hotkey_key_acc": 0.7575757575757576, "hotkey_key_n": 165, "value_acc": 0.8661971830985915, "value_n": 426, "step_success_strict": 0.6605092361457813 }, "mod:vision": { "n": 4994, "op_acc": 0.855826992390869, "element_top1": 0.7734338200659444, "step_success": 0.7092511013215859, "auto@1%": 0.16219463356027233, "auto@2%": 0.26892270724869843, "hotkey_key_acc": 0.8058455114822547, "hotkey_key_n": 479, "value_acc": 0.8179271708683473, "value_n": 357, "step_success_strict": 0.6918301962354826 }, "op:CLICK": { "n": 6557, "op_acc": 0.9237456153728839, "element_top1": 0.7305170047277718, "step_success": 0.6917797773371969, "auto@1%": 0.17355497941131615, "auto@2%": 0.27527832850388895, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.6917797773371969 }, "op:DONE": { "n": 160, "op_acc": 0.875, "element_top1": null, "step_success": 0.875, "auto@1%": 0.48125, "auto@2%": 0.6375, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.875 }, "op:DOUBLE_CLICK": { "n": 119, "op_acc": 0.44537815126050423, "element_top1": 0.7815126050420168, "step_success": 0.42016806722689076, "auto@1%": 0.0, "auto@2%": 0.0, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.42016806722689076 }, "op:DRAG": { "n": 137, "op_acc": 0.5693430656934306, "element_top1": null, "step_success": 0.5693430656934306, "auto@1%": 0.1897810218978102, "auto@2%": 0.1897810218978102, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.5693430656934306 }, "op:HOTKEY": { "n": 720, "op_acc": 0.7819444444444444, "element_top1": null, "step_success": 0.7819444444444444, "auto@1%": 0.5791666666666667, "auto@2%": 0.6180555555555556, "hotkey_key_acc": 0.7934782608695652, "hotkey_key_n": 644, "value_acc": null, "value_n": 0, "step_success_strict": 0.6805555555555556 }, "op:RIGHT_CLICK": { "n": 93, "op_acc": 0.5268817204301075, "element_top1": 0.7419354838709677, "step_success": 0.46236559139784944, "auto@1%": 0.10752688172043011, "auto@2%": 0.10752688172043011, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.46236559139784944 }, "op:SCROLL": { "n": 284, "op_acc": 0.7464788732394366, "element_top1": 0.7640845070422535, "step_success": 0.5845070422535211, "auto@1%": 0.11971830985915492, "auto@2%": 0.11971830985915492, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.5845070422535211 }, "op:SELECT": { "n": 31, "op_acc": 0.8387096774193549, "element_top1": 0.9032258064516129, "step_success": 0.7741935483870968, "auto@1%": 0.6451612903225806, "auto@2%": 0.6451612903225806, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.7741935483870968 }, "op:SET_VALUE": { "n": 242, "op_acc": 0.7107438016528925, "element_top1": 0.9214876033057852, "step_success": 0.6652892561983471, "auto@1%": 0.06198347107438017, "auto@2%": 0.06198347107438017, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": 0.8154506437768241, "value_n": 233, "step_success_strict": 0.5785123966942148 }, "op:TYPE_TEXT": { "n": 551, "op_acc": 0.8185117967332124, "element_top1": 0.8874773139745916, "step_success": 0.7495462794918331, "auto@1%": 0.24500907441016334, "auto@2%": 0.2831215970961887, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": 0.8563636363636363, "value_n": 550, "step_success_strict": 0.6569872958257713 }, "op:WAIT": { "n": 106, "op_acc": 0.6792452830188679, "element_top1": null, "step_success": 0.6792452830188679, "auto@1%": 0.16037735849056603, "auto@2%": 0.16037735849056603, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.6792452830188679 } } }, "calibrated": { "n_questions": 18854, "n_steps": 9000, "questions": { "all": { "n": 18854, "acc": 0.8255542590431738, "nll": 0.5759684619807672, "brier": 0.2415858693668246, "ece": 0.011065454019417914, "auto@1%": 0.3090060464622892, "auto@2%": 0.5286411371592236 }, "mod:full": { "n": 8860, "acc": 0.8258465011286682, "nll": 0.5426569199239745, "brier": 0.23472349892834699, "ece": 0.016077250694546564, "auto@1%": 0.39785553047404065, "auto@2%": 0.5343115124153499 }, "mod:vision": { "n": 9994, "acc": 0.8252951771062638, "nll": 0.6055002072902712, "brier": 0.24766957980157658, "ece": 0.011613404306611499, "auto@1%": 0.23724234540724434, "auto@2%": 0.5012007204322594 }, "q:click_target": { "n": 6769, "acc": 0.7315703944452652, "nll": 0.9186379018046859, "brier": 0.35063962522032743, "ece": 0.017903943986485604, "auto@1%": 0.19633623873541142, "auto@2%": 0.3609100310237849 }, "q:click_target|mod:full": { "n": 2982, "acc": 0.6894701542588867, "nll": 0.9916620619258089, "brier": 0.3885699440853336, "ece": 0.028073393950339448, "auto@1%": 0.3075117370892019, "auto@2%": 0.3561368209255533 }, "q:click_target|mod:vision": { "n": 3787, "acc": 0.7647214153683655, "nll": 0.8611364374579239, "brier": 0.3207721282952025, "ece": 0.02522712125449486, "auto@1%": 0.2112490097702667, "auto@2%": 0.3585951940850277 }, "q:op": { "n": 9000, "acc": 0.8747777777777778, "nll": 0.41109552728328835, "brier": 0.1881643006091161, "ece": 0.027887270021343957, "auto@1%": 0.4728888888888889, "auto@2%": 0.6281111111111111 }, "q:op|mod:full": { "n": 4006, "acc": 0.8984023964053919, "nll": 0.3386421776225732, "brier": 0.15681739249583776, "ece": 0.03659893292338151, "auto@1%": 0.5319520718921618, "auto@2%": 0.6662506240639041 }, "q:op|mod:vision": { "n": 4994, "acc": 0.855826992390869, "nll": 0.4692148942718397, "brier": 0.21330961777006785, "ece": 0.0234003205175579, "auto@1%": 0.4669603524229075, "auto@2%": 0.6021225470564677 }, "q:scroll_target": { "n": 284, "acc": 0.7640845070422535, "nll": 0.6508194589556844, "brier": 0.292453199311092, "ece": 0.05964354811915089, "auto@1%": 0.528169014084507, "auto@2%": 0.5633802816901409 }, "q:scroll_target|mod:full": { "n": 183, "acc": 0.8087431693989071, "nll": 0.3336389861650783, "brier": 0.2203359511972443, "ece": 0.07096846774592282, "auto@1%": 0.5956284153005464, "auto@2%": 0.6065573770491803 }, "q:scroll_target|mod:vision": { "n": 101, "acc": 0.6831683168316832, "nll": 1.2255127908436143, "brier": 0.42312108450746927, "ece": 0.12651085530483008, "auto@1%": 0.42574257425742573, "auto@2%": 0.42574257425742573 }, "q:select_target": { "n": 31, "acc": 0.9032258064516129, "nll": 0.14205801001819574, "brier": 0.10010005623509433, "ece": 0.06967246525847405, "auto@1%": 0.8387096774193549, "auto@2%": 0.8387096774193549 }, "q:select_target|mod:full": { "n": 31, "acc": 0.9032258064516129, "nll": 0.14205801001819574, "brier": 0.10010005623509433, "ece": 0.06967246525847405, "auto@1%": 0.8387096774193549, "auto@2%": 0.8387096774193549 }, "q:set_value_target": { "n": 793, "acc": 0.8978562421185372, "nll": 0.3453675760949502, "brier": 0.15109203911045183, "ece": 0.036194069842637525, "auto@1%": 0.6582597730138714, "auto@2%": 0.7667087011349306 }, "q:set_value_target|mod:full": { "n": 435, "acc": 0.903448275862069, "nll": 0.25961273364538356, "brier": 0.14131568516459533, "ece": 0.03645868658074942, "auto@1%": 0.6919540229885057, "auto@2%": 0.8091954022988506 }, "q:set_value_target|mod:vision": { "n": 358, "acc": 0.8910614525139665, "nll": 0.44956689583115567, "brier": 0.16297112840220485, "ece": 0.037533499975045806, "auto@1%": 0.6396648044692738, "auto@2%": 0.7178770949720671 }, "q:subgoal_done": { "n": 1977, "acc": 0.9018715225088518, "nll": 0.24181925484198044, "brier": 0.14260286043573228, "ece": 0.01994882289817495, "auto@1%": 0.15174506828528073, "auto@2%": 0.661608497723824 }, "q:subgoal_done|mod:full": { "n": 1223, "acc": 0.8937040065412919, "nll": 0.2582274786534001, "brier": 0.1535791079684229, "ece": 0.021707061575793954, "auto@1%": 0.18887980376124286, "auto@2%": 0.6230580539656582 }, "q:subgoal_done|mod:vision": { "n": 754, "acc": 0.9151193633952255, "nll": 0.21520485468101727, "brier": 0.12479921224941845, "ece": 0.018088822384504093, "auto@1%": 0.5848806366047745, "auto@2%": 0.7228116710875332 } }, "steps": { "all": { "n": 9000, "op_acc": 0.8747777777777778, "element_top1": 0.7501586898565443, "step_success": 0.694, "auto@1%": 0.1801111111111111, "auto@2%": 0.2767777777777778, "hotkey_key_acc": 0.7934782608695652, "hotkey_key_n": 644, "value_acc": 0.8441890166028098, "value_n": 783, "step_success_strict": 0.6778888888888889 }, "mod:full": { "n": 4006, "op_acc": 0.8984023964053919, "element_top1": 0.7229413384742496, "step_success": 0.6749875187219171, "auto@1%": 0.21742386420369444, "auto@2%": 0.2825761357963055, "hotkey_key_acc": 0.7575757575757576, "hotkey_key_n": 165, "value_acc": 0.8661971830985915, "value_n": 426, "step_success_strict": 0.6605092361457813 }, "mod:vision": { "n": 4994, "op_acc": 0.855826992390869, "element_top1": 0.7734338200659444, "step_success": 0.7092511013215859, "auto@1%": 0.16219463356027233, "auto@2%": 0.26892270724869843, "hotkey_key_acc": 0.8058455114822547, "hotkey_key_n": 479, "value_acc": 0.8179271708683473, "value_n": 357, "step_success_strict": 0.6918301962354826 }, "op:CLICK": { "n": 6557, "op_acc": 0.9237456153728839, "element_top1": 0.7305170047277718, "step_success": 0.6917797773371969, "auto@1%": 0.17355497941131615, "auto@2%": 0.27527832850388895, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.6917797773371969 }, "op:DONE": { "n": 160, "op_acc": 0.875, "element_top1": null, "step_success": 0.875, "auto@1%": 0.48125, "auto@2%": 0.6375, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.875 }, "op:DOUBLE_CLICK": { "n": 119, "op_acc": 0.44537815126050423, "element_top1": 0.7815126050420168, "step_success": 0.42016806722689076, "auto@1%": 0.0, "auto@2%": 0.0, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.42016806722689076 }, "op:DRAG": { "n": 137, "op_acc": 0.5693430656934306, "element_top1": null, "step_success": 0.5693430656934306, "auto@1%": 0.1897810218978102, "auto@2%": 0.1897810218978102, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.5693430656934306 }, "op:HOTKEY": { "n": 720, "op_acc": 0.7819444444444444, "element_top1": null, "step_success": 0.7819444444444444, "auto@1%": 0.5791666666666667, "auto@2%": 0.6180555555555556, "hotkey_key_acc": 0.7934782608695652, "hotkey_key_n": 644, "value_acc": null, "value_n": 0, "step_success_strict": 0.6805555555555556 }, "op:RIGHT_CLICK": { "n": 93, "op_acc": 0.5268817204301075, "element_top1": 0.7419354838709677, "step_success": 0.46236559139784944, "auto@1%": 0.10752688172043011, "auto@2%": 0.10752688172043011, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.46236559139784944 }, "op:SCROLL": { "n": 284, "op_acc": 0.7464788732394366, "element_top1": 0.7640845070422535, "step_success": 0.5845070422535211, "auto@1%": 0.11971830985915492, "auto@2%": 0.11971830985915492, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.5845070422535211 }, "op:SELECT": { "n": 31, "op_acc": 0.8387096774193549, "element_top1": 0.9032258064516129, "step_success": 0.7741935483870968, "auto@1%": 0.6451612903225806, "auto@2%": 0.6451612903225806, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.7741935483870968 }, "op:SET_VALUE": { "n": 242, "op_acc": 0.7107438016528925, "element_top1": 0.9214876033057852, "step_success": 0.6652892561983471, "auto@1%": 0.06198347107438017, "auto@2%": 0.06198347107438017, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": 0.8154506437768241, "value_n": 233, "step_success_strict": 0.5785123966942148 }, "op:TYPE_TEXT": { "n": 551, "op_acc": 0.8185117967332124, "element_top1": 0.8874773139745916, "step_success": 0.7495462794918331, "auto@1%": 0.24500907441016334, "auto@2%": 0.2831215970961887, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": 0.8563636363636363, "value_n": 550, "step_success_strict": 0.6569872958257713 }, "op:WAIT": { "n": 106, "op_acc": 0.6792452830188679, "element_top1": null, "step_success": 0.6792452830188679, "auto@1%": 0.16037735849056603, "auto@2%": 0.16037735849056603, "hotkey_key_acc": null, "hotkey_key_n": 0, "value_acc": null, "value_n": 0, "step_success_strict": 0.6792452830188679 } } }, "calibration_report": { "all": { "n": 3771, "ece_before": 0.01747626450483553, "ece_after": 0.018764672656685816, "nll_before": 0.5712372033403514, "nll_after": 0.5699958148241562, "execute_share": 0.5452134712277911, "execute_error": 0.02383268482490275, "escalate_share": 0.15141872182444974 }, "mod:full": { "n": 1772, "ece_before": 0.02814794127457049, "ece_after": 0.02590856275493896, "nll_before": 0.5519384331521392, "nll_after": 0.5517056593417647, "execute_share": 0.5564334085778782, "execute_error": 0.026369168356997985, "escalate_share": 0.14503386004514673 }, "mod:vision": { "n": 1999, "ece_before": 0.017960368637130675, "ece_after": 0.021329443853398466, "nll_before": 0.5883444673591168, "nll_after": 0.5862089991737299, "execute_share": 0.5352676338169085, "execute_error": 0.021495327102803774, "escalate_share": 0.1570785392696348 }, "qtype:choice": { "n": 3375, "ece_before": 0.01432174752193021, "ece_after": 0.0219802066016441, "nll_before": 0.6036172560395445, "nll_after": 0.6031795218505003, "execute_share": 0.5306666666666666, "execute_error": 0.022333891680625384, "escalate_share": 0.1691851851851852 }, "qtype:noul": { "n": 396, "ece_before": 0.05879165844354522, "ece_after": 0.03624674956609409, "nll_before": 0.2952708451085907, "nll_after": 0.28718012994054226, "execute_share": 0.6691919191919192, "execute_error": 0.03396226415094339, "escalate_share": 0.0 } } }