| { |
| "raw": { |
| "n_questions": 18854, |
| "n_steps": 9000, |
| "questions": { |
| "all": { |
| "n": 18854, |
| "acc": 0.8255542590431738, |
| "nll": 0.5790347412414116, |
| "brier": 0.24178541787764835, |
| "ece": 0.015306594949512841, |
| "auto@1%": 0.261111700434921, |
| "auto@2%": 0.505887344860507 |
| }, |
| "mod:full": { |
| "n": 8860, |
| "acc": 0.8258465011286682, |
| "nll": 0.5460403093428967, |
| "brier": 0.2353815103078951, |
| "ece": 0.02396489798709713, |
| "auto@1%": 0.3975169300225734, |
| "auto@2%": 0.5248306997742663 |
| }, |
| "mod:vision": { |
| "n": 9994, |
| "acc": 0.8252951771062638, |
| "nll": 0.6082853582737151, |
| "brier": 0.24746268634553045, |
| "ece": 0.014838912165115925, |
| "auto@1%": 0.23334000400240143, |
| "auto@2%": 0.4729837902741645 |
| }, |
| "q:click_target": { |
| "n": 6769, |
| "acc": 0.7315703944452652, |
| "nll": 0.9211588503065505, |
| "brier": 0.35088032487150467, |
| "ece": 0.024903446149637493, |
| "auto@1%": 0.1920520017727877, |
| "auto@2%": 0.35721672329738513 |
| }, |
| "q:click_target|mod:full": { |
| "n": 2982, |
| "acc": 0.6894701542588867, |
| "nll": 0.9940949638772786, |
| "brier": 0.3890555680254928, |
| "ece": 0.029380558689749914, |
| "auto@1%": 0.30818242790073774, |
| "auto@2%": 0.358148893360161 |
| }, |
| "q:click_target|mod:vision": { |
| "n": 3787, |
| "acc": 0.7647214153683655, |
| "nll": 0.8637267165151825, |
| "brier": 0.3208199670459983, |
| "ece": 0.028248248277880146, |
| "auto@1%": 0.2072880908370742, |
| "auto@2%": 0.35780301029838923 |
| }, |
| "q:op": { |
| "n": 9000, |
| "acc": 0.8747777777777778, |
| "nll": 0.4105662676475764, |
| "brier": 0.18736618582303657, |
| "ece": 0.022732439812579774, |
| "auto@1%": 0.478, |
| "auto@2%": 0.6192222222222222 |
| }, |
| "q:op|mod:full": { |
| "n": 4006, |
| "acc": 0.8984023964053919, |
| "nll": 0.33737300407069964, |
| "brier": 0.15619840828495632, |
| "ece": 0.03404717479862959, |
| "auto@1%": 0.5304543185222167, |
| "auto@2%": 0.6635047428856715 |
| }, |
| "q:op|mod:vision": { |
| "n": 4994, |
| "acc": 0.855826992390869, |
| "nll": 0.46927916590327695, |
| "brier": 0.21236781113692316, |
| "ece": 0.018079401005932097, |
| "auto@1%": 0.4677613135762915, |
| "auto@2%": 0.5989187024429315 |
| }, |
| "q:scroll_target": { |
| "n": 284, |
| "acc": 0.7640845070422535, |
| "nll": 0.6463162961051102, |
| "brier": 0.29124844724738846, |
| "ece": 0.057572235068107845, |
| "auto@1%": 0.528169014084507, |
| "auto@2%": 0.5633802816901409 |
| }, |
| "q:scroll_target|mod:full": { |
| "n": 183, |
| "acc": 0.8087431693989071, |
| "nll": 0.3304048752836481, |
| "brier": 0.2195146899241896, |
| "ece": 0.06336668038983957, |
| "auto@1%": 0.5956284153005464, |
| "auto@2%": 0.6065573770491803 |
| }, |
| "q:scroll_target|mod:vision": { |
| "n": 101, |
| "acc": 0.6831683168316832, |
| "nll": 1.2187102566034032, |
| "brier": 0.4212214926943726, |
| "ece": 0.11197083875428544, |
| "auto@1%": 0.43564356435643564, |
| "auto@2%": 0.43564356435643564 |
| }, |
| "q:select_target": { |
| "n": 31, |
| "acc": 0.9032258064516129, |
| "nll": 0.14361699906863964, |
| "brier": 0.10132333183097682, |
| "ece": 0.08140645009870662, |
| "auto@1%": 0.8387096774193549, |
| "auto@2%": 0.8387096774193549 |
| }, |
| "q:select_target|mod:full": { |
| "n": 31, |
| "acc": 0.9032258064516129, |
| "nll": 0.14361699906863964, |
| "brier": 0.10132333183097682, |
| "ece": 0.08140645009870662, |
| "auto@1%": 0.8387096774193549, |
| "auto@2%": 0.8387096774193549 |
| }, |
| "q:set_value_target": { |
| "n": 793, |
| "acc": 0.8978562421185372, |
| "nll": 0.34646736053673494, |
| "brier": 0.15054134477454564, |
| "ece": 0.03312025252472526, |
| "auto@1%": 0.6658259773013872, |
| "auto@2%": 0.7679697351828499 |
| }, |
| "q:set_value_target|mod:full": { |
| "n": 435, |
| "acc": 0.903448275862069, |
| "nll": 0.26054433734160354, |
| "brier": 0.1409436269607447, |
| "ece": 0.03741812265298925, |
| "auto@1%": 0.6919540229885057, |
| "auto@2%": 0.8091954022988506 |
| }, |
| "q:set_value_target|mod:vision": { |
| "n": 358, |
| "acc": 0.8910614525139665, |
| "nll": 0.45087103397215994, |
| "brier": 0.16220337619634287, |
| "ece": 0.03798207489591525, |
| "auto@1%": 0.6564245810055865, |
| "auto@2%": 0.7262569832402235 |
| }, |
| "q:subgoal_done": { |
| "n": 1977, |
| "acc": 0.9018715225088518, |
| "nll": 0.2650206235921775, |
| "brier": 0.14768983737552052, |
| "ece": 0.05846293989826484, |
| "auto@1%": 0.4496712190187152, |
| "auto@2%": 0.6332827516439049 |
| }, |
| "q:subgoal_done|mod:full": { |
| "n": 1223, |
| "acc": 0.8937040065412919, |
| "nll": 0.281076613489096, |
| "brier": 0.15941370778052744, |
| "ece": 0.06024373789737829, |
| "auto@1%": 0.18887980376124286, |
| "auto@2%": 0.6230580539656582 |
| }, |
| "q:subgoal_done|mod:vision": { |
| "n": 754, |
| "acc": 0.9151193633952255, |
| "nll": 0.23897755244637994, |
| "brier": 0.1286735329918024, |
| "ece": 0.05731703782999492, |
| "auto@1%": 0.5848806366047745, |
| "auto@2%": 0.7228116710875332 |
| } |
| }, |
| "steps": { |
| "all": { |
| "n": 9000, |
| "op_acc": 0.8747777777777778, |
| "element_top1": 0.7501586898565443, |
| "step_success": 0.694, |
| "auto@1%": 0.1801111111111111, |
| "auto@2%": 0.2767777777777778, |
| "hotkey_key_acc": 0.7934782608695652, |
| "hotkey_key_n": 644, |
| "value_acc": 0.8441890166028098, |
| "value_n": 783, |
| "step_success_strict": 0.6778888888888889 |
| }, |
| "mod:full": { |
| "n": 4006, |
| "op_acc": 0.8984023964053919, |
| "element_top1": 0.7229413384742496, |
| "step_success": 0.6749875187219171, |
| "auto@1%": 0.21742386420369444, |
| "auto@2%": 0.2825761357963055, |
| "hotkey_key_acc": 0.7575757575757576, |
| "hotkey_key_n": 165, |
| "value_acc": 0.8661971830985915, |
| "value_n": 426, |
| "step_success_strict": 0.6605092361457813 |
| }, |
| "mod:vision": { |
| "n": 4994, |
| "op_acc": 0.855826992390869, |
| "element_top1": 0.7734338200659444, |
| "step_success": 0.7092511013215859, |
| "auto@1%": 0.16219463356027233, |
| "auto@2%": 0.26892270724869843, |
| "hotkey_key_acc": 0.8058455114822547, |
| "hotkey_key_n": 479, |
| "value_acc": 0.8179271708683473, |
| "value_n": 357, |
| "step_success_strict": 0.6918301962354826 |
| }, |
| "op:CLICK": { |
| "n": 6557, |
| "op_acc": 0.9237456153728839, |
| "element_top1": 0.7305170047277718, |
| "step_success": 0.6917797773371969, |
| "auto@1%": 0.17355497941131615, |
| "auto@2%": 0.27527832850388895, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.6917797773371969 |
| }, |
| "op:DONE": { |
| "n": 160, |
| "op_acc": 0.875, |
| "element_top1": null, |
| "step_success": 0.875, |
| "auto@1%": 0.48125, |
| "auto@2%": 0.6375, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.875 |
| }, |
| "op:DOUBLE_CLICK": { |
| "n": 119, |
| "op_acc": 0.44537815126050423, |
| "element_top1": 0.7815126050420168, |
| "step_success": 0.42016806722689076, |
| "auto@1%": 0.0, |
| "auto@2%": 0.0, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.42016806722689076 |
| }, |
| "op:DRAG": { |
| "n": 137, |
| "op_acc": 0.5693430656934306, |
| "element_top1": null, |
| "step_success": 0.5693430656934306, |
| "auto@1%": 0.1897810218978102, |
| "auto@2%": 0.1897810218978102, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.5693430656934306 |
| }, |
| "op:HOTKEY": { |
| "n": 720, |
| "op_acc": 0.7819444444444444, |
| "element_top1": null, |
| "step_success": 0.7819444444444444, |
| "auto@1%": 0.5791666666666667, |
| "auto@2%": 0.6180555555555556, |
| "hotkey_key_acc": 0.7934782608695652, |
| "hotkey_key_n": 644, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.6805555555555556 |
| }, |
| "op:RIGHT_CLICK": { |
| "n": 93, |
| "op_acc": 0.5268817204301075, |
| "element_top1": 0.7419354838709677, |
| "step_success": 0.46236559139784944, |
| "auto@1%": 0.10752688172043011, |
| "auto@2%": 0.10752688172043011, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.46236559139784944 |
| }, |
| "op:SCROLL": { |
| "n": 284, |
| "op_acc": 0.7464788732394366, |
| "element_top1": 0.7640845070422535, |
| "step_success": 0.5845070422535211, |
| "auto@1%": 0.11971830985915492, |
| "auto@2%": 0.11971830985915492, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.5845070422535211 |
| }, |
| "op:SELECT": { |
| "n": 31, |
| "op_acc": 0.8387096774193549, |
| "element_top1": 0.9032258064516129, |
| "step_success": 0.7741935483870968, |
| "auto@1%": 0.6451612903225806, |
| "auto@2%": 0.6451612903225806, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.7741935483870968 |
| }, |
| "op:SET_VALUE": { |
| "n": 242, |
| "op_acc": 0.7107438016528925, |
| "element_top1": 0.9214876033057852, |
| "step_success": 0.6652892561983471, |
| "auto@1%": 0.06198347107438017, |
| "auto@2%": 0.06198347107438017, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": 0.8154506437768241, |
| "value_n": 233, |
| "step_success_strict": 0.5785123966942148 |
| }, |
| "op:TYPE_TEXT": { |
| "n": 551, |
| "op_acc": 0.8185117967332124, |
| "element_top1": 0.8874773139745916, |
| "step_success": 0.7495462794918331, |
| "auto@1%": 0.24500907441016334, |
| "auto@2%": 0.2831215970961887, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": 0.8563636363636363, |
| "value_n": 550, |
| "step_success_strict": 0.6569872958257713 |
| }, |
| "op:WAIT": { |
| "n": 106, |
| "op_acc": 0.6792452830188679, |
| "element_top1": null, |
| "step_success": 0.6792452830188679, |
| "auto@1%": 0.16037735849056603, |
| "auto@2%": 0.16037735849056603, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.6792452830188679 |
| } |
| } |
| }, |
| "calibrated": { |
| "n_questions": 18854, |
| "n_steps": 9000, |
| "questions": { |
| "all": { |
| "n": 18854, |
| "acc": 0.8255542590431738, |
| "nll": 0.5759684619807672, |
| "brier": 0.2415858693668246, |
| "ece": 0.011065454019417914, |
| "auto@1%": 0.3090060464622892, |
| "auto@2%": 0.5286411371592236 |
| }, |
| "mod:full": { |
| "n": 8860, |
| "acc": 0.8258465011286682, |
| "nll": 0.5426569199239745, |
| "brier": 0.23472349892834699, |
| "ece": 0.016077250694546564, |
| "auto@1%": 0.39785553047404065, |
| "auto@2%": 0.5343115124153499 |
| }, |
| "mod:vision": { |
| "n": 9994, |
| "acc": 0.8252951771062638, |
| "nll": 0.6055002072902712, |
| "brier": 0.24766957980157658, |
| "ece": 0.011613404306611499, |
| "auto@1%": 0.23724234540724434, |
| "auto@2%": 0.5012007204322594 |
| }, |
| "q:click_target": { |
| "n": 6769, |
| "acc": 0.7315703944452652, |
| "nll": 0.9186379018046859, |
| "brier": 0.35063962522032743, |
| "ece": 0.017903943986485604, |
| "auto@1%": 0.19633623873541142, |
| "auto@2%": 0.3609100310237849 |
| }, |
| "q:click_target|mod:full": { |
| "n": 2982, |
| "acc": 0.6894701542588867, |
| "nll": 0.9916620619258089, |
| "brier": 0.3885699440853336, |
| "ece": 0.028073393950339448, |
| "auto@1%": 0.3075117370892019, |
| "auto@2%": 0.3561368209255533 |
| }, |
| "q:click_target|mod:vision": { |
| "n": 3787, |
| "acc": 0.7647214153683655, |
| "nll": 0.8611364374579239, |
| "brier": 0.3207721282952025, |
| "ece": 0.02522712125449486, |
| "auto@1%": 0.2112490097702667, |
| "auto@2%": 0.3585951940850277 |
| }, |
| "q:op": { |
| "n": 9000, |
| "acc": 0.8747777777777778, |
| "nll": 0.41109552728328835, |
| "brier": 0.1881643006091161, |
| "ece": 0.027887270021343957, |
| "auto@1%": 0.4728888888888889, |
| "auto@2%": 0.6281111111111111 |
| }, |
| "q:op|mod:full": { |
| "n": 4006, |
| "acc": 0.8984023964053919, |
| "nll": 0.3386421776225732, |
| "brier": 0.15681739249583776, |
| "ece": 0.03659893292338151, |
| "auto@1%": 0.5319520718921618, |
| "auto@2%": 0.6662506240639041 |
| }, |
| "q:op|mod:vision": { |
| "n": 4994, |
| "acc": 0.855826992390869, |
| "nll": 0.4692148942718397, |
| "brier": 0.21330961777006785, |
| "ece": 0.0234003205175579, |
| "auto@1%": 0.4669603524229075, |
| "auto@2%": 0.6021225470564677 |
| }, |
| "q:scroll_target": { |
| "n": 284, |
| "acc": 0.7640845070422535, |
| "nll": 0.6508194589556844, |
| "brier": 0.292453199311092, |
| "ece": 0.05964354811915089, |
| "auto@1%": 0.528169014084507, |
| "auto@2%": 0.5633802816901409 |
| }, |
| "q:scroll_target|mod:full": { |
| "n": 183, |
| "acc": 0.8087431693989071, |
| "nll": 0.3336389861650783, |
| "brier": 0.2203359511972443, |
| "ece": 0.07096846774592282, |
| "auto@1%": 0.5956284153005464, |
| "auto@2%": 0.6065573770491803 |
| }, |
| "q:scroll_target|mod:vision": { |
| "n": 101, |
| "acc": 0.6831683168316832, |
| "nll": 1.2255127908436143, |
| "brier": 0.42312108450746927, |
| "ece": 0.12651085530483008, |
| "auto@1%": 0.42574257425742573, |
| "auto@2%": 0.42574257425742573 |
| }, |
| "q:select_target": { |
| "n": 31, |
| "acc": 0.9032258064516129, |
| "nll": 0.14205801001819574, |
| "brier": 0.10010005623509433, |
| "ece": 0.06967246525847405, |
| "auto@1%": 0.8387096774193549, |
| "auto@2%": 0.8387096774193549 |
| }, |
| "q:select_target|mod:full": { |
| "n": 31, |
| "acc": 0.9032258064516129, |
| "nll": 0.14205801001819574, |
| "brier": 0.10010005623509433, |
| "ece": 0.06967246525847405, |
| "auto@1%": 0.8387096774193549, |
| "auto@2%": 0.8387096774193549 |
| }, |
| "q:set_value_target": { |
| "n": 793, |
| "acc": 0.8978562421185372, |
| "nll": 0.3453675760949502, |
| "brier": 0.15109203911045183, |
| "ece": 0.036194069842637525, |
| "auto@1%": 0.6582597730138714, |
| "auto@2%": 0.7667087011349306 |
| }, |
| "q:set_value_target|mod:full": { |
| "n": 435, |
| "acc": 0.903448275862069, |
| "nll": 0.25961273364538356, |
| "brier": 0.14131568516459533, |
| "ece": 0.03645868658074942, |
| "auto@1%": 0.6919540229885057, |
| "auto@2%": 0.8091954022988506 |
| }, |
| "q:set_value_target|mod:vision": { |
| "n": 358, |
| "acc": 0.8910614525139665, |
| "nll": 0.44956689583115567, |
| "brier": 0.16297112840220485, |
| "ece": 0.037533499975045806, |
| "auto@1%": 0.6396648044692738, |
| "auto@2%": 0.7178770949720671 |
| }, |
| "q:subgoal_done": { |
| "n": 1977, |
| "acc": 0.9018715225088518, |
| "nll": 0.24181925484198044, |
| "brier": 0.14260286043573228, |
| "ece": 0.01994882289817495, |
| "auto@1%": 0.15174506828528073, |
| "auto@2%": 0.661608497723824 |
| }, |
| "q:subgoal_done|mod:full": { |
| "n": 1223, |
| "acc": 0.8937040065412919, |
| "nll": 0.2582274786534001, |
| "brier": 0.1535791079684229, |
| "ece": 0.021707061575793954, |
| "auto@1%": 0.18887980376124286, |
| "auto@2%": 0.6230580539656582 |
| }, |
| "q:subgoal_done|mod:vision": { |
| "n": 754, |
| "acc": 0.9151193633952255, |
| "nll": 0.21520485468101727, |
| "brier": 0.12479921224941845, |
| "ece": 0.018088822384504093, |
| "auto@1%": 0.5848806366047745, |
| "auto@2%": 0.7228116710875332 |
| } |
| }, |
| "steps": { |
| "all": { |
| "n": 9000, |
| "op_acc": 0.8747777777777778, |
| "element_top1": 0.7501586898565443, |
| "step_success": 0.694, |
| "auto@1%": 0.1801111111111111, |
| "auto@2%": 0.2767777777777778, |
| "hotkey_key_acc": 0.7934782608695652, |
| "hotkey_key_n": 644, |
| "value_acc": 0.8441890166028098, |
| "value_n": 783, |
| "step_success_strict": 0.6778888888888889 |
| }, |
| "mod:full": { |
| "n": 4006, |
| "op_acc": 0.8984023964053919, |
| "element_top1": 0.7229413384742496, |
| "step_success": 0.6749875187219171, |
| "auto@1%": 0.21742386420369444, |
| "auto@2%": 0.2825761357963055, |
| "hotkey_key_acc": 0.7575757575757576, |
| "hotkey_key_n": 165, |
| "value_acc": 0.8661971830985915, |
| "value_n": 426, |
| "step_success_strict": 0.6605092361457813 |
| }, |
| "mod:vision": { |
| "n": 4994, |
| "op_acc": 0.855826992390869, |
| "element_top1": 0.7734338200659444, |
| "step_success": 0.7092511013215859, |
| "auto@1%": 0.16219463356027233, |
| "auto@2%": 0.26892270724869843, |
| "hotkey_key_acc": 0.8058455114822547, |
| "hotkey_key_n": 479, |
| "value_acc": 0.8179271708683473, |
| "value_n": 357, |
| "step_success_strict": 0.6918301962354826 |
| }, |
| "op:CLICK": { |
| "n": 6557, |
| "op_acc": 0.9237456153728839, |
| "element_top1": 0.7305170047277718, |
| "step_success": 0.6917797773371969, |
| "auto@1%": 0.17355497941131615, |
| "auto@2%": 0.27527832850388895, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.6917797773371969 |
| }, |
| "op:DONE": { |
| "n": 160, |
| "op_acc": 0.875, |
| "element_top1": null, |
| "step_success": 0.875, |
| "auto@1%": 0.48125, |
| "auto@2%": 0.6375, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.875 |
| }, |
| "op:DOUBLE_CLICK": { |
| "n": 119, |
| "op_acc": 0.44537815126050423, |
| "element_top1": 0.7815126050420168, |
| "step_success": 0.42016806722689076, |
| "auto@1%": 0.0, |
| "auto@2%": 0.0, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.42016806722689076 |
| }, |
| "op:DRAG": { |
| "n": 137, |
| "op_acc": 0.5693430656934306, |
| "element_top1": null, |
| "step_success": 0.5693430656934306, |
| "auto@1%": 0.1897810218978102, |
| "auto@2%": 0.1897810218978102, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.5693430656934306 |
| }, |
| "op:HOTKEY": { |
| "n": 720, |
| "op_acc": 0.7819444444444444, |
| "element_top1": null, |
| "step_success": 0.7819444444444444, |
| "auto@1%": 0.5791666666666667, |
| "auto@2%": 0.6180555555555556, |
| "hotkey_key_acc": 0.7934782608695652, |
| "hotkey_key_n": 644, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.6805555555555556 |
| }, |
| "op:RIGHT_CLICK": { |
| "n": 93, |
| "op_acc": 0.5268817204301075, |
| "element_top1": 0.7419354838709677, |
| "step_success": 0.46236559139784944, |
| "auto@1%": 0.10752688172043011, |
| "auto@2%": 0.10752688172043011, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.46236559139784944 |
| }, |
| "op:SCROLL": { |
| "n": 284, |
| "op_acc": 0.7464788732394366, |
| "element_top1": 0.7640845070422535, |
| "step_success": 0.5845070422535211, |
| "auto@1%": 0.11971830985915492, |
| "auto@2%": 0.11971830985915492, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.5845070422535211 |
| }, |
| "op:SELECT": { |
| "n": 31, |
| "op_acc": 0.8387096774193549, |
| "element_top1": 0.9032258064516129, |
| "step_success": 0.7741935483870968, |
| "auto@1%": 0.6451612903225806, |
| "auto@2%": 0.6451612903225806, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.7741935483870968 |
| }, |
| "op:SET_VALUE": { |
| "n": 242, |
| "op_acc": 0.7107438016528925, |
| "element_top1": 0.9214876033057852, |
| "step_success": 0.6652892561983471, |
| "auto@1%": 0.06198347107438017, |
| "auto@2%": 0.06198347107438017, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": 0.8154506437768241, |
| "value_n": 233, |
| "step_success_strict": 0.5785123966942148 |
| }, |
| "op:TYPE_TEXT": { |
| "n": 551, |
| "op_acc": 0.8185117967332124, |
| "element_top1": 0.8874773139745916, |
| "step_success": 0.7495462794918331, |
| "auto@1%": 0.24500907441016334, |
| "auto@2%": 0.2831215970961887, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": 0.8563636363636363, |
| "value_n": 550, |
| "step_success_strict": 0.6569872958257713 |
| }, |
| "op:WAIT": { |
| "n": 106, |
| "op_acc": 0.6792452830188679, |
| "element_top1": null, |
| "step_success": 0.6792452830188679, |
| "auto@1%": 0.16037735849056603, |
| "auto@2%": 0.16037735849056603, |
| "hotkey_key_acc": null, |
| "hotkey_key_n": 0, |
| "value_acc": null, |
| "value_n": 0, |
| "step_success_strict": 0.6792452830188679 |
| } |
| } |
| }, |
| "calibration_report": { |
| "all": { |
| "n": 3771, |
| "ece_before": 0.01747626450483553, |
| "ece_after": 0.018764672656685816, |
| "nll_before": 0.5712372033403514, |
| "nll_after": 0.5699958148241562, |
| "execute_share": 0.5452134712277911, |
| "execute_error": 0.02383268482490275, |
| "escalate_share": 0.15141872182444974 |
| }, |
| "mod:full": { |
| "n": 1772, |
| "ece_before": 0.02814794127457049, |
| "ece_after": 0.02590856275493896, |
| "nll_before": 0.5519384331521392, |
| "nll_after": 0.5517056593417647, |
| "execute_share": 0.5564334085778782, |
| "execute_error": 0.026369168356997985, |
| "escalate_share": 0.14503386004514673 |
| }, |
| "mod:vision": { |
| "n": 1999, |
| "ece_before": 0.017960368637130675, |
| "ece_after": 0.021329443853398466, |
| "nll_before": 0.5883444673591168, |
| "nll_after": 0.5862089991737299, |
| "execute_share": 0.5352676338169085, |
| "execute_error": 0.021495327102803774, |
| "escalate_share": 0.1570785392696348 |
| }, |
| "qtype:choice": { |
| "n": 3375, |
| "ece_before": 0.01432174752193021, |
| "ece_after": 0.0219802066016441, |
| "nll_before": 0.6036172560395445, |
| "nll_after": 0.6031795218505003, |
| "execute_share": 0.5306666666666666, |
| "execute_error": 0.022333891680625384, |
| "escalate_share": 0.1691851851851852 |
| }, |
| "qtype:noul": { |
| "n": 396, |
| "ece_before": 0.05879165844354522, |
| "ece_after": 0.03624674956609409, |
| "nll_before": 0.2952708451085907, |
| "nll_after": 0.28718012994054226, |
| "execute_share": 0.6691919191919192, |
| "execute_error": 0.03396226415094339, |
| "escalate_share": 0.0 |
| } |
| } |
| } |
