husky8 commited on
Commit
c4fc9be
·
verified ·
1 Parent(s): f8ea613

Update the archive film, chapter index and selected timing summary

Browse files
benchmark/averages.json CHANGED
@@ -1,23 +1,23 @@
1
  {
2
- "film_revision": "57119135e10cad54d8fe97fc986dbaf722d30ffd",
3
- "film_sha256": "80737a1e6dd61517c0372167e677cabd94f08bc32968d9141e046684da172a54",
4
- "duration_s": 252.4,
5
- "chapters": 41,
6
- "webarena_ids": 34,
7
  "grades_revision": "1bde02a4c7addbbe3776762d164f57b30dcf9f23",
8
  "aggregate_definition": "Sum stock elapsed seconds divided by sum edited elapsed seconds. Each included comparison must pass in both arms.",
9
  "film_main_takes": {
10
  "selection": "One scheduled main comparison per chapter against upstream stock. Includes joint passes across several edits. Excludes additional takes and the published-search baseline.",
11
  "summary": {
12
- "pairs": 12,
13
- "stock_total_s": 258.0532383330865,
14
- "edited_total_s": 104.78757195803337,
15
- "mean_stock_s": 21.504436527757207,
16
- "mean_edited_s": 8.732297663169447,
17
- "time_speedup": 2.462632099505415,
18
- "time_reduction_pct": 59.39304128290881,
19
- "arithmetic_mean_pair_speedup": 3.168493381285156,
20
- "geometric_mean_pair_speedup": 2.9322712983326094
21
  },
22
  "pairs": [
23
  {
@@ -44,7 +44,7 @@
44
  "chapter": 3,
45
  "title": "Complete an autocomplete field",
46
  "task": "Seed 103",
47
- "environment": "BrowserGym \u00b7 MiniWoB++",
48
  "device": "M5 Max MacBook Pro",
49
  "checkpoint": "Two-row swap",
50
  "stock_s": 15.463160666055046,
@@ -54,7 +54,7 @@
54
  "chapter": 4,
55
  "title": "Reply to an email",
56
  "task": "Seed 104",
57
- "environment": "BrowserGym \u00b7 MiniWoB++",
58
  "device": "M5 Max MacBook Pro",
59
  "checkpoint": "Two-row swap",
60
  "stock_s": 26.014819292002358,
@@ -72,26 +72,16 @@
72
  },
73
  {
74
  "chapter": 15,
75
- "title": "Find the total of the latest pending order",
76
- "task": "189",
77
- "environment": "WebArena",
78
- "device": "M5 Max MacBook Pro",
79
- "checkpoint": "Published stop-row weights",
80
- "stock_s": 33.850683916010894,
81
- "edited_s": 33.2397506670095
82
- },
83
- {
84
- "chapter": 16,
85
  "title": "Find the total of the latest completed order",
86
  "task": "190",
87
  "environment": "WebArena",
88
  "device": "M5 Max MacBook Pro",
89
- "checkpoint": "Frozen pending-order weights",
90
  "stock_s": 45.501502291997895,
91
  "edited_s": 18.32999724999536
92
  },
93
  {
94
- "chapter": 22,
95
  "title": "Search for a USB Wi-Fi adapter",
96
  "task": "274",
97
  "environment": "WebArena",
@@ -101,7 +91,7 @@
101
  "edited_s": 4.72369825001806
102
  },
103
  {
104
- "chapter": 23,
105
  "title": "Search for Xbox",
106
  "task": "275",
107
  "environment": "WebArena",
@@ -111,7 +101,7 @@
111
  "edited_s": 4.95540237496607
112
  },
113
  {
114
- "chapter": 24,
115
  "title": "Search for Switch accessories",
116
  "task": "276",
117
  "environment": "WebArena",
@@ -121,7 +111,7 @@
121
  "edited_s": 5.341348875081167
122
  },
123
  {
124
- "chapter": 25,
125
  "title": "Search for iPhone 13 batteries",
126
  "task": "277",
127
  "environment": "WebArena",
@@ -131,7 +121,7 @@
131
  "edited_s": 5.821562124998309
132
  },
133
  {
134
- "chapter": 26,
135
  "title": "Search for green tea bags",
136
  "task": "278",
137
  "environment": "WebArena",
 
1
  {
2
+ "film_revision": "57e635d4bbb026b1c54017a41425ac06bd9500f6",
3
+ "film_sha256": "719c3a13b5812dc9886e828e4ffefb2f7b3242565917a6ff6eae2cd2e601ffc2",
4
+ "duration_s": 246.05,
5
+ "chapters": 40,
6
+ "webarena_ids": 33,
7
  "grades_revision": "1bde02a4c7addbbe3776762d164f57b30dcf9f23",
8
  "aggregate_definition": "Sum stock elapsed seconds divided by sum edited elapsed seconds. Each included comparison must pass in both arms.",
9
  "film_main_takes": {
10
  "selection": "One scheduled main comparison per chapter against upstream stock. Includes joint passes across several edits. Excludes additional takes and the published-search baseline.",
11
  "summary": {
12
+ "pairs": 11,
13
+ "stock_total_s": 224.2025544170756,
14
+ "edited_total_s": 71.54782129102387,
15
+ "mean_stock_s": 20.382050401552327,
16
+ "mean_edited_s": 6.504347390093079,
17
+ "time_speedup": 3.1336042156353296,
18
+ "time_reduction_pct": 68.08786524442262,
19
+ "arithmetic_mean_pair_speedup": 3.363958270690793,
20
+ "geometric_mean_pair_speedup": 3.2281833349070617
21
  },
22
  "pairs": [
23
  {
 
44
  "chapter": 3,
45
  "title": "Complete an autocomplete field",
46
  "task": "Seed 103",
47
+ "environment": "BrowserGym · MiniWoB++",
48
  "device": "M5 Max MacBook Pro",
49
  "checkpoint": "Two-row swap",
50
  "stock_s": 15.463160666055046,
 
54
  "chapter": 4,
55
  "title": "Reply to an email",
56
  "task": "Seed 104",
57
+ "environment": "BrowserGym · MiniWoB++",
58
  "device": "M5 Max MacBook Pro",
59
  "checkpoint": "Two-row swap",
60
  "stock_s": 26.014819292002358,
 
72
  },
73
  {
74
  "chapter": 15,
 
 
 
 
 
 
 
 
 
 
75
  "title": "Find the total of the latest completed order",
76
  "task": "190",
77
  "environment": "WebArena",
78
  "device": "M5 Max MacBook Pro",
79
+ "checkpoint": "Frozen click and stop weights",
80
  "stock_s": 45.501502291997895,
81
  "edited_s": 18.32999724999536
82
  },
83
  {
84
+ "chapter": 21,
85
  "title": "Search for a USB Wi-Fi adapter",
86
  "task": "274",
87
  "environment": "WebArena",
 
91
  "edited_s": 4.72369825001806
92
  },
93
  {
94
+ "chapter": 22,
95
  "title": "Search for Xbox",
96
  "task": "275",
97
  "environment": "WebArena",
 
101
  "edited_s": 4.95540237496607
102
  },
103
  {
104
+ "chapter": 23,
105
  "title": "Search for Switch accessories",
106
  "task": "276",
107
  "environment": "WebArena",
 
111
  "edited_s": 5.341348875081167
112
  },
113
  {
114
+ "chapter": 24,
115
  "title": "Search for iPhone 13 batteries",
116
  "task": "277",
117
  "environment": "WebArena",
 
121
  "edited_s": 5.821562124998309
122
  },
123
  {
124
+ "chapter": 25,
125
  "title": "Search for green tea bags",
126
  "task": "278",
127
  "environment": "WebArena",
benchmark/browser-benchmark-in-order.mp4 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:80737a1e6dd61517c0372167e677cabd94f08bc32968d9141e046684da172a54
3
- size 9221640
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:719c3a13b5812dc9886e828e4ffefb2f7b3242565917a6ff6eae2cd2e601ffc2
3
+ size 9852600
benchmark/camera-sources.zip CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d8c142dcb313a717a7104cc90bc16c18a1f1e6c48551fa89cf83e8b5f92d2028
3
  size 1546421
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ed80961967716e3e196ac0d75b5093281c73514925f079e6172c5bb54f720599
3
  size 1546421
benchmark/chapters.json CHANGED
@@ -1,15 +1,15 @@
1
  {
2
- "sha256": "80737a1e6dd61517c0372167e677cabd94f08bc32968d9141e046684da172a54",
3
  "fps": 20,
4
  "resolution": [
5
  1200,
6
  900
7
  ],
8
- "duration_s": 252.4,
9
- "task_count": 41,
10
  "opening_s": 6.15,
11
- "closing_start_s": 248.9,
12
- "selection": "One actually recorded case per distinct task. Form then cart, five MiniWoB++ tasks, then every graded WebArena ID in numeric order. Comparison arms are from the same case and checkpoint study. The complete graded time is independent of the excerpt length.",
13
  "chapters": [
14
  {
15
  "number": 1,
@@ -97,7 +97,7 @@
97
  "number": 3,
98
  "start_s": 14.65,
99
  "duration_s": 4.75,
100
- "environment": "BrowserGym \u00b7 MiniWoB++",
101
  "task": "Seed 103",
102
  "title": "Complete an autocomplete field",
103
  "shared_elapsed_start_s": 0,
@@ -138,7 +138,7 @@
138
  "number": 4,
139
  "start_s": 19.4,
140
  "duration_s": 4.75,
141
- "environment": "BrowserGym \u00b7 MiniWoB++",
142
  "task": "Seed 104",
143
  "title": "Reply to an email",
144
  "shared_elapsed_start_s": 1.9337746249977499,
@@ -179,7 +179,7 @@
179
  "number": 5,
180
  "start_s": 24.15,
181
  "duration_s": 2.95,
182
- "environment": "BrowserGym \u00b7 MiniWoB++",
183
  "task": "Seed 101",
184
  "title": "Like every post from one account",
185
  "shared_elapsed_start_s": null,
@@ -220,7 +220,7 @@
220
  "number": 6,
221
  "start_s": 27.1,
222
  "duration_s": 2.95,
223
- "environment": "BrowserGym \u00b7 MiniWoB++",
224
  "task": "Seed 101",
225
  "title": "Pick a date",
226
  "shared_elapsed_start_s": null,
@@ -261,7 +261,7 @@
261
  "number": 7,
262
  "start_s": 30.05,
263
  "duration_s": 2.95,
264
- "environment": "BrowserGym \u00b7 MiniWoB++",
265
  "task": "Seed 101",
266
  "title": "Book the shortest one-way flight",
267
  "shared_elapsed_start_s": null,
@@ -406,7 +406,7 @@
406
  "grader_status": "failure"
407
  },
408
  {
409
- "arm": "Frozen pending-order weights",
410
  "elapsed_s": 0.7192069169832394,
411
  "passed": false,
412
  "device": "M5 Max MacBook Pro",
@@ -910,286 +910,7 @@
910
  {
911
  "number": 15,
912
  "start_s": 75.1,
913
- "duration_s": 21.45,
914
- "environment": "WebArena",
915
- "task": "189",
916
- "title": "Find the total of the latest pending order",
917
- "shared_elapsed_start_s": 29.8897506670095,
918
- "arms": [
919
- {
920
- "arm": "Stock Inkling 2 Mini",
921
- "elapsed_s": 33.850683916010894,
922
- "passed": true,
923
- "device": "M5 Max MacBook Pro",
924
- "receipt_sha256": "9dd3fed47db798b1f67d44f794379eda89f3c949e8a10c0a65ced54dc05f5512",
925
- "receipt": "189-original.json",
926
- "recording": {
927
- "kind": "native browser recording",
928
- "source_sha256": "9d35421225e57628bada9376e92528ac7a677e5629efde93d4c44aa7661dd037",
929
- "start_s": 29.89041875102557,
930
- "source_duration_s": 33.85,
931
- "available_frames": 79
932
- },
933
- "grader_status": "success"
934
- },
935
- {
936
- "arm": "Published stop-row weights",
937
- "elapsed_s": 33.2397506670095,
938
- "passed": true,
939
- "device": "M5 Max MacBook Pro",
940
- "receipt_sha256": "e26130900a6a30683ec77ad0f32b975f5f6f4be9b25df86c9e5caf1c800f16c1",
941
- "receipt": "189-edited.json",
942
- "recording": {
943
- "kind": "native browser recording",
944
- "source_sha256": "4b91334778448d041a1b3f59fa2188cd2cc44dbb0c14d5f8dfce324e6a9c8072",
945
- "start_s": 29.89049062503036,
946
- "source_duration_s": 33.45,
947
- "available_frames": 69
948
- },
949
- "grader_status": "success"
950
- }
951
- ],
952
- "additional_takes": [
953
- {
954
- "start_s": 79.15,
955
- "duration_s": 2.65,
956
- "shared_elapsed_start_s": 2.2052947090240194,
957
- "arms": [
958
- {
959
- "arm": "Stock Inkling 2 Mini",
960
- "elapsed_s": 23.85580195800867,
961
- "passed": true,
962
- "device": "M5 Max MacBook Pro",
963
- "receipt_sha256": "b1634b0f8243545d4097ffe4ca933354bde835338404e687caf3aae5c2018efc",
964
- "receipt": "189-original.json",
965
- "recording": {
966
- "kind": "native browser recording",
967
- "source_sha256": "1ee1ac955ae3bda5d4734587196fc9c44130579aa394ed8db92f8f008b490ab0",
968
- "start_s": 2.205978625989519,
969
- "source_duration_s": 23.85,
970
- "available_frames": 53
971
- },
972
- "grader_status": "success"
973
- },
974
- {
975
- "arm": "Pending click + stop edit",
976
- "elapsed_s": 4.1552947090240195,
977
- "passed": true,
978
- "device": "M5 Max MacBook Pro",
979
- "receipt_sha256": "e6c644893422e06b9a286c7b2a9ddf26a1cde522b86c712bbf9ca2de49158152",
980
- "receipt": "189-edited.json",
981
- "recording": {
982
- "kind": "native browser recording",
983
- "source_sha256": "3cfe1ad7af674a69fbeca895f5c882f54b60d2cf811bcb95d1bb15552a817703",
984
- "start_s": 2.2063615839695556,
985
- "source_duration_s": 4.35,
986
- "available_frames": 40
987
- },
988
- "grader_status": "success"
989
- }
990
- ]
991
- },
992
- {
993
- "start_s": 81.8,
994
- "duration_s": 2.95,
995
- "shared_elapsed_start_s": 1.3365153330378232,
996
- "arms": [
997
- {
998
- "arm": "Stock Inkling 2 Mini",
999
- "elapsed_s": 12.520012957975268,
1000
- "passed": true,
1001
- "device": "RTX 5090",
1002
- "receipt_sha256": "637ddc2bcc9efed7959e36faf0409d31339d847d40b7cdc22e4a90a383178681",
1003
- "receipt": "189-original.json",
1004
- "recording": {
1005
- "kind": "native browser recording",
1006
- "source_sha256": "d6dc8ea68b740d7ee8708a1a761b95a3f56f6a5ea020f8bdecde4fb3ba82bf4b",
1007
- "start_s": 1.3377253330545498,
1008
- "source_duration_s": 15.75,
1009
- "available_frames": 59
1010
- },
1011
- "grader_status": "success"
1012
- },
1013
- {
1014
- "arm": "Frozen pending-order weights",
1015
- "elapsed_s": 3.5865153330378234,
1016
- "passed": true,
1017
- "device": "RTX 5090",
1018
- "receipt_sha256": "ad87f98adb0c4f50778203c7ef93c0da8ce1340f16a480be81ab0a965fd84639",
1019
- "receipt": "189-edited.json",
1020
- "recording": {
1021
- "kind": "native browser recording",
1022
- "source_sha256": "7dac24884bef50b5afebd54a69aeeedd5fdd67abbad3e484f7ddc8e978b055ef",
1023
- "start_s": 1.3375340419588608,
1024
- "source_duration_s": 4.25,
1025
- "available_frames": 46
1026
- },
1027
- "grader_status": "success"
1028
- }
1029
- ]
1030
- },
1031
- {
1032
- "start_s": 84.75,
1033
- "duration_s": 2.95,
1034
- "shared_elapsed_start_s": 3.054031042032875,
1035
- "arms": [
1036
- {
1037
- "arm": "Stock Inkling 2 Mini",
1038
- "elapsed_s": 33.172848040936515,
1039
- "passed": true,
1040
- "device": "M5 Max MacBook Pro",
1041
- "receipt_sha256": "6065551e3f226bb44ba2969600abebd38fd0b913cf16f5f4d94fab45ce14fd96",
1042
- "receipt": "189-original.json",
1043
- "recording": {
1044
- "kind": "native browser recording",
1045
- "source_sha256": "c022881a4a31ff876167f106dba50a18a847ce86de0f5657f322ea6ccf10a997",
1046
- "start_s": 3.0548228340921924,
1047
- "source_duration_s": 33.2,
1048
- "available_frames": 59
1049
- },
1050
- "grader_status": "success"
1051
- },
1052
- {
1053
- "arm": "Completed-order stop edit",
1054
- "elapsed_s": 5.304031042032875,
1055
- "passed": true,
1056
- "device": "M5 Max MacBook Pro",
1057
- "receipt_sha256": "dba900de9b60dde45bb9417e4d62258b90e9d90cecd61dba335258157244dd16",
1058
- "receipt": "189-edited.json",
1059
- "recording": {
1060
- "kind": "native browser recording",
1061
- "source_sha256": "0cd40fcf46fd1d432df3e67061b94ae37d77bd390c03a0c904bfcccb8b5d4ded",
1062
- "start_s": 3.05476329207886,
1063
- "source_duration_s": 5.5,
1064
- "available_frames": 46
1065
- },
1066
- "grader_status": "success"
1067
- }
1068
- ]
1069
- },
1070
- {
1071
- "start_s": 87.7,
1072
- "duration_s": 2.95,
1073
- "shared_elapsed_start_s": 3.0029928329167888,
1074
- "arms": [
1075
- {
1076
- "arm": "Stock Inkling 2 Mini",
1077
- "elapsed_s": 33.1291674580425,
1078
- "passed": true,
1079
- "device": "M5 Max MacBook Pro",
1080
- "receipt_sha256": "32a8f504c05053537034dfbb42a8e60a8001f905b80217e28e9d08bc37ef638c",
1081
- "receipt": "189-original.json",
1082
- "recording": {
1083
- "kind": "native browser recording",
1084
- "source_sha256": "1ea97126fa12b6406a6cb817782c3a92b22b0e8085a4c1ef4cb276bf0dab9e31",
1085
- "start_s": 3.0037926249206066,
1086
- "source_duration_s": 33.1,
1087
- "available_frames": 59
1088
- },
1089
- "grader_status": "success"
1090
- },
1091
- {
1092
- "arm": "Cross-device order stop",
1093
- "elapsed_s": 5.252992832916789,
1094
- "passed": true,
1095
- "device": "M5 Max MacBook Pro",
1096
- "receipt_sha256": "13aaad04bde4944080119ae795b247c8d5da61c67a8946ae6d563b83e58ec72e",
1097
- "receipt": "189-edited.json",
1098
- "recording": {
1099
- "kind": "native browser recording",
1100
- "source_sha256": "b1bf7f929afa1478e03a8d2aa16c2b69d4631cbcd4d8fb469b17e21bb969f8c9",
1101
- "start_s": 3.0037339159753174,
1102
- "source_duration_s": 5.45,
1103
- "available_frames": 46
1104
- },
1105
- "grader_status": "success"
1106
- }
1107
- ]
1108
- },
1109
- {
1110
- "start_s": 90.65,
1111
- "duration_s": 2.95,
1112
- "shared_elapsed_start_s": 0.6171108750859273,
1113
- "arms": [
1114
- {
1115
- "arm": "Stock Inkling 2 Mini",
1116
- "elapsed_s": 6.77029658399988,
1117
- "passed": true,
1118
- "device": "RTX 5090",
1119
- "receipt_sha256": "0cea03e6cc7571e55dc7bf0fcfdae72c74474be28a708ed8c8a8ef89d405058b",
1120
- "receipt": "189-original.json",
1121
- "recording": {
1122
- "kind": "native browser recording",
1123
- "source_sha256": "4a5813e8ccab420942171b54d76c3c8c6979f2238075a2f1615da63647d27d5f",
1124
- "start_s": 0.6180617911741135,
1125
- "source_duration_s": 6.8,
1126
- "available_frames": 59
1127
- },
1128
- "grader_status": "success"
1129
- },
1130
- {
1131
- "arm": "Order-history routing trial",
1132
- "elapsed_s": 2.8671108750859275,
1133
- "passed": true,
1134
- "device": "RTX 5090",
1135
- "receipt_sha256": "38994d8c0ccc902ff2a20f85752d8996348439c8f2de2fed16ac66e0e130ccb8",
1136
- "receipt": "189-edited.json",
1137
- "recording": {
1138
- "kind": "native browser recording",
1139
- "source_sha256": "77e0dd320302f1be3b54c6052cb774f2f3cf67981508c2f3bb041138002d4d0c",
1140
- "start_s": 0.6183844580082221,
1141
- "source_duration_s": 3.5,
1142
- "available_frames": 46
1143
- },
1144
- "grader_status": "success"
1145
- }
1146
- ]
1147
- },
1148
- {
1149
- "start_s": 93.6,
1150
- "duration_s": 2.95,
1151
- "shared_elapsed_start_s": 3.0999951249687,
1152
- "arms": [
1153
- {
1154
- "arm": "Stock Inkling 2 Mini",
1155
- "elapsed_s": 24.47346133296378,
1156
- "passed": true,
1157
- "device": "M5 Max MacBook Pro",
1158
- "receipt_sha256": "4bd09806c228658ab91f939594cf2394cbc6a88ae34782ce77d31b3fe1a67e76",
1159
- "receipt": "189-original.json",
1160
- "recording": {
1161
- "kind": "native browser recording",
1162
- "source_sha256": "2b9b390beb267c67ef41ce4fed6b5896d46cae845ecb7d0410b58873ec2ff49b",
1163
- "start_s": 3.1007044999860227,
1164
- "source_duration_s": 24.45,
1165
- "available_frames": 59
1166
- },
1167
- "grader_status": "success"
1168
- },
1169
- {
1170
- "arm": "Order-history routing trial",
1171
- "elapsed_s": 5.3499951249687,
1172
- "passed": true,
1173
- "device": "M5 Max MacBook Pro",
1174
- "receipt_sha256": "2b0995cf755f61451029ee24ada6b0a5674e282ac40d3ba4a86addaa3e575671",
1175
- "receipt": "189-edited.json",
1176
- "recording": {
1177
- "kind": "native browser recording",
1178
- "source_sha256": "bc0f54dce5dc9be7ef20fff1a96e6aec22b02998c7e389d82b4707a07e59e197",
1179
- "start_s": 3.1010409999871626,
1180
- "source_duration_s": 5.55,
1181
- "available_frames": 46
1182
- },
1183
- "grader_status": "success"
1184
- }
1185
- ]
1186
- }
1187
- ]
1188
- },
1189
- {
1190
- "number": 16,
1191
- "start_s": 96.55,
1192
- "duration_s": 27.25,
1193
  "environment": "WebArena",
1194
  "task": "190",
1195
  "title": "Find the total of the latest completed order",
@@ -1212,7 +933,7 @@
1212
  "grader_status": "success"
1213
  },
1214
  {
1215
- "arm": "Frozen pending-order weights",
1216
  "elapsed_s": 18.32999724999536,
1217
  "passed": true,
1218
  "device": "M5 Max MacBook Pro",
@@ -1230,7 +951,7 @@
1230
  ],
1231
  "additional_takes": [
1232
  {
1233
- "start_s": 100.2,
1234
  "duration_s": 2.95,
1235
  "shared_elapsed_start_s": 2.940150791895576,
1236
  "arms": [
@@ -1269,7 +990,7 @@
1269
  ]
1270
  },
1271
  {
1272
- "start_s": 103.15,
1273
  "duration_s": 2.95,
1274
  "shared_elapsed_start_s": 6.681119083077647,
1275
  "arms": [
@@ -1308,7 +1029,7 @@
1308
  ]
1309
  },
1310
  {
1311
- "start_s": 106.1,
1312
  "duration_s": 2.95,
1313
  "shared_elapsed_start_s": null,
1314
  "arms": [
@@ -1347,7 +1068,7 @@
1347
  ]
1348
  },
1349
  {
1350
- "start_s": 109.05,
1351
  "duration_s": 2.95,
1352
  "shared_elapsed_start_s": 2.8771139580057934,
1353
  "arms": [
@@ -1386,7 +1107,7 @@
1386
  ]
1387
  },
1388
  {
1389
- "start_s": 112.0,
1390
  "duration_s": 2.95,
1391
  "shared_elapsed_start_s": 1.092322333017364,
1392
  "arms": [
@@ -1425,7 +1146,7 @@
1425
  ]
1426
  },
1427
  {
1428
- "start_s": 114.95,
1429
  "duration_s": 2.95,
1430
  "shared_elapsed_start_s": 2.38902004098054,
1431
  "arms": [
@@ -1464,7 +1185,7 @@
1464
  ]
1465
  },
1466
  {
1467
- "start_s": 117.9,
1468
  "duration_s": 2.95,
1469
  "shared_elapsed_start_s": 0.5983604160137472,
1470
  "arms": [
@@ -1503,7 +1224,46 @@
1503
  ]
1504
  },
1505
  {
1506
- "start_s": 120.85,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1507
  "duration_s": 2.95,
1508
  "shared_elapsed_start_s": 3.5121810829732567,
1509
  "arms": [
@@ -1540,12 +1300,51 @@
1540
  "grader_status": "success"
1541
  }
1542
  ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1543
  }
1544
  ]
1545
  },
1546
  {
1547
- "number": 17,
1548
- "start_s": 123.8,
1549
  "duration_s": 8.1,
1550
  "environment": "WebArena",
1551
  "task": "191",
@@ -1587,7 +1386,7 @@
1587
  ],
1588
  "additional_takes": [
1589
  {
1590
- "start_s": 126.0,
1591
  "duration_s": 2.95,
1592
  "shared_elapsed_start_s": null,
1593
  "arms": [
@@ -1626,7 +1425,7 @@
1626
  ]
1627
  },
1628
  {
1629
- "start_s": 128.95,
1630
  "duration_s": 2.95,
1631
  "shared_elapsed_start_s": null,
1632
  "arms": [
@@ -1667,9 +1466,9 @@
1667
  ]
1668
  },
1669
  {
1670
- "number": 18,
1671
- "start_s": 131.9,
1672
- "duration_s": 5.1,
1673
  "environment": "WebArena",
1674
  "task": "231",
1675
  "title": "Find a canceled order number",
@@ -1708,8 +1507,8 @@
1708
  ],
1709
  "additional_takes": [
1710
  {
1711
- "start_s": 134.1,
1712
- "duration_s": 2.9,
1713
  "shared_elapsed_start_s": null,
1714
  "arms": [
1715
  {
@@ -1722,9 +1521,9 @@
1722
  "recording": {
1723
  "kind": "native browser recording",
1724
  "source_sha256": "3aea1b0be0d4e8bd1125a74b743474e3f4723109d8f6ecb2b6767bc86ce17ee2",
1725
- "start_s": 28.477904957975262,
1726
  "source_duration_s": 32.35,
1727
- "available_frames": 58
1728
  },
1729
  "grader_status": "failure"
1730
  },
@@ -1738,9 +1537,9 @@
1738
  "recording": {
1739
  "kind": "native browser recording",
1740
  "source_sha256": "fd230a0bf607e34cda779e74694430fb50e01c81b4fea13b87cecdac971afb6b",
1741
- "start_s": 2.222678832965903,
1742
  "source_duration_s": 5.3,
1743
- "available_frames": 58
1744
  },
1745
  "grader_status": "success"
1746
  }
@@ -1749,8 +1548,8 @@
1749
  ]
1750
  },
1751
  {
1752
- "number": 19,
1753
- "start_s": 137.0,
1754
  "duration_s": 2.2,
1755
  "environment": "WebArena",
1756
  "task": "260",
@@ -1790,8 +1589,8 @@
1790
  ]
1791
  },
1792
  {
1793
- "number": 20,
1794
- "start_s": 139.2,
1795
  "duration_s": 2.2,
1796
  "environment": "WebArena",
1797
  "task": "269",
@@ -1831,12 +1630,12 @@
1831
  ]
1832
  },
1833
  {
1834
- "number": 21,
1835
- "start_s": 141.4,
1836
  "duration_s": 2.2,
1837
  "environment": "WebArena",
1838
  "task": "270",
1839
- "title": "Filter men\u2019s shoes below $30",
1840
  "shared_elapsed_start_s": null,
1841
  "arms": [
1842
  {
@@ -1872,9 +1671,9 @@
1872
  ]
1873
  },
1874
  {
1875
- "number": 22,
1876
- "start_s": 143.6,
1877
- "duration_s": 7.65,
1878
  "environment": "WebArena",
1879
  "task": "274",
1880
  "title": "Search for a USB Wi-Fi adapter",
@@ -1915,9 +1714,9 @@
1915
  ],
1916
  "additional_takes": [
1917
  {
1918
- "start_s": 148.35,
1919
- "duration_s": 2.9,
1920
- "shared_elapsed_start_s": 3.1853435839992015,
1921
  "arms": [
1922
  {
1923
  "arm": "Stock Inkling 2 Mini",
@@ -1929,9 +1728,9 @@
1929
  "recording": {
1930
  "kind": "native browser recording",
1931
  "source_sha256": "af12cc9808d1e46d8a9c2c52e2e5a547b9e1345a484eec137aa3090d7aa8f212",
1932
- "start_s": 3.1860501260496674,
1933
  "source_duration_s": 14.6,
1934
- "available_frames": 58
1935
  },
1936
  "grader_status": "success"
1937
  },
@@ -1945,9 +1744,9 @@
1945
  "recording": {
1946
  "kind": "native browser recording",
1947
  "source_sha256": "e897298b69c97db8c118e4c2317b0c5f90298da7dc204658cf5e7d67cf008e52",
1948
- "start_s": 3.1860644170781596,
1949
  "source_duration_s": 6.05,
1950
- "available_frames": 45
1951
  },
1952
  "grader_status": "success"
1953
  }
@@ -1956,9 +1755,9 @@
1956
  ]
1957
  },
1958
  {
1959
- "number": 23,
1960
- "start_s": 151.25,
1961
- "duration_s": 17.65,
1962
  "environment": "WebArena",
1963
  "task": "275",
1964
  "title": "Search for Xbox",
@@ -1999,9 +1798,9 @@
1999
  ],
2000
  "additional_takes": [
2001
  {
2002
- "start_s": 157.3,
2003
- "duration_s": 2.9,
2004
- "shared_elapsed_start_s": 2.8089218330103902,
2005
  "arms": [
2006
  {
2007
  "arm": "Stock Inkling 2 Mini",
@@ -2013,9 +1812,9 @@
2013
  "recording": {
2014
  "kind": "native browser recording",
2015
  "source_sha256": "90909532e5e7a03ef11e431f670b650c64adc4b87ed6173b91b4474a451e325d",
2016
- "start_s": 2.809686042089015,
2017
  "source_duration_s": 5.3,
2018
- "available_frames": 45
2019
  },
2020
  "grader_status": "success"
2021
  },
@@ -2029,18 +1828,18 @@
2029
  "recording": {
2030
  "kind": "native browser recording",
2031
  "source_sha256": "d126217723fea7e12c378ec0e931a570d40178157550c328b307bbb8b58a935e",
2032
- "start_s": 2.8094927080208434,
2033
  "source_duration_s": 7.1,
2034
- "available_frames": 58
2035
  },
2036
  "grader_status": "success"
2037
  }
2038
  ]
2039
  },
2040
  {
2041
- "start_s": 160.2,
2042
- "duration_s": 2.9,
2043
- "shared_elapsed_start_s": 2.820188458962366,
2044
  "arms": [
2045
  {
2046
  "arm": "Stock Inkling 2 Mini",
@@ -2052,9 +1851,9 @@
2052
  "recording": {
2053
  "kind": "native browser recording",
2054
  "source_sha256": "14429436b695064bb95a4db5ad46cd2f1c92e6a2283c2aa9175d0cdb19be67fd",
2055
- "start_s": 2.821443958976306,
2056
  "source_duration_s": 5.05,
2057
- "available_frames": 44
2058
  },
2059
  "grader_status": "success"
2060
  },
@@ -2068,18 +1867,18 @@
2068
  "recording": {
2069
  "kind": "native browser recording",
2070
  "source_sha256": "524f90fa5fa0edb6001e43e3e61ed5b73c053ba1139d7b494a0ff7eb185c340e",
2071
- "start_s": 2.8214176670415325,
2072
  "source_duration_s": 7.0,
2073
- "available_frames": 58
2074
  },
2075
  "grader_status": "success"
2076
  }
2077
  ]
2078
  },
2079
  {
2080
- "start_s": 163.1,
2081
- "duration_s": 2.9,
2082
- "shared_elapsed_start_s": 0.8024331250460819,
2083
  "arms": [
2084
  {
2085
  "arm": "Stock Inkling 2 Mini",
@@ -2091,9 +1890,9 @@
2091
  "recording": {
2092
  "kind": "native browser recording",
2093
  "source_sha256": "3d8f81550117b42ee36ada4ccd70dff5808fd4c6c9bf9685eb305daa60e4ffa7",
2094
- "start_s": 0.8034510829718784,
2095
  "source_duration_s": 4.95,
2096
- "available_frames": 58
2097
  },
2098
  "grader_status": "success"
2099
  },
@@ -2107,18 +1906,57 @@
2107
  "recording": {
2108
  "kind": "native browser recording",
2109
  "source_sha256": "993a915623f7851110f0178c347bd5ae29ad90318f9e9c1c9ace7c0bf53aff2a",
2110
- "start_s": 0.8031419170554728,
2111
  "source_duration_s": 4.1,
2112
- "available_frames": 45
2113
  },
2114
  "grader_status": "success"
2115
  }
2116
  ]
2117
  },
2118
  {
2119
- "start_s": 166.0,
2120
- "duration_s": 2.9,
2121
- "shared_elapsed_start_s": 2.9194580829469485,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2122
  "arms": [
2123
  {
2124
  "arm": "Stock Inkling 2 Mini",
@@ -2130,9 +1968,9 @@
2130
  "recording": {
2131
  "kind": "native browser recording",
2132
  "source_sha256": "00c066740a18093c3dab6ab4a99c066338f6c2e60a0d3675214d560b8a216371",
2133
- "start_s": 2.9201725419377906,
2134
  "source_duration_s": 21.05,
2135
- "available_frames": 58
2136
  },
2137
  "grader_status": "success"
2138
  },
@@ -2146,9 +1984,48 @@
2146
  "recording": {
2147
  "kind": "native browser recording",
2148
  "source_sha256": "c3836f0939f129721c89b1c4493bc3339533ffd48694876bc341252d06cbca13",
2149
- "start_s": 2.920131207979284,
2150
  "source_duration_s": 5.6,
2151
- "available_frames": 45
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2152
  },
2153
  "grader_status": "success"
2154
  }
@@ -2157,9 +2034,9 @@
2157
  ]
2158
  },
2159
  {
2160
- "number": 24,
2161
- "start_s": 168.9,
2162
- "duration_s": 10.7,
2163
  "environment": "WebArena",
2164
  "task": "276",
2165
  "title": "Search for Switch accessories",
@@ -2200,9 +2077,9 @@
2200
  ],
2201
  "additional_takes": [
2202
  {
2203
- "start_s": 173.65,
2204
- "duration_s": 3.0,
2205
- "shared_elapsed_start_s": 1.39850212498568,
2206
  "arms": [
2207
  {
2208
  "arm": "Stock Inkling 2 Mini",
@@ -2214,9 +2091,9 @@
2214
  "recording": {
2215
  "kind": "native browser recording",
2216
  "source_sha256": "3981b646f3ef012a477b5324d53c26779b4d267bd4328d4668097d98fd59b05a",
2217
- "start_s": 1.3991462500300258,
2218
  "source_duration_s": 5.55,
2219
- "available_frames": 60
2220
  }
2221
  },
2222
  {
@@ -2229,15 +2106,15 @@
2229
  "recording": {
2230
  "kind": "native browser recording",
2231
  "source_sha256": "eb3e5fcb71a4735f73bf19c46aea6f60f27fd7f6df72236f44b18d133cb47950",
2232
- "start_s": 1.3991312080062925,
2233
  "source_duration_s": 4.7,
2234
- "available_frames": 47
2235
  }
2236
  }
2237
  ]
2238
  },
2239
  {
2240
- "start_s": 176.65,
2241
  "duration_s": 2.95,
2242
  "shared_elapsed_start_s": 3.2479067909298465,
2243
  "arms": [
@@ -2278,8 +2155,8 @@
2278
  ]
2279
  },
2280
  {
2281
- "number": 25,
2282
- "start_s": 179.6,
2283
  "duration_s": 7.7,
2284
  "environment": "WebArena",
2285
  "task": "277",
@@ -2321,7 +2198,7 @@
2321
  ],
2322
  "additional_takes": [
2323
  {
2324
- "start_s": 184.35,
2325
  "duration_s": 2.95,
2326
  "shared_elapsed_start_s": 3.629690458998084,
2327
  "arms": [
@@ -2362,8 +2239,8 @@
2362
  ]
2363
  },
2364
  {
2365
- "number": 26,
2366
- "start_s": 187.3,
2367
  "duration_s": 13.3,
2368
  "environment": "WebArena",
2369
  "task": "278",
@@ -2405,7 +2282,7 @@
2405
  ],
2406
  "additional_takes": [
2407
  {
2408
- "start_s": 192.05,
2409
  "duration_s": 3.05,
2410
  "shared_elapsed_start_s": null,
2411
  "arms": [
@@ -2444,7 +2321,7 @@
2444
  ]
2445
  },
2446
  {
2447
- "start_s": 195.1,
2448
  "duration_s": 2.55,
2449
  "shared_elapsed_start_s": 3.232748709013686,
2450
  "arms": [
@@ -2483,7 +2360,7 @@
2483
  ]
2484
  },
2485
  {
2486
- "start_s": 197.65,
2487
  "duration_s": 2.95,
2488
  "shared_elapsed_start_s": 3.86688183399383,
2489
  "arms": [
@@ -2524,8 +2401,8 @@
2524
  ]
2525
  },
2526
  {
2527
- "number": 27,
2528
- "start_s": 200.6,
2529
  "duration_s": 2.2,
2530
  "environment": "WebArena",
2531
  "task": "279",
@@ -2567,12 +2444,12 @@
2567
  ]
2568
  },
2569
  {
2570
- "number": 28,
2571
- "start_s": 202.8,
2572
- "duration_s": 11.05,
2573
  "environment": "WebArena",
2574
  "task": "283",
2575
- "title": "Find Xbox controllers released in 2020\u20132021",
2576
  "shared_elapsed_start_s": null,
2577
  "arms": [
2578
  {
@@ -2610,7 +2487,7 @@
2610
  ],
2611
  "additional_takes": [
2612
  {
2613
- "start_s": 205.0,
2614
  "duration_s": 2.95,
2615
  "shared_elapsed_start_s": null,
2616
  "arms": [
@@ -2649,7 +2526,7 @@
2649
  ]
2650
  },
2651
  {
2652
- "start_s": 207.95,
2653
  "duration_s": 2.95,
2654
  "shared_elapsed_start_s": null,
2655
  "arms": [
@@ -2688,7 +2565,7 @@
2688
  ]
2689
  },
2690
  {
2691
- "start_s": 210.9,
2692
  "duration_s": 2.95,
2693
  "shared_elapsed_start_s": null,
2694
  "arms": [
@@ -2725,12 +2602,51 @@
2725
  "grader_status": "failure"
2726
  }
2727
  ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2728
  }
2729
  ]
2730
  },
2731
  {
2732
- "number": 29,
2733
- "start_s": 213.85,
2734
  "duration_s": 2.2,
2735
  "environment": "WebArena",
2736
  "task": "313",
@@ -2770,8 +2686,8 @@
2770
  ]
2771
  },
2772
  {
2773
- "number": 30,
2774
- "start_s": 216.05,
2775
  "duration_s": 2.2,
2776
  "environment": "WebArena",
2777
  "task": "324",
@@ -2811,8 +2727,8 @@
2811
  ]
2812
  },
2813
  {
2814
- "number": 31,
2815
- "start_s": 218.25,
2816
  "duration_s": 2.2,
2817
  "environment": "WebArena",
2818
  "task": "358",
@@ -2852,8 +2768,8 @@
2852
  ]
2853
  },
2854
  {
2855
- "number": 32,
2856
- "start_s": 220.45,
2857
  "duration_s": 2.2,
2858
  "environment": "WebArena",
2859
  "task": "399",
@@ -2893,8 +2809,8 @@
2893
  ]
2894
  },
2895
  {
2896
- "number": 33,
2897
- "start_s": 222.65,
2898
  "duration_s": 2.2,
2899
  "environment": "WebArena",
2900
  "task": "404",
@@ -2934,8 +2850,8 @@
2934
  ]
2935
  },
2936
  {
2937
- "number": 34,
2938
- "start_s": 224.85,
2939
  "duration_s": 2.2,
2940
  "environment": "WebArena",
2941
  "task": "431",
@@ -3017,8 +2933,8 @@
3017
  ]
3018
  },
3019
  {
3020
- "number": 35,
3021
- "start_s": 227.05,
3022
  "duration_s": 3.55,
3023
  "environment": "WebArena",
3024
  "task": "432",
@@ -3088,8 +3004,8 @@
3088
  ]
3089
  },
3090
  {
3091
- "number": 36,
3092
- "start_s": 230.6,
3093
  "duration_s": 2.2,
3094
  "environment": "WebArena",
3095
  "task": "516",
@@ -3129,8 +3045,8 @@
3129
  ]
3130
  },
3131
  {
3132
- "number": 37,
3133
- "start_s": 232.8,
3134
  "duration_s": 2.2,
3135
  "environment": "WebArena",
3136
  "task": "595",
@@ -3170,8 +3086,8 @@
3170
  ]
3171
  },
3172
  {
3173
- "number": 38,
3174
- "start_s": 235.0,
3175
  "duration_s": 2.2,
3176
  "environment": "WebArena",
3177
  "task": "600",
@@ -3196,8 +3112,8 @@
3196
  ]
3197
  },
3198
  {
3199
- "number": 39,
3200
- "start_s": 237.2,
3201
  "duration_s": 7.3,
3202
  "environment": "WebArena",
3203
  "task": "630",
@@ -3239,7 +3155,7 @@
3239
  ],
3240
  "additional_takes": [
3241
  {
3242
- "start_s": 239.4,
3243
  "duration_s": 2.55,
3244
  "shared_elapsed_start_s": 18.707142707984893,
3245
  "arms": [
@@ -3276,7 +3192,7 @@
3276
  ]
3277
  },
3278
  {
3279
- "start_s": 241.95,
3280
  "duration_s": 2.55,
3281
  "shared_elapsed_start_s": null,
3282
  "arms": [
@@ -3317,8 +3233,8 @@
3317
  ]
3318
  },
3319
  {
3320
- "number": 40,
3321
- "start_s": 244.5,
3322
  "duration_s": 2.2,
3323
  "environment": "WebArena",
3324
  "task": "653",
@@ -3358,8 +3274,8 @@
3358
  ]
3359
  },
3360
  {
3361
- "number": 41,
3362
- "start_s": 246.7,
3363
  "duration_s": 2.2,
3364
  "environment": "WebArena",
3365
  "task": "689",
@@ -3405,5 +3321,5 @@
3405
  }
3406
  ]
3407
  },
3408
- "embedded_chapters": 43
3409
  }
 
1
  {
2
+ "sha256": "719c3a13b5812dc9886e828e4ffefb2f7b3242565917a6ff6eae2cd2e601ffc2",
3
  "fps": 20,
4
  "resolution": [
5
  1200,
6
  900
7
  ],
8
+ "duration_s": 246.05,
9
+ "task_count": 40,
10
  "opening_s": 6.15,
11
+ "closing_start_s": 242.55,
12
+ "selection": "Selected recorded browser tasks in benchmark order. Task times cover the full run. The film includes passing and failing grades.",
13
  "chapters": [
14
  {
15
  "number": 1,
 
97
  "number": 3,
98
  "start_s": 14.65,
99
  "duration_s": 4.75,
100
+ "environment": "BrowserGym · MiniWoB++",
101
  "task": "Seed 103",
102
  "title": "Complete an autocomplete field",
103
  "shared_elapsed_start_s": 0,
 
138
  "number": 4,
139
  "start_s": 19.4,
140
  "duration_s": 4.75,
141
+ "environment": "BrowserGym · MiniWoB++",
142
  "task": "Seed 104",
143
  "title": "Reply to an email",
144
  "shared_elapsed_start_s": 1.9337746249977499,
 
179
  "number": 5,
180
  "start_s": 24.15,
181
  "duration_s": 2.95,
182
+ "environment": "BrowserGym · MiniWoB++",
183
  "task": "Seed 101",
184
  "title": "Like every post from one account",
185
  "shared_elapsed_start_s": null,
 
220
  "number": 6,
221
  "start_s": 27.1,
222
  "duration_s": 2.95,
223
+ "environment": "BrowserGym · MiniWoB++",
224
  "task": "Seed 101",
225
  "title": "Pick a date",
226
  "shared_elapsed_start_s": null,
 
261
  "number": 7,
262
  "start_s": 30.05,
263
  "duration_s": 2.95,
264
+ "environment": "BrowserGym · MiniWoB++",
265
  "task": "Seed 101",
266
  "title": "Book the shortest one-way flight",
267
  "shared_elapsed_start_s": null,
 
406
  "grader_status": "failure"
407
  },
408
  {
409
+ "arm": "Frozen click and stop weights",
410
  "elapsed_s": 0.7192069169832394,
411
  "passed": false,
412
  "device": "M5 Max MacBook Pro",
 
910
  {
911
  "number": 15,
912
  "start_s": 75.1,
913
+ "duration_s": 33.15,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
914
  "environment": "WebArena",
915
  "task": "190",
916
  "title": "Find the total of the latest completed order",
 
933
  "grader_status": "success"
934
  },
935
  {
936
+ "arm": "Frozen click and stop weights",
937
  "elapsed_s": 18.32999724999536,
938
  "passed": true,
939
  "device": "M5 Max MacBook Pro",
 
951
  ],
952
  "additional_takes": [
953
  {
954
+ "start_s": 78.75,
955
  "duration_s": 2.95,
956
  "shared_elapsed_start_s": 2.940150791895576,
957
  "arms": [
 
990
  ]
991
  },
992
  {
993
+ "start_s": 81.7,
994
  "duration_s": 2.95,
995
  "shared_elapsed_start_s": 6.681119083077647,
996
  "arms": [
 
1029
  ]
1030
  },
1031
  {
1032
+ "start_s": 84.65,
1033
  "duration_s": 2.95,
1034
  "shared_elapsed_start_s": null,
1035
  "arms": [
 
1068
  ]
1069
  },
1070
  {
1071
+ "start_s": 87.6,
1072
  "duration_s": 2.95,
1073
  "shared_elapsed_start_s": 2.8771139580057934,
1074
  "arms": [
 
1107
  ]
1108
  },
1109
  {
1110
+ "start_s": 90.55,
1111
  "duration_s": 2.95,
1112
  "shared_elapsed_start_s": 1.092322333017364,
1113
  "arms": [
 
1146
  ]
1147
  },
1148
  {
1149
+ "start_s": 93.5,
1150
  "duration_s": 2.95,
1151
  "shared_elapsed_start_s": 2.38902004098054,
1152
  "arms": [
 
1185
  ]
1186
  },
1187
  {
1188
+ "start_s": 96.45,
1189
  "duration_s": 2.95,
1190
  "shared_elapsed_start_s": 0.5983604160137472,
1191
  "arms": [
 
1224
  ]
1225
  },
1226
  {
1227
+ "start_s": 99.4,
1228
+ "duration_s": 2.95,
1229
+ "shared_elapsed_start_s": 2.878219416947104,
1230
+ "arms": [
1231
+ {
1232
+ "arm": "Stock Inkling 2 Mini",
1233
+ "elapsed_s": 44.78415783296805,
1234
+ "passed": true,
1235
+ "device": "M5 Max MacBook Pro",
1236
+ "receipt_sha256": "81671d792843cc52402bf24f0314e1cb015a940fa171d07cb0354021766938d1",
1237
+ "receipt": "190-original.json",
1238
+ "recording": {
1239
+ "kind": "native browser recording",
1240
+ "source_sha256": "f2dfbabfe77f82e4515c5d0a302ae94ead3fa360478d4c05c5b87932ba21c770",
1241
+ "start_s": 2.8790539589244872,
1242
+ "source_duration_s": 44.75,
1243
+ "available_frames": 59
1244
+ },
1245
+ "grader_status": "success"
1246
+ },
1247
+ {
1248
+ "arm": "Search submit stop",
1249
+ "elapsed_s": 5.128219416947104,
1250
+ "passed": true,
1251
+ "device": "M5 Max MacBook Pro",
1252
+ "receipt_sha256": "c411be851762f8cd21854df7d9fee8b198d0d7096d8087eae99c342405e246bd",
1253
+ "receipt": "190-edited.json",
1254
+ "recording": {
1255
+ "kind": "native browser recording",
1256
+ "source_sha256": "f3872bae954176fc4003c34437964000593da0774029c6f29e051f1f06e44a2f",
1257
+ "start_s": 2.8790000419830903,
1258
+ "source_duration_s": 5.35,
1259
+ "available_frames": 46
1260
+ },
1261
+ "grader_status": "success"
1262
+ }
1263
+ ]
1264
+ },
1265
+ {
1266
+ "start_s": 102.35,
1267
  "duration_s": 2.95,
1268
  "shared_elapsed_start_s": 3.5121810829732567,
1269
  "arms": [
 
1300
  "grader_status": "success"
1301
  }
1302
  ]
1303
+ },
1304
+ {
1305
+ "start_s": 105.3,
1306
+ "duration_s": 2.95,
1307
+ "shared_elapsed_start_s": 3.061038124957122,
1308
+ "arms": [
1309
+ {
1310
+ "arm": "Stock Inkling 2 Mini",
1311
+ "elapsed_s": 45.43856291694101,
1312
+ "passed": true,
1313
+ "device": "M5 Max MacBook Pro",
1314
+ "receipt_sha256": "45df24edf60219d0e61d64a02c69e35e048d11f7d491ee3b2109cfd85cf2cc71",
1315
+ "receipt": "190-original.json",
1316
+ "recording": {
1317
+ "kind": "native browser recording",
1318
+ "source_sha256": "ffbcc9cad73b37a6350063524e03cd3e3643732816e9f4e15dfbf481238e3b0b",
1319
+ "start_s": 3.061735332943499,
1320
+ "source_duration_s": 45.4,
1321
+ "available_frames": 59
1322
+ },
1323
+ "grader_status": "success"
1324
+ },
1325
+ {
1326
+ "arm": "Product-link selection edit",
1327
+ "elapsed_s": 5.311038124957122,
1328
+ "passed": true,
1329
+ "device": "M5 Max MacBook Pro",
1330
+ "receipt_sha256": "c465c7b586943e052d903a67139c94bf11c27aaeb1af0762f721614009925fb2",
1331
+ "receipt": "190-edited.json",
1332
+ "recording": {
1333
+ "kind": "native browser recording",
1334
+ "source_sha256": "5727b53a048fbdc1e198bb17f97cbbb3a63ca7f899f2c118d85bbe06914221ba",
1335
+ "start_s": 3.0618657909799367,
1336
+ "source_duration_s": 5.55,
1337
+ "available_frames": 46
1338
+ },
1339
+ "grader_status": "success"
1340
+ }
1341
+ ]
1342
  }
1343
  ]
1344
  },
1345
  {
1346
+ "number": 16,
1347
+ "start_s": 108.25,
1348
  "duration_s": 8.1,
1349
  "environment": "WebArena",
1350
  "task": "191",
 
1386
  ],
1387
  "additional_takes": [
1388
  {
1389
+ "start_s": 110.45,
1390
  "duration_s": 2.95,
1391
  "shared_elapsed_start_s": null,
1392
  "arms": [
 
1425
  ]
1426
  },
1427
  {
1428
+ "start_s": 113.4,
1429
  "duration_s": 2.95,
1430
  "shared_elapsed_start_s": null,
1431
  "arms": [
 
1466
  ]
1467
  },
1468
  {
1469
+ "number": 17,
1470
+ "start_s": 116.35,
1471
+ "duration_s": 5.15,
1472
  "environment": "WebArena",
1473
  "task": "231",
1474
  "title": "Find a canceled order number",
 
1507
  ],
1508
  "additional_takes": [
1509
  {
1510
+ "start_s": 118.55,
1511
+ "duration_s": 2.95,
1512
  "shared_elapsed_start_s": null,
1513
  "arms": [
1514
  {
 
1521
  "recording": {
1522
  "kind": "native browser recording",
1523
  "source_sha256": "3aea1b0be0d4e8bd1125a74b743474e3f4723109d8f6ecb2b6767bc86ce17ee2",
1524
+ "start_s": 28.42790495797526,
1525
  "source_duration_s": 32.35,
1526
+ "available_frames": 59
1527
  },
1528
  "grader_status": "failure"
1529
  },
 
1537
  "recording": {
1538
  "kind": "native browser recording",
1539
  "source_sha256": "fd230a0bf607e34cda779e74694430fb50e01c81b4fea13b87cecdac971afb6b",
1540
+ "start_s": 2.172678832965903,
1541
  "source_duration_s": 5.3,
1542
+ "available_frames": 59
1543
  },
1544
  "grader_status": "success"
1545
  }
 
1548
  ]
1549
  },
1550
  {
1551
+ "number": 18,
1552
+ "start_s": 121.5,
1553
  "duration_s": 2.2,
1554
  "environment": "WebArena",
1555
  "task": "260",
 
1589
  ]
1590
  },
1591
  {
1592
+ "number": 19,
1593
+ "start_s": 123.7,
1594
  "duration_s": 2.2,
1595
  "environment": "WebArena",
1596
  "task": "269",
 
1630
  ]
1631
  },
1632
  {
1633
+ "number": 20,
1634
+ "start_s": 125.9,
1635
  "duration_s": 2.2,
1636
  "environment": "WebArena",
1637
  "task": "270",
1638
+ "title": "Filter men’s shoes below $30",
1639
  "shared_elapsed_start_s": null,
1640
  "arms": [
1641
  {
 
1671
  ]
1672
  },
1673
  {
1674
+ "number": 21,
1675
+ "start_s": 128.1,
1676
+ "duration_s": 7.7,
1677
  "environment": "WebArena",
1678
  "task": "274",
1679
  "title": "Search for a USB Wi-Fi adapter",
 
1714
  ],
1715
  "additional_takes": [
1716
  {
1717
+ "start_s": 132.85,
1718
+ "duration_s": 2.95,
1719
+ "shared_elapsed_start_s": 3.1353435839992017,
1720
  "arms": [
1721
  {
1722
  "arm": "Stock Inkling 2 Mini",
 
1728
  "recording": {
1729
  "kind": "native browser recording",
1730
  "source_sha256": "af12cc9808d1e46d8a9c2c52e2e5a547b9e1345a484eec137aa3090d7aa8f212",
1731
+ "start_s": 3.1360501260496676,
1732
  "source_duration_s": 14.6,
1733
+ "available_frames": 59
1734
  },
1735
  "grader_status": "success"
1736
  },
 
1744
  "recording": {
1745
  "kind": "native browser recording",
1746
  "source_sha256": "e897298b69c97db8c118e4c2317b0c5f90298da7dc204658cf5e7d67cf008e52",
1747
+ "start_s": 3.1360644170781597,
1748
  "source_duration_s": 6.05,
1749
+ "available_frames": 46
1750
  },
1751
  "grader_status": "success"
1752
  }
 
1755
  ]
1756
  },
1757
  {
1758
+ "number": 22,
1759
+ "start_s": 135.8,
1760
+ "duration_s": 23.75,
1761
  "environment": "WebArena",
1762
  "task": "275",
1763
  "title": "Search for Xbox",
 
1798
  ],
1799
  "additional_takes": [
1800
  {
1801
+ "start_s": 141.85,
1802
+ "duration_s": 2.95,
1803
+ "shared_elapsed_start_s": 2.7589218330103904,
1804
  "arms": [
1805
  {
1806
  "arm": "Stock Inkling 2 Mini",
 
1812
  "recording": {
1813
  "kind": "native browser recording",
1814
  "source_sha256": "90909532e5e7a03ef11e431f670b650c64adc4b87ed6173b91b4474a451e325d",
1815
+ "start_s": 2.7596860420890152,
1816
  "source_duration_s": 5.3,
1817
+ "available_frames": 46
1818
  },
1819
  "grader_status": "success"
1820
  },
 
1828
  "recording": {
1829
  "kind": "native browser recording",
1830
  "source_sha256": "d126217723fea7e12c378ec0e931a570d40178157550c328b307bbb8b58a935e",
1831
+ "start_s": 2.7594927080208436,
1832
  "source_duration_s": 7.1,
1833
+ "available_frames": 59
1834
  },
1835
  "grader_status": "success"
1836
  }
1837
  ]
1838
  },
1839
  {
1840
+ "start_s": 144.8,
1841
+ "duration_s": 2.95,
1842
+ "shared_elapsed_start_s": 2.770188458962366,
1843
  "arms": [
1844
  {
1845
  "arm": "Stock Inkling 2 Mini",
 
1851
  "recording": {
1852
  "kind": "native browser recording",
1853
  "source_sha256": "14429436b695064bb95a4db5ad46cd2f1c92e6a2283c2aa9175d0cdb19be67fd",
1854
+ "start_s": 2.771443958976306,
1855
  "source_duration_s": 5.05,
1856
+ "available_frames": 45
1857
  },
1858
  "grader_status": "success"
1859
  },
 
1867
  "recording": {
1868
  "kind": "native browser recording",
1869
  "source_sha256": "524f90fa5fa0edb6001e43e3e61ed5b73c053ba1139d7b494a0ff7eb185c340e",
1870
+ "start_s": 2.7714176670415327,
1871
  "source_duration_s": 7.0,
1872
+ "available_frames": 59
1873
  },
1874
  "grader_status": "success"
1875
  }
1876
  ]
1877
  },
1878
  {
1879
+ "start_s": 147.75,
1880
+ "duration_s": 2.95,
1881
+ "shared_elapsed_start_s": 0.7524331250460816,
1882
  "arms": [
1883
  {
1884
  "arm": "Stock Inkling 2 Mini",
 
1890
  "recording": {
1891
  "kind": "native browser recording",
1892
  "source_sha256": "3d8f81550117b42ee36ada4ccd70dff5808fd4c6c9bf9685eb305daa60e4ffa7",
1893
+ "start_s": 0.7534510829718781,
1894
  "source_duration_s": 4.95,
1895
+ "available_frames": 59
1896
  },
1897
  "grader_status": "success"
1898
  },
 
1906
  "recording": {
1907
  "kind": "native browser recording",
1908
  "source_sha256": "993a915623f7851110f0178c347bd5ae29ad90318f9e9c1c9ace7c0bf53aff2a",
1909
+ "start_s": 0.7531419170554725,
1910
  "source_duration_s": 4.1,
1911
+ "available_frames": 46
1912
  },
1913
  "grader_status": "success"
1914
  }
1915
  ]
1916
  },
1917
  {
1918
+ "start_s": 150.7,
1919
+ "duration_s": 2.95,
1920
+ "shared_elapsed_start_s": 2.7378833330003545,
1921
+ "arms": [
1922
+ {
1923
+ "arm": "Stock Inkling 2 Mini",
1924
+ "elapsed_s": 18.785939250024967,
1925
+ "passed": true,
1926
+ "device": "M5 Max MacBook Pro",
1927
+ "receipt_sha256": "09ef4ded08f0db1e74a46302902dfb168fe246285d9b04cfd369299b0295b51a",
1928
+ "receipt": "275-original.json",
1929
+ "recording": {
1930
+ "kind": "native browser recording",
1931
+ "source_sha256": "dd891d17e6c96baf3fa4ce46698790adacd99ed449f0fbef594d347192a8703e",
1932
+ "start_s": 2.7385854159947485,
1933
+ "source_duration_s": 18.95,
1934
+ "available_frames": 59
1935
+ },
1936
+ "grader_status": "success"
1937
+ },
1938
+ {
1939
+ "arm": "Search submit stop",
1940
+ "elapsed_s": 4.9878833330003545,
1941
+ "passed": true,
1942
+ "device": "M5 Max MacBook Pro",
1943
+ "receipt_sha256": "833b8f3c747ffc8120093122d485f7fc663f123a1da4f1b0f46c6c90ed0411ff",
1944
+ "receipt": "275-edited.json",
1945
+ "recording": {
1946
+ "kind": "native browser recording",
1947
+ "source_sha256": "1a651a7722aff065265042313363f0d86e9dd5b1078bd0141355601cca71a3e4",
1948
+ "start_s": 2.738696665968746,
1949
+ "source_duration_s": 5.8,
1950
+ "available_frames": 46
1951
+ },
1952
+ "grader_status": "success"
1953
+ }
1954
+ ]
1955
+ },
1956
+ {
1957
+ "start_s": 153.65,
1958
+ "duration_s": 2.95,
1959
+ "shared_elapsed_start_s": 2.8694580829469487,
1960
  "arms": [
1961
  {
1962
  "arm": "Stock Inkling 2 Mini",
 
1968
  "recording": {
1969
  "kind": "native browser recording",
1970
  "source_sha256": "00c066740a18093c3dab6ab4a99c066338f6c2e60a0d3675214d560b8a216371",
1971
+ "start_s": 2.870172541937791,
1972
  "source_duration_s": 21.05,
1973
+ "available_frames": 59
1974
  },
1975
  "grader_status": "success"
1976
  },
 
1984
  "recording": {
1985
  "kind": "native browser recording",
1986
  "source_sha256": "c3836f0939f129721c89b1c4493bc3339533ffd48694876bc341252d06cbca13",
1987
+ "start_s": 2.8701312079792842,
1988
  "source_duration_s": 5.6,
1989
+ "available_frames": 46
1990
+ },
1991
+ "grader_status": "success"
1992
+ }
1993
+ ]
1994
+ },
1995
+ {
1996
+ "start_s": 156.6,
1997
+ "duration_s": 2.95,
1998
+ "shared_elapsed_start_s": 3.021435666945763,
1999
+ "arms": [
2000
+ {
2001
+ "arm": "Stock Inkling 2 Mini",
2002
+ "elapsed_s": 19.23649004206527,
2003
+ "passed": true,
2004
+ "device": "M5 Max MacBook Pro",
2005
+ "receipt_sha256": "b1661afd4c77b723a5770615088a5b6825721c8fafc3dd409cfa136405873f5d",
2006
+ "receipt": "275-original.json",
2007
+ "recording": {
2008
+ "kind": "native browser recording",
2009
+ "source_sha256": "cff138a1b744586afb236948d3794e48f440edf63ae0526523e7476211b05019",
2010
+ "start_s": 3.022113249870017,
2011
+ "source_duration_s": 19.25,
2012
+ "available_frames": 59
2013
+ },
2014
+ "grader_status": "success"
2015
+ },
2016
+ {
2017
+ "arm": "Product-link selection edit",
2018
+ "elapsed_s": 5.271435666945763,
2019
+ "passed": true,
2020
+ "device": "M5 Max MacBook Pro",
2021
+ "receipt_sha256": "74d1c41ba0b33fc4c739af5b06c9b26266834783fa088edbdd95563dee27d5aa",
2022
+ "receipt": "275-edited.json",
2023
+ "recording": {
2024
+ "kind": "native browser recording",
2025
+ "source_sha256": "846cac34d51a1b98428873e4aa4387ccc195acb6ee1dac258398ee84b0454e21",
2026
+ "start_s": 3.022121167043224,
2027
+ "source_duration_s": 5.8,
2028
+ "available_frames": 46
2029
  },
2030
  "grader_status": "success"
2031
  }
 
2034
  ]
2035
  },
2036
  {
2037
+ "number": 23,
2038
+ "start_s": 159.55,
2039
+ "duration_s": 10.75,
2040
  "environment": "WebArena",
2041
  "task": "276",
2042
  "title": "Search for Switch accessories",
 
2077
  ],
2078
  "additional_takes": [
2079
  {
2080
+ "start_s": 164.3,
2081
+ "duration_s": 3.05,
2082
+ "shared_elapsed_start_s": 1.3485021249856801,
2083
  "arms": [
2084
  {
2085
  "arm": "Stock Inkling 2 Mini",
 
2091
  "recording": {
2092
  "kind": "native browser recording",
2093
  "source_sha256": "3981b646f3ef012a477b5324d53c26779b4d267bd4328d4668097d98fd59b05a",
2094
+ "start_s": 1.349146250030026,
2095
  "source_duration_s": 5.55,
2096
+ "available_frames": 61
2097
  }
2098
  },
2099
  {
 
2106
  "recording": {
2107
  "kind": "native browser recording",
2108
  "source_sha256": "eb3e5fcb71a4735f73bf19c46aea6f60f27fd7f6df72236f44b18d133cb47950",
2109
+ "start_s": 1.3491312080062927,
2110
  "source_duration_s": 4.7,
2111
+ "available_frames": 48
2112
  }
2113
  }
2114
  ]
2115
  },
2116
  {
2117
+ "start_s": 167.35,
2118
  "duration_s": 2.95,
2119
  "shared_elapsed_start_s": 3.2479067909298465,
2120
  "arms": [
 
2155
  ]
2156
  },
2157
  {
2158
+ "number": 24,
2159
+ "start_s": 170.3,
2160
  "duration_s": 7.7,
2161
  "environment": "WebArena",
2162
  "task": "277",
 
2198
  ],
2199
  "additional_takes": [
2200
  {
2201
+ "start_s": 175.05,
2202
  "duration_s": 2.95,
2203
  "shared_elapsed_start_s": 3.629690458998084,
2204
  "arms": [
 
2239
  ]
2240
  },
2241
  {
2242
+ "number": 25,
2243
+ "start_s": 178.0,
2244
  "duration_s": 13.3,
2245
  "environment": "WebArena",
2246
  "task": "278",
 
2282
  ],
2283
  "additional_takes": [
2284
  {
2285
+ "start_s": 182.75,
2286
  "duration_s": 3.05,
2287
  "shared_elapsed_start_s": null,
2288
  "arms": [
 
2321
  ]
2322
  },
2323
  {
2324
+ "start_s": 185.8,
2325
  "duration_s": 2.55,
2326
  "shared_elapsed_start_s": 3.232748709013686,
2327
  "arms": [
 
2360
  ]
2361
  },
2362
  {
2363
+ "start_s": 188.35,
2364
  "duration_s": 2.95,
2365
  "shared_elapsed_start_s": 3.86688183399383,
2366
  "arms": [
 
2401
  ]
2402
  },
2403
  {
2404
+ "number": 26,
2405
+ "start_s": 191.3,
2406
  "duration_s": 2.2,
2407
  "environment": "WebArena",
2408
  "task": "279",
 
2444
  ]
2445
  },
2446
  {
2447
+ "number": 27,
2448
+ "start_s": 193.5,
2449
+ "duration_s": 14.0,
2450
  "environment": "WebArena",
2451
  "task": "283",
2452
+ "title": "Find Xbox controllers released in 2020–2021",
2453
  "shared_elapsed_start_s": null,
2454
  "arms": [
2455
  {
 
2487
  ],
2488
  "additional_takes": [
2489
  {
2490
+ "start_s": 195.7,
2491
  "duration_s": 2.95,
2492
  "shared_elapsed_start_s": null,
2493
  "arms": [
 
2526
  ]
2527
  },
2528
  {
2529
+ "start_s": 198.65,
2530
  "duration_s": 2.95,
2531
  "shared_elapsed_start_s": null,
2532
  "arms": [
 
2565
  ]
2566
  },
2567
  {
2568
+ "start_s": 201.6,
2569
  "duration_s": 2.95,
2570
  "shared_elapsed_start_s": null,
2571
  "arms": [
 
2602
  "grader_status": "failure"
2603
  }
2604
  ]
2605
+ },
2606
+ {
2607
+ "start_s": 204.55,
2608
+ "duration_s": 2.95,
2609
+ "shared_elapsed_start_s": null,
2610
+ "arms": [
2611
+ {
2612
+ "arm": "Stock Inkling 2 Mini",
2613
+ "elapsed_s": 83.67859258397948,
2614
+ "passed": false,
2615
+ "device": "M5 Max MacBook Pro",
2616
+ "receipt_sha256": "caa357c2cfbe6aa3e5071ea4911e793ae704149abec00613e955397f1b3fd9f0",
2617
+ "receipt": "283-original.json",
2618
+ "recording": {
2619
+ "kind": "native browser recording",
2620
+ "source_sha256": "3c41d6f534fe265e807a15743f36af902a3ef846a34cb201b30f4e935276b024",
2621
+ "start_s": 80.72925879196264,
2622
+ "source_duration_s": 85.35,
2623
+ "available_frames": 59
2624
+ },
2625
+ "grader_status": "failure"
2626
+ },
2627
+ {
2628
+ "arm": "Product-link selection edit",
2629
+ "elapsed_s": 9.97932625003159,
2630
+ "passed": true,
2631
+ "device": "M5 Max MacBook Pro",
2632
+ "receipt_sha256": "815093b92848d5256a73998a7ea848405874b2fe8342deb4191a9d7ca1cf367b",
2633
+ "receipt": "283-edited.json",
2634
+ "recording": {
2635
+ "kind": "native browser recording",
2636
+ "source_sha256": "3dc5df389d5f36faecc16700018661c6aa29f198d75d5951ae0e1d1712aa5f31",
2637
+ "start_s": 7.030045042000711,
2638
+ "source_duration_s": 10.3,
2639
+ "available_frames": 59
2640
+ },
2641
+ "grader_status": "success"
2642
+ }
2643
+ ]
2644
  }
2645
  ]
2646
  },
2647
  {
2648
+ "number": 28,
2649
+ "start_s": 207.5,
2650
  "duration_s": 2.2,
2651
  "environment": "WebArena",
2652
  "task": "313",
 
2686
  ]
2687
  },
2688
  {
2689
+ "number": 29,
2690
+ "start_s": 209.7,
2691
  "duration_s": 2.2,
2692
  "environment": "WebArena",
2693
  "task": "324",
 
2727
  ]
2728
  },
2729
  {
2730
+ "number": 30,
2731
+ "start_s": 211.9,
2732
  "duration_s": 2.2,
2733
  "environment": "WebArena",
2734
  "task": "358",
 
2768
  ]
2769
  },
2770
  {
2771
+ "number": 31,
2772
+ "start_s": 214.1,
2773
  "duration_s": 2.2,
2774
  "environment": "WebArena",
2775
  "task": "399",
 
2809
  ]
2810
  },
2811
  {
2812
+ "number": 32,
2813
+ "start_s": 216.3,
2814
  "duration_s": 2.2,
2815
  "environment": "WebArena",
2816
  "task": "404",
 
2850
  ]
2851
  },
2852
  {
2853
+ "number": 33,
2854
+ "start_s": 218.5,
2855
  "duration_s": 2.2,
2856
  "environment": "WebArena",
2857
  "task": "431",
 
2933
  ]
2934
  },
2935
  {
2936
+ "number": 34,
2937
+ "start_s": 220.7,
2938
  "duration_s": 3.55,
2939
  "environment": "WebArena",
2940
  "task": "432",
 
3004
  ]
3005
  },
3006
  {
3007
+ "number": 35,
3008
+ "start_s": 224.25,
3009
  "duration_s": 2.2,
3010
  "environment": "WebArena",
3011
  "task": "516",
 
3045
  ]
3046
  },
3047
  {
3048
+ "number": 36,
3049
+ "start_s": 226.45,
3050
  "duration_s": 2.2,
3051
  "environment": "WebArena",
3052
  "task": "595",
 
3086
  ]
3087
  },
3088
  {
3089
+ "number": 37,
3090
+ "start_s": 228.65,
3091
  "duration_s": 2.2,
3092
  "environment": "WebArena",
3093
  "task": "600",
 
3112
  ]
3113
  },
3114
  {
3115
+ "number": 38,
3116
+ "start_s": 230.85,
3117
  "duration_s": 7.3,
3118
  "environment": "WebArena",
3119
  "task": "630",
 
3155
  ],
3156
  "additional_takes": [
3157
  {
3158
+ "start_s": 233.05,
3159
  "duration_s": 2.55,
3160
  "shared_elapsed_start_s": 18.707142707984893,
3161
  "arms": [
 
3192
  ]
3193
  },
3194
  {
3195
+ "start_s": 235.6,
3196
  "duration_s": 2.55,
3197
  "shared_elapsed_start_s": null,
3198
  "arms": [
 
3233
  ]
3234
  },
3235
  {
3236
+ "number": 39,
3237
+ "start_s": 238.15,
3238
  "duration_s": 2.2,
3239
  "environment": "WebArena",
3240
  "task": "653",
 
3274
  ]
3275
  },
3276
  {
3277
+ "number": 40,
3278
+ "start_s": 240.35,
3279
  "duration_s": 2.2,
3280
  "environment": "WebArena",
3281
  "task": "689",
 
3321
  }
3322
  ]
3323
  },
3324
+ "embedded_chapters": 42
3325
  }
benchmark/film-receipts.zip CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:bb08583011bb3b9dd239ca31fbbfb9bb6081833d3d89cfa55fc55ad1c656f23a
3
- size 369543
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9305a4e81e27b6d05ae09c485b5f0d8db931f5bcc21081f8ccea7d11cca88e0d
3
+ size 360268