husky8 commited on
Commit
4ae35ff
·
verified ·
1 Parent(s): 83dd675

Show the edited model full width at its recorded speed

Browse files
benchmark/browser-speedups-poster.jpg CHANGED

Git LFS Details

  • SHA256: 0424cd1298a8ea5377ef4f37dc8f1895b8d22f9edbb0300f3ece13bf2c34c3db
  • Pointer size: 131 Bytes
  • Size of remote file: 178 kB

Git LFS Details

  • SHA256: a8ff11dede89b0b8e2e3d2f4f37abe69931e1f81876d4922da92cf3e530a0343
  • Pointer size: 131 Bytes
  • Size of remote file: 251 kB
benchmark/browser-speedups.mp4 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:220a64db8806755e3f9c57a422e0f12c67d8759fe21ffa3df55d38416569efb9
3
- size 7333099
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46730653d4b3e253f90fef69e8bc02de6733a2ec83451ea36edd664192b0c182
3
+ size 4363224
benchmark/speedups-chapters.json CHANGED
@@ -1,24 +1,73 @@
1
  {
2
- "sha256": "220a64db8806755e3f9c57a422e0f12c67d8759fe21ffa3df55d38416569efb9",
3
- "fps": 20,
4
  "resolution": [
5
- 1440,
6
- 820
7
  ],
8
- "duration_s": 84.35,
9
  "task_count": 13,
10
- "opening_s": 11.1,
11
- "closing_start_s": 79.35,
12
- "selection": "Same 13 passing comparisons. Browser excerpts are aligned to each recorded action, with an annotated still at the target and a hold on its result. Displayed run times are full task times; the closing mean uses the 13 chapters and excludes the separate opener.",
 
 
 
 
13
  "chapters": [
14
  {
15
  "number": 1,
16
- "start_s": 11.1,
17
- "duration_s": 5.25,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
  "environment": "Selenium",
19
  "task": "Case 02",
20
  "title": "Fill and submit a form",
21
- "shared_elapsed_start_s": null,
22
  "arms": [
23
  {
24
  "arm": "Original",
@@ -30,7 +79,7 @@
30
  "recording": {
31
  "kind": "native browser recording",
32
  "source_sha256": "c094391f4bdc6545669730fd7dc5dfe30137e5ef2b63c178921d783a4de7f8bc",
33
- "start_s": 7.0333,
34
  "source_duration_s": 15.44
35
  }
36
  },
@@ -44,50 +93,27 @@
44
  "recording": {
45
  "kind": "native browser recording",
46
  "source_sha256": "f80ae6b4df0da070fa403b8b00a77542577ae73977a4a16fa5645f6f977dcc65",
47
- "start_s": 1.4852,
48
  "source_duration_s": 8.84
49
  }
50
  }
51
  ],
52
- "action_excerpts": [
53
- {
54
- "action_elapsed_s": 8.5833,
55
- "source_start_s": 7.0333,
56
- "source_stop_s": 9.6333,
57
- "marker_source": "recorded pointer",
58
- "marker": [
59
- 399,
60
- 400
61
- ],
62
- "result_frame_elapsed_s": 8.723281833035871,
63
- "action_hold_s": 0.8,
64
- "result_hold_s": 1.85,
65
- "source_sha256": "c094391f4bdc6545669730fd7dc5dfe30137e5ef2b63c178921d783a4de7f8bc"
66
- },
67
- {
68
- "action_elapsed_s": 3.0352,
69
- "source_start_s": 1.4852,
70
- "source_stop_s": 4.0852,
71
- "marker_source": "recorded pointer",
72
- "marker": [
73
- 387,
74
- 383
75
- ],
76
- "result_frame_elapsed_s": 3.135238249944523,
77
- "action_hold_s": 0.8,
78
- "result_hold_s": 1.85,
79
- "source_sha256": "f80ae6b4df0da070fa403b8b00a77542577ae73977a4a16fa5645f6f977dcc65"
80
- }
81
- ]
82
  },
83
  {
84
- "number": 2,
85
- "start_s": 16.35,
86
- "duration_s": 5.25,
87
  "environment": "Selenium + Automation Exercise",
88
  "task": "Case 03",
89
  "title": "Set quantity to two and add the dress",
90
- "shared_elapsed_start_s": null,
91
  "arms": [
92
  {
93
  "arm": "Original",
@@ -99,7 +125,7 @@
99
  "recording": {
100
  "kind": "native browser recording",
101
  "source_sha256": "cb68912b3f5d8c99bbca757b933501f4e4978a9de8b4c47966faa5418ca5394f",
102
- "start_s": 14.1189,
103
  "source_duration_s": 18.44
104
  }
105
  },
@@ -113,50 +139,27 @@
113
  "recording": {
114
  "kind": "native browser recording",
115
  "source_sha256": "7da366d3d7889ee801626bf6c4a6f38104490ccd51dab650ebda4f28d44b1c4d",
116
- "start_s": 4.3523000000000005,
117
  "source_duration_s": 8.12
118
  }
119
  }
120
  ],
121
- "action_excerpts": [
122
- {
123
- "action_elapsed_s": 15.6689,
124
- "source_start_s": 14.1189,
125
- "source_stop_s": 16.7189,
126
- "marker_source": "recorded pointer",
127
- "marker": [
128
- 1101,
129
- 319
130
- ],
131
- "result_frame_elapsed_s": 17.038890917012466,
132
- "action_hold_s": 0.8,
133
- "result_hold_s": 1.85,
134
- "source_sha256": "cb68912b3f5d8c99bbca757b933501f4e4978a9de8b4c47966faa5418ca5394f"
135
- },
136
- {
137
- "action_elapsed_s": 5.9023,
138
- "source_start_s": 4.3523000000000005,
139
- "source_stop_s": 6.9523,
140
- "marker_source": "recorded pointer",
141
- "marker": [
142
- 1101,
143
- 322
144
- ],
145
- "result_frame_elapsed_s": 7.232266958029941,
146
- "action_hold_s": 0.8,
147
- "result_hold_s": 1.85,
148
- "source_sha256": "7da366d3d7889ee801626bf6c4a6f38104490ccd51dab650ebda4f28d44b1c4d"
149
- }
150
- ]
151
  },
152
  {
153
- "number": 3,
154
- "start_s": 21.6,
155
- "duration_s": 5.25,
156
  "environment": "BrowserGym · MiniWoB++",
157
  "task": "Seed 103",
158
  "title": "Complete an autocomplete field",
159
- "shared_elapsed_start_s": null,
160
  "arms": [
161
  {
162
  "arm": "Original",
@@ -167,7 +170,7 @@
167
  "receipt": "use-autocomplete-103-original.json",
168
  "recording": {
169
  "kind": "timestamped browser captures",
170
- "start_s": 13.713160666055046,
171
  "source_sha256": "3768c12d71ff98aafc5104cb764816e43a53371c8a5b5532d9a33df538c04d0f",
172
  "timing_manifest_sha256": "08490f9e128d0d41dc1da13f69388e39b2888f9acc2a2802fb83ece1f6bca894",
173
  "frame_count": 123
@@ -182,52 +185,29 @@
182
  "receipt": "use-autocomplete-103-edited.json",
183
  "recording": {
184
  "kind": "timestamped browser captures",
185
- "start_s": 1.6051289580063892,
186
  "source_sha256": "f01a27f87a8e46e5921345e4e56d68636eca684525547d67abd81371958968a2",
187
  "timing_manifest_sha256": "3a2e2d6a4a2f3fc2ec4ad2de24126f5c20c9091ccd3089312d6b5d7f3f13a5d7",
188
  "frame_count": 28
189
  }
190
  }
191
  ],
192
- "action_excerpts": [
193
- {
194
- "action_elapsed_s": 15.263160666055047,
195
- "source_start_s": 13.713160666055046,
196
- "source_stop_s": 15.50987195968628,
197
- "marker_source": "control",
198
- "marker": [
199
- 115,
200
- 223
201
- ],
202
- "result_frame_elapsed_s": 15.50987195968628,
203
- "action_hold_s": 0.8,
204
- "result_hold_s": 1.85,
205
- "source_sha256": "3768c12d71ff98aafc5104cb764816e43a53371c8a5b5532d9a33df538c04d0f"
206
- },
207
- {
208
- "action_elapsed_s": 3.1551289580063893,
209
- "source_start_s": 1.6051289580063892,
210
- "source_stop_s": 3.4008638858795166,
211
- "marker_source": "control",
212
- "marker": [
213
- 115,
214
- 223
215
- ],
216
- "result_frame_elapsed_s": 3.4008638858795166,
217
- "action_hold_s": 0.8,
218
- "result_hold_s": 1.85,
219
- "source_sha256": "f01a27f87a8e46e5921345e4e56d68636eca684525547d67abd81371958968a2"
220
- }
221
- ]
222
  },
223
  {
224
- "number": 4,
225
- "start_s": 26.85,
226
- "duration_s": 5.25,
227
  "environment": "BrowserGym · MiniWoB++",
228
  "task": "Seed 104",
229
  "title": "Reply to an email",
230
- "shared_elapsed_start_s": null,
231
  "arms": [
232
  {
233
  "arm": "Original",
@@ -238,7 +218,7 @@
238
  "receipt": "email-inbox-104-original.json",
239
  "recording": {
240
  "kind": "timestamped browser captures",
241
- "start_s": 24.264819292002358,
242
  "source_sha256": "1b8cda1812e06a017e5b2dc72a7e25fdefb16d89e0cfffb25ee081a9e38878f7",
243
  "timing_manifest_sha256": "27eb1ae06a81a62873e5e94ddfc5c3eef534ddaf7d76624fb5ae93557707ab60",
244
  "frame_count": 206
@@ -253,52 +233,29 @@
253
  "receipt": "email-inbox-104-edited.json",
254
  "recording": {
255
  "kind": "timestamped browser captures",
256
- "start_s": 4.23377462499775,
257
  "source_sha256": "ed5227ad3c7a9a44c11b380bd85960c691ee8cc46fa1be754a45061d0dacf3b1",
258
  "timing_manifest_sha256": "c0284e5f6372e4d63e1a835bb30390cca3ed62ec86514b1602a44b366e990b5c",
259
  "frame_count": 49
260
  }
261
  }
262
  ],
263
- "action_excerpts": [
264
- {
265
- "action_elapsed_s": 25.81481929200236,
266
- "source_start_s": 24.264819292002358,
267
- "source_stop_s": 26.026827096939087,
268
- "marker_source": "control",
269
- "marker": [
270
- 294,
271
- 122
272
- ],
273
- "result_frame_elapsed_s": 26.026827096939087,
274
- "action_hold_s": 0.8,
275
- "result_hold_s": 1.85,
276
- "source_sha256": "1b8cda1812e06a017e5b2dc72a7e25fdefb16d89e0cfffb25ee081a9e38878f7"
277
- },
278
- {
279
- "action_elapsed_s": 5.78377462499775,
280
- "source_start_s": 4.23377462499775,
281
- "source_stop_s": 6.045209169387817,
282
- "marker_source": "control",
283
- "marker": [
284
- 294,
285
- 122
286
- ],
287
- "result_frame_elapsed_s": 6.045209169387817,
288
- "action_hold_s": 0.8,
289
- "result_hold_s": 1.85,
290
- "source_sha256": "ed5227ad3c7a9a44c11b380bd85960c691ee8cc46fa1be754a45061d0dacf3b1"
291
- }
292
- ]
293
  },
294
  {
295
- "number": 5,
296
- "start_s": 32.1,
297
- "duration_s": 5.25,
298
  "environment": "WebArena",
299
  "task": "188",
300
  "title": "Find the total of the latest canceled order",
301
- "shared_elapsed_start_s": null,
302
  "arms": [
303
  {
304
  "arm": "Stock Inkling 2 Mini",
@@ -310,7 +267,7 @@
310
  "recording": {
311
  "kind": "native browser recording",
312
  "source_sha256": "ec642f8c46a74743cac1669a727e102b9abc253393196d9deb9ad44b050d6577",
313
- "start_s": 18.880599999999998,
314
  "source_duration_s": 27.8
315
  },
316
  "grader_status": "success"
@@ -325,51 +282,28 @@
325
  "recording": {
326
  "kind": "native browser recording",
327
  "source_sha256": "4d72502042f6f1d1e91ea088b8e29a95929ef8f80a60d9e45185d6ee83c3122f",
328
- "start_s": 2.4579000000000004,
329
  "source_duration_s": 5.45
330
  },
331
  "grader_status": "success"
332
  }
333
  ],
334
- "action_excerpts": [
335
- {
336
- "action_elapsed_s": 20.4306,
337
- "source_start_s": 18.880599999999998,
338
- "source_stop_s": 21.4806,
339
- "marker_source": "recorded pointer",
340
- "marker": [
341
- 139,
342
- 326
343
- ],
344
- "result_frame_elapsed_s": 27.749273541965522,
345
- "action_hold_s": 0.8,
346
- "result_hold_s": 1.85,
347
- "source_sha256": "ec642f8c46a74743cac1669a727e102b9abc253393196d9deb9ad44b050d6577"
348
- },
349
- {
350
- "action_elapsed_s": 4.0079,
351
- "source_start_s": 2.4579000000000004,
352
- "source_stop_s": 5.0579,
353
- "marker_source": "recorded pointer",
354
- "marker": [
355
- 139,
356
- 326
357
- ],
358
- "result_frame_elapsed_s": 5.295377749991603,
359
- "action_hold_s": 0.8,
360
- "result_hold_s": 1.85,
361
- "source_sha256": "4d72502042f6f1d1e91ea088b8e29a95929ef8f80a60d9e45185d6ee83c3122f"
362
- }
363
- ]
364
  },
365
  {
366
- "number": 6,
367
- "start_s": 37.35,
368
- "duration_s": 5.25,
369
  "environment": "WebArena",
370
  "task": "189",
371
  "title": "Find the total of the latest pending order",
372
- "shared_elapsed_start_s": null,
373
  "arms": [
374
  {
375
  "arm": "Stock Inkling 2 Mini",
@@ -381,7 +315,7 @@
381
  "recording": {
382
  "kind": "native browser recording",
383
  "source_sha256": "9d35421225e57628bada9376e92528ac7a677e5629efde93d4c44aa7661dd037",
384
- "start_s": 9.595699999999999,
385
  "source_duration_s": 33.85
386
  },
387
  "grader_status": "success"
@@ -396,122 +330,28 @@
396
  "recording": {
397
  "kind": "native browser recording",
398
  "source_sha256": "4b91334778448d041a1b3f59fa2188cd2cc44dbb0c14d5f8dfce324e6a9c8072",
399
- "start_s": 30.572100000000002,
400
  "source_duration_s": 33.45
401
  },
402
  "grader_status": "success"
403
  }
404
  ],
405
- "action_excerpts": [
406
- {
407
- "action_elapsed_s": 11.1457,
408
- "source_start_s": 9.595699999999999,
409
- "source_stop_s": 12.1957,
410
- "marker_source": "recorded pointer",
411
- "marker": [
412
- 139,
413
- 326
414
- ],
415
- "result_frame_elapsed_s": 33.79933191598393,
416
- "action_hold_s": 0.8,
417
- "result_hold_s": 1.85,
418
- "source_sha256": "9d35421225e57628bada9376e92528ac7a677e5629efde93d4c44aa7661dd037"
419
- },
420
- {
421
- "action_elapsed_s": 32.1221,
422
- "source_start_s": 30.572100000000002,
423
- "source_stop_s": 33.1721,
424
- "marker_source": "recorded pointer",
425
- "marker": [
426
- 139,
427
- 326
428
- ],
429
- "result_frame_elapsed_s": 33.3197506670095,
430
- "action_hold_s": 0.8,
431
- "result_hold_s": 1.85,
432
- "source_sha256": "4b91334778448d041a1b3f59fa2188cd2cc44dbb0c14d5f8dfce324e6a9c8072"
433
- }
434
- ]
435
- },
436
- {
437
- "number": 7,
438
- "start_s": 42.6,
439
- "duration_s": 5.25,
440
- "environment": "WebArena",
441
- "task": "190",
442
- "title": "Find the total of the latest completed order",
443
- "shared_elapsed_start_s": null,
444
- "arms": [
445
- {
446
- "arm": "Stock Inkling 2 Mini",
447
- "elapsed_s": 45.501502291997895,
448
- "passed": true,
449
- "device": "M5 Max MacBook Pro",
450
- "receipt_sha256": "0c5039a54ece418d72927e94ef46bf347afc2f7c4328924b409d0bd8029f30b9",
451
- "receipt": "190-original.json",
452
- "recording": {
453
- "kind": "native browser recording",
454
- "source_sha256": "aef068d2419ff13564764c7d28509ff34185b09fa4e7c68e4365eba05171aa2f",
455
- "start_s": 19.228199999999998,
456
- "source_duration_s": 45.5
457
- },
458
- "grader_status": "success"
459
- },
460
- {
461
- "arm": "Frozen pending-order weights",
462
- "elapsed_s": 18.32999724999536,
463
- "passed": true,
464
- "device": "M5 Max MacBook Pro",
465
- "receipt_sha256": "c7266a5d572d38b90d3949ed3958f7a547c3e495cd2d2d13089c111af9e022d9",
466
- "receipt": "190-edited.json",
467
- "recording": {
468
- "kind": "native browser recording",
469
- "source_sha256": "e2ecd40026fc920fedc2bd53366aed4636b1e97fc769c771694a3f1516581c5a",
470
- "start_s": 15.661099999999998,
471
- "source_duration_s": 18.55
472
- },
473
- "grader_status": "success"
474
- }
475
- ],
476
- "action_excerpts": [
477
- {
478
- "action_elapsed_s": 20.7782,
479
- "source_start_s": 19.228199999999998,
480
- "source_stop_s": 21.8282,
481
- "marker_source": "recorded pointer",
482
- "marker": [
483
- 139,
484
- 326
485
- ],
486
- "result_frame_elapsed_s": 45.44928350001574,
487
- "action_hold_s": 0.8,
488
- "result_hold_s": 1.85,
489
- "source_sha256": "aef068d2419ff13564764c7d28509ff34185b09fa4e7c68e4365eba05171aa2f"
490
- },
491
- {
492
- "action_elapsed_s": 17.2111,
493
- "source_start_s": 15.661099999999998,
494
- "source_stop_s": 18.2611,
495
- "marker_source": "recorded pointer",
496
- "marker": [
497
- 139,
498
- 326
499
- ],
500
- "result_frame_elapsed_s": 18.409997249995357,
501
- "action_hold_s": 0.8,
502
- "result_hold_s": 1.85,
503
- "source_sha256": "e2ecd40026fc920fedc2bd53366aed4636b1e97fc769c771694a3f1516581c5a"
504
- }
505
- ]
506
  },
507
  {
508
  "number": 8,
509
- "start_s": 47.85,
510
- "duration_s": 5.25,
511
  "environment": "WebArena",
512
  "task": "274",
513
  "title": "Search for a USB Wi-Fi adapter",
514
- "shared_elapsed_start_s": null,
515
  "arms": [
516
  {
517
  "arm": "Stock Inkling 2 Mini",
@@ -523,7 +363,7 @@
523
  "recording": {
524
  "kind": "native browser recording",
525
  "source_sha256": "df3bab2792f9ec4726fd9a419418f76bfcb25aeea3d2c690b523bc34e26a9ea5",
526
- "start_s": 7.0302999999999995,
527
  "source_duration_s": 14.75
528
  },
529
  "grader_status": "success"
@@ -538,51 +378,28 @@
538
  "recording": {
539
  "kind": "native browser recording",
540
  "source_sha256": "bc53c19aaaf7f2477f261e63856cf1cc31ce09d01560717ffa08080d39d0e005",
541
- "start_s": 1.5211,
542
  "source_duration_s": 5.3
543
  },
544
  "grader_status": "success"
545
  }
546
  ],
547
- "action_excerpts": [
548
- {
549
- "action_elapsed_s": 8.5803,
550
- "source_start_s": 7.0302999999999995,
551
- "source_stop_s": 9.6303,
552
- "marker_source": "recorded pointer",
553
- "marker": [
554
- 1111,
555
- 90
556
- ],
557
- "result_frame_elapsed_s": 14.699298291979357,
558
- "action_hold_s": 0.8,
559
- "result_hold_s": 1.85,
560
- "source_sha256": "df3bab2792f9ec4726fd9a419418f76bfcb25aeea3d2c690b523bc34e26a9ea5"
561
- },
562
- {
563
- "action_elapsed_s": 3.0711,
564
- "source_start_s": 1.5211,
565
- "source_stop_s": 4.1211,
566
- "marker_source": "recorded pointer",
567
- "marker": [
568
- 1111,
569
- 90
570
- ],
571
- "result_frame_elapsed_s": 4.80369825001806,
572
- "action_hold_s": 0.8,
573
- "result_hold_s": 1.85,
574
- "source_sha256": "bc53c19aaaf7f2477f261e63856cf1cc31ce09d01560717ffa08080d39d0e005"
575
- }
576
- ]
577
  },
578
  {
579
  "number": 9,
580
- "start_s": 53.1,
581
- "duration_s": 5.25,
582
  "environment": "WebArena",
583
  "task": "275",
584
  "title": "Search for Xbox",
585
- "shared_elapsed_start_s": null,
586
  "arms": [
587
  {
588
  "arm": "Stock Inkling 2 Mini",
@@ -594,7 +411,7 @@
594
  "recording": {
595
  "kind": "native browser recording",
596
  "source_sha256": "e7a404ab8b53fe2ea72cf7c1e558149909bd42afbf57e486f01c6e342c409840",
597
- "start_s": 11.8033,
598
  "source_duration_s": 20.65
599
  },
600
  "grader_status": "success"
@@ -609,51 +426,28 @@
609
  "recording": {
610
  "kind": "native browser recording",
611
  "source_sha256": "7084446b535c1c10f48e0c70ec9b825b62d19fd07290cd8865872aaa31b2b212",
612
- "start_s": 1.7573999999999999,
613
  "source_duration_s": 5.65
614
  },
615
  "grader_status": "success"
616
  }
617
  ],
618
- "action_excerpts": [
619
- {
620
- "action_elapsed_s": 13.3533,
621
- "source_start_s": 11.8033,
622
- "source_stop_s": 14.403300000000002,
623
- "marker_source": "recorded pointer",
624
- "marker": [
625
- 1111,
626
- 90
627
- ],
628
- "result_frame_elapsed_s": 20.50676358402241,
629
- "action_hold_s": 0.8,
630
- "result_hold_s": 1.85,
631
- "source_sha256": "e7a404ab8b53fe2ea72cf7c1e558149909bd42afbf57e486f01c6e342c409840"
632
- },
633
- {
634
- "action_elapsed_s": 3.3074,
635
- "source_start_s": 1.7573999999999999,
636
- "source_stop_s": 4.3574,
637
- "marker_source": "recorded pointer",
638
- "marker": [
639
- 1111,
640
- 90
641
- ],
642
- "result_frame_elapsed_s": 5.03540237496607,
643
- "action_hold_s": 0.8,
644
- "result_hold_s": 1.85,
645
- "source_sha256": "7084446b535c1c10f48e0c70ec9b825b62d19fd07290cd8865872aaa31b2b212"
646
- }
647
- ]
648
  },
649
  {
650
  "number": 10,
651
- "start_s": 58.35,
652
- "duration_s": 5.25,
653
  "environment": "WebArena",
654
  "task": "276",
655
  "title": "Search for Switch accessories",
656
- "shared_elapsed_start_s": null,
657
  "arms": [
658
  {
659
  "arm": "Stock Inkling 2 Mini",
@@ -665,7 +459,7 @@
665
  "recording": {
666
  "kind": "native browser recording",
667
  "source_sha256": "00d41ba656ae96772e431aed9aa84fdc9ddd8e57a3cedec0a3347eb37e73b1e9",
668
- "start_s": 5.146,
669
  "source_duration_s": 13.1
670
  },
671
  "grader_status": "success"
@@ -680,51 +474,28 @@
680
  "recording": {
681
  "kind": "native browser recording",
682
  "source_sha256": "b6aaf024685e66ba3d9d34a3b80f4ea03006abda6c5b3c715c7d3dc49dca19ce",
683
- "start_s": 2.0591999999999997,
684
  "source_duration_s": 5.9
685
  },
686
  "grader_status": "success"
687
  }
688
  ],
689
- "action_excerpts": [
690
- {
691
- "action_elapsed_s": 6.696,
692
- "source_start_s": 5.146,
693
- "source_stop_s": 7.7459999999999996,
694
- "marker_source": "recorded pointer",
695
- "marker": [
696
- 1111,
697
- 90
698
- ],
699
- "result_frame_elapsed_s": 13.01084349995479,
700
- "action_hold_s": 0.8,
701
- "result_hold_s": 1.85,
702
- "source_sha256": "00d41ba656ae96772e431aed9aa84fdc9ddd8e57a3cedec0a3347eb37e73b1e9"
703
- },
704
- {
705
- "action_elapsed_s": 3.6092,
706
- "source_start_s": 2.0591999999999997,
707
- "source_stop_s": 4.6592,
708
- "marker_source": "recorded pointer",
709
- "marker": [
710
- 1111,
711
- 90
712
- ],
713
- "result_frame_elapsed_s": 5.421348875081167,
714
- "action_hold_s": 0.8,
715
- "result_hold_s": 1.85,
716
- "source_sha256": "b6aaf024685e66ba3d9d34a3b80f4ea03006abda6c5b3c715c7d3dc49dca19ce"
717
- }
718
- ]
719
  },
720
  {
721
  "number": 11,
722
- "start_s": 63.6,
723
- "duration_s": 5.25,
724
  "environment": "WebArena",
725
  "task": "277",
726
  "title": "Search for iPhone 13 batteries",
727
- "shared_elapsed_start_s": null,
728
  "arms": [
729
  {
730
  "arm": "Stock Inkling 2 Mini",
@@ -736,7 +507,7 @@
736
  "recording": {
737
  "kind": "native browser recording",
738
  "source_sha256": "1ae3bb1db3e4d47e53d8b06684843b2aef8ce876e6becf70b0d9184665605b13",
739
- "start_s": 9.808399999999999,
740
  "source_duration_s": 17.15
741
  },
742
  "grader_status": "success"
@@ -751,51 +522,28 @@
751
  "recording": {
752
  "kind": "native browser recording",
753
  "source_sha256": "1f34cfc44b67f4ae188868d7fa95b2a92af7abe7f2519e71cf57ff4c9c160417",
754
- "start_s": 2.2832,
755
  "source_duration_s": 6.35
756
  },
757
  "grader_status": "success"
758
  }
759
  ],
760
- "action_excerpts": [
761
- {
762
- "action_elapsed_s": 11.3584,
763
- "source_start_s": 9.808399999999999,
764
- "source_stop_s": 12.4084,
765
- "marker_source": "recorded pointer",
766
- "marker": [
767
- 1111,
768
- 90
769
- ],
770
- "result_frame_elapsed_s": 17.04133550000377,
771
- "action_hold_s": 0.8,
772
- "result_hold_s": 1.85,
773
- "source_sha256": "1ae3bb1db3e4d47e53d8b06684843b2aef8ce876e6becf70b0d9184665605b13"
774
- },
775
- {
776
- "action_elapsed_s": 3.8332,
777
- "source_start_s": 2.2832,
778
- "source_stop_s": 4.8832,
779
- "marker_source": "recorded pointer",
780
- "marker": [
781
- 1111,
782
- 90
783
- ],
784
- "result_frame_elapsed_s": 5.901562124998309,
785
- "action_hold_s": 0.8,
786
- "result_hold_s": 1.85,
787
- "source_sha256": "1f34cfc44b67f4ae188868d7fa95b2a92af7abe7f2519e71cf57ff4c9c160417"
788
- }
789
- ]
790
  },
791
  {
792
  "number": 12,
793
- "start_s": 68.85,
794
- "duration_s": 5.25,
795
  "environment": "WebArena",
796
  "task": "278",
797
  "title": "Search for green tea bags",
798
- "shared_elapsed_start_s": null,
799
  "arms": [
800
  {
801
  "arm": "Stock Inkling 2 Mini",
@@ -807,7 +555,7 @@
807
  "recording": {
808
  "kind": "native browser recording",
809
  "source_sha256": "b4597dc162e0d24dd8a47b9236efd3498e95a803905150d3626b8df2d4e612d6",
810
- "start_s": 12.1756,
811
  "source_duration_s": 19.05
812
  },
813
  "grader_status": "success"
@@ -822,51 +570,28 @@
822
  "recording": {
823
  "kind": "native browser recording",
824
  "source_sha256": "97ccca271791b8e5eb58a6aaba1e836be5f34b51055a8d07fa394d7c9ca94016",
825
- "start_s": 2.5599,
826
  "source_duration_s": 8.1
827
  },
828
  "grader_status": "success"
829
  }
830
  ],
831
- "action_excerpts": [
832
- {
833
- "action_elapsed_s": 13.7256,
834
- "source_start_s": 12.1756,
835
- "source_stop_s": 14.7756,
836
- "marker_source": "recorded pointer",
837
- "marker": [
838
- 1111,
839
- 90
840
- ],
841
- "result_frame_elapsed_s": 18.919734166976994,
842
- "action_hold_s": 0.8,
843
- "result_hold_s": 1.85,
844
- "source_sha256": "b4597dc162e0d24dd8a47b9236efd3498e95a803905150d3626b8df2d4e612d6"
845
- },
846
- {
847
- "action_elapsed_s": 4.1099,
848
- "source_start_s": 2.5599,
849
- "source_stop_s": 5.1598999999999995,
850
- "marker_source": "recorded pointer",
851
- "marker": [
852
- 1111,
853
- 90
854
- ],
855
- "result_frame_elapsed_s": 7.694025874994695,
856
- "action_hold_s": 0.8,
857
- "result_hold_s": 1.85,
858
- "source_sha256": "97ccca271791b8e5eb58a6aaba1e836be5f34b51055a8d07fa394d7c9ca94016"
859
- }
860
- ]
861
  },
862
  {
863
  "number": 13,
864
- "start_s": 74.1,
865
- "duration_s": 5.25,
866
  "environment": "WebArena",
867
  "task": "630",
868
  "title": "Ask for headphone recommendations",
869
- "shared_elapsed_start_s": null,
870
  "arms": [
871
  {
872
  "arm": "Original",
@@ -878,7 +603,7 @@
878
  "recording": {
879
  "kind": "native browser recording",
880
  "source_sha256": "39f0add873acca0cc39b1f0928ef80902590f07023e4dfdf829706f2a267f1c4",
881
- "start_s": 14.567499999999999,
882
  "source_duration_s": 24.5
883
  }
884
  },
@@ -892,99 +617,20 @@
892
  "recording": {
893
  "kind": "native browser recording",
894
  "source_sha256": "ab7eaa11036cd84ee646ad4c74113973420e28b0dadeaa5e07350deee879bb1b",
895
- "start_s": 11.829899999999999,
896
  "source_duration_s": 20.55
897
  }
898
  }
899
  ],
900
- "action_excerpts": [
901
- {
902
- "action_elapsed_s": 16.1175,
903
- "source_start_s": 14.567499999999999,
904
- "source_stop_s": 17.1675,
905
- "marker_source": "recorded pointer",
906
- "marker": [
907
- 138,
908
- 642
909
- ],
910
- "result_frame_elapsed_s": 24.44875562498346,
911
- "action_hold_s": 0.8,
912
- "result_hold_s": 1.85,
913
- "source_sha256": "39f0add873acca0cc39b1f0928ef80902590f07023e4dfdf829706f2a267f1c4"
914
- },
915
- {
916
- "action_elapsed_s": 13.3799,
917
- "source_start_s": 11.829899999999999,
918
- "source_stop_s": 14.4299,
919
- "marker_source": "recorded pointer",
920
- "marker": [
921
- 138,
922
- 642
923
- ],
924
- "result_frame_elapsed_s": 20.498692833003588,
925
- "action_hold_s": 0.8,
926
- "result_hold_s": 1.85,
927
- "source_sha256": "ab7eaa11036cd84ee646ad4c74113973420e28b0dadeaa5e07350deee879bb1b"
928
- }
929
- ]
930
  }
931
  ],
932
- "opening_race": {
933
- "task": "190",
934
- "device": "M5 Max MacBook Pro",
935
- "playback_rate": 1,
936
- "duration_s": 11.1,
937
- "arms": [
938
- {
939
- "name": "Stock Inkling 2 Mini",
940
- "seconds": 45.88679650006816,
941
- "receipt_sha256": "db7cc933937f077603d615ef906b8ccd107f4b80f9e4303af200341f55a6e809",
942
- "recording_sha256": "001a63b86d63e645ef77f990d03c3f2b17ea4dc17c1d6a34cc91e3b4126bca25"
943
- },
944
- {
945
- "name": "Cross-device order stop",
946
- "seconds": 5.127113958005793,
947
- "receipt_sha256": "de99cd3b9ffdab3dcde0a6ba3b7ab53c4f78c893b5e04a1a42cd729a32ca8816",
948
- "recording_sha256": "1c0b7325f4c3119182342a5017acce4187ed18802c46dddd919c151140565054"
949
- }
950
- ],
951
- "start_s": 0.0,
952
- "action_excerpts": [
953
- {
954
- "action_elapsed_s": 21.6657,
955
- "source_start_s": 20.1157,
956
- "source_stop_s": 22.715700000000002,
957
- "marker_source": "recorded pointer",
958
- "marker": [
959
- 139,
960
- 326
961
- ],
962
- "result_frame_elapsed_s": 45.849230958055706,
963
- "action_hold_s": 0.8,
964
- "result_hold_s": 2.1,
965
- "source_sha256": "001a63b86d63e645ef77f990d03c3f2b17ea4dc17c1d6a34cc91e3b4126bca25"
966
- },
967
- {
968
- "action_elapsed_s": 3.9993,
969
- "source_start_s": 2.4493,
970
- "source_stop_s": 5.0493,
971
- "marker_source": "recorded pointer",
972
- "marker": [
973
- 139,
974
- 326
975
- ],
976
- "result_frame_elapsed_s": 5.2071139580057935,
977
- "action_hold_s": 0.8,
978
- "result_hold_s": 2.1,
979
- "source_sha256": "1c0b7325f4c3119182342a5017acce4187ed18802c46dddd919c151140565054"
980
- }
981
- ],
982
- "shared_elapsed_start_s": null,
983
- "shared_clock_opener_s": 5.6
984
- },
985
- "embedded_chapters": 15,
986
- "display_labels": [
987
- "Original weights",
988
- "Edited weights"
989
- ]
990
  }
 
1
  {
2
+ "sha256": "46730653d4b3e253f90fef69e8bc02de6733a2ec83451ea36edd664192b0c182",
3
+ "fps": 25,
4
  "resolution": [
5
+ 1200,
6
+ 868
7
  ],
8
+ "duration_s": 132.6,
9
  "task_count": 13,
10
+ "closing_start_s": 129.0,
11
+ "playback_rate": 1,
12
+ "display_labels": [
13
+ "Original weights",
14
+ "Edited weights"
15
+ ],
16
+ "selection": "One uninterrupted edited run per task at capture speed, followed by a labeled result still. Timings use the paired passing runs. The completed-order lookup uses the previously featured fastest take and appears once.",
17
  "chapters": [
18
  {
19
  "number": 1,
20
+ "start_s": 0.0,
21
+ "duration_s": 6.44,
22
+ "environment": "WebArena",
23
+ "task": "190",
24
+ "title": "Find the total of the latest completed order",
25
+ "arms": [
26
+ {
27
+ "arm": "Original",
28
+ "elapsed_s": 45.88679650006816,
29
+ "passed": true,
30
+ "device": "M5 Max MacBook Pro",
31
+ "receipt_sha256": "db7cc933937f077603d615ef906b8ccd107f4b80f9e4303af200341f55a6e809",
32
+ "receipt": "190-original.json",
33
+ "recording": {
34
+ "kind": "native browser recording",
35
+ "source_sha256": "001a63b86d63e645ef77f990d03c3f2b17ea4dc17c1d6a34cc91e3b4126bca25",
36
+ "start_s": 0
37
+ },
38
+ "grader_status": "success"
39
+ },
40
+ {
41
+ "arm": "Cross-device order stop",
42
+ "elapsed_s": 5.127113958005793,
43
+ "passed": true,
44
+ "device": "M5 Max MacBook Pro",
45
+ "receipt_sha256": "de99cd3b9ffdab3dcde0a6ba3b7ab53c4f78c893b5e04a1a42cd729a32ca8816",
46
+ "receipt": "190-edited.json",
47
+ "recording": {
48
+ "kind": "native browser recording",
49
+ "source_sha256": "1c0b7325f4c3119182342a5017acce4187ed18802c46dddd919c151140565054",
50
+ "start_s": 0
51
+ },
52
+ "grader_status": "success"
53
+ }
54
+ ],
55
+ "display_title": "Find the latest completed order total",
56
+ "playback_rate": 1.0,
57
+ "shown_arm": "edited",
58
+ "task_start_s": 0,
59
+ "task_end_s": 5.127113958005793,
60
+ "result_still_start_s": 5.16,
61
+ "result_still_duration_s": 1.28,
62
+ "frame_clock": "capture timestamps"
63
+ },
64
+ {
65
+ "number": 2,
66
+ "start_s": 6.44,
67
+ "duration_s": 4.36,
68
  "environment": "Selenium",
69
  "task": "Case 02",
70
  "title": "Fill and submit a form",
 
71
  "arms": [
72
  {
73
  "arm": "Original",
 
79
  "recording": {
80
  "kind": "native browser recording",
81
  "source_sha256": "c094391f4bdc6545669730fd7dc5dfe30137e5ef2b63c178921d783a4de7f8bc",
82
+ "start_s": 0,
83
  "source_duration_s": 15.44
84
  }
85
  },
 
93
  "recording": {
94
  "kind": "native browser recording",
95
  "source_sha256": "f80ae6b4df0da070fa403b8b00a77542577ae73977a4a16fa5645f6f977dcc65",
96
+ "start_s": 0,
97
  "source_duration_s": 8.84
98
  }
99
  }
100
  ],
101
+ "display_title": "Fill out and submit a form",
102
+ "playback_rate": 1.0,
103
+ "shown_arm": "edited",
104
+ "task_start_s": 0,
105
+ "task_end_s": 3.055238249944523,
106
+ "result_still_start_s": 9.52,
107
+ "result_still_duration_s": 1.28,
108
+ "frame_clock": "capture timestamps"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
109
  },
110
  {
111
+ "number": 3,
112
+ "start_s": 10.8,
113
+ "duration_s": 8.44,
114
  "environment": "Selenium + Automation Exercise",
115
  "task": "Case 03",
116
  "title": "Set quantity to two and add the dress",
 
117
  "arms": [
118
  {
119
  "arm": "Original",
 
125
  "recording": {
126
  "kind": "native browser recording",
127
  "source_sha256": "cb68912b3f5d8c99bbca757b933501f4e4978a9de8b4c47966faa5418ca5394f",
128
+ "start_s": 0,
129
  "source_duration_s": 18.44
130
  }
131
  },
 
139
  "recording": {
140
  "kind": "native browser recording",
141
  "source_sha256": "7da366d3d7889ee801626bf6c4a6f38104490ccd51dab650ebda4f28d44b1c4d",
142
+ "start_s": 0,
143
  "source_duration_s": 8.12
144
  }
145
  }
146
  ],
147
+ "display_title": "Fill a form and add two dresses to the cart",
148
+ "playback_rate": 1.0,
149
+ "shown_arm": "edited",
150
+ "task_start_s": 0,
151
+ "task_end_s": 7.152266958029941,
152
+ "result_still_start_s": 17.96,
153
+ "result_still_duration_s": 1.28,
154
+ "frame_clock": "capture timestamps"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
155
  },
156
  {
157
+ "number": 4,
158
+ "start_s": 19.24,
159
+ "duration_s": 4.64,
160
  "environment": "BrowserGym · MiniWoB++",
161
  "task": "Seed 103",
162
  "title": "Complete an autocomplete field",
 
163
  "arms": [
164
  {
165
  "arm": "Original",
 
170
  "receipt": "use-autocomplete-103-original.json",
171
  "recording": {
172
  "kind": "timestamped browser captures",
173
+ "start_s": 0,
174
  "source_sha256": "3768c12d71ff98aafc5104cb764816e43a53371c8a5b5532d9a33df538c04d0f",
175
  "timing_manifest_sha256": "08490f9e128d0d41dc1da13f69388e39b2888f9acc2a2802fb83ece1f6bca894",
176
  "frame_count": 123
 
185
  "receipt": "use-autocomplete-103-edited.json",
186
  "recording": {
187
  "kind": "timestamped browser captures",
188
+ "start_s": 0,
189
  "source_sha256": "f01a27f87a8e46e5921345e4e56d68636eca684525547d67abd81371958968a2",
190
  "timing_manifest_sha256": "3a2e2d6a4a2f3fc2ec4ad2de24126f5c20c9091ccd3089312d6b5d7f3f13a5d7",
191
  "frame_count": 28
192
  }
193
  }
194
  ],
195
+ "display_title": "Complete an autocomplete field",
196
+ "playback_rate": 1.0,
197
+ "shown_arm": "edited",
198
+ "task_start_s": 0,
199
+ "task_end_s": 3.3551289580063894,
200
+ "result_still_start_s": 22.6,
201
+ "result_still_duration_s": 1.28,
202
+ "frame_clock": "capture timestamps"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
203
  },
204
  {
205
+ "number": 5,
206
+ "start_s": 23.88,
207
+ "duration_s": 7.28,
208
  "environment": "BrowserGym · MiniWoB++",
209
  "task": "Seed 104",
210
  "title": "Reply to an email",
 
211
  "arms": [
212
  {
213
  "arm": "Original",
 
218
  "receipt": "email-inbox-104-original.json",
219
  "recording": {
220
  "kind": "timestamped browser captures",
221
+ "start_s": 0,
222
  "source_sha256": "1b8cda1812e06a017e5b2dc72a7e25fdefb16d89e0cfffb25ee081a9e38878f7",
223
  "timing_manifest_sha256": "27eb1ae06a81a62873e5e94ddfc5c3eef534ddaf7d76624fb5ae93557707ab60",
224
  "frame_count": 206
 
233
  "receipt": "email-inbox-104-edited.json",
234
  "recording": {
235
  "kind": "timestamped browser captures",
236
+ "start_s": 0,
237
  "source_sha256": "ed5227ad3c7a9a44c11b380bd85960c691ee8cc46fa1be754a45061d0dacf3b1",
238
  "timing_manifest_sha256": "c0284e5f6372e4d63e1a835bb30390cca3ed62ec86514b1602a44b366e990b5c",
239
  "frame_count": 49
240
  }
241
  }
242
  ],
243
+ "display_title": "Reply to an email",
244
+ "playback_rate": 1.0,
245
+ "shown_arm": "edited",
246
+ "task_start_s": 0,
247
+ "task_end_s": 5.98377462499775,
248
+ "result_still_start_s": 29.88,
249
+ "result_still_duration_s": 1.28,
250
+ "frame_clock": "capture timestamps"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
251
  },
252
  {
253
+ "number": 6,
254
+ "start_s": 31.16,
255
+ "duration_s": 6.52,
256
  "environment": "WebArena",
257
  "task": "188",
258
  "title": "Find the total of the latest canceled order",
 
259
  "arms": [
260
  {
261
  "arm": "Stock Inkling 2 Mini",
 
267
  "recording": {
268
  "kind": "native browser recording",
269
  "source_sha256": "ec642f8c46a74743cac1669a727e102b9abc253393196d9deb9ad44b050d6577",
270
+ "start_s": 0,
271
  "source_duration_s": 27.8
272
  },
273
  "grader_status": "success"
 
282
  "recording": {
283
  "kind": "native browser recording",
284
  "source_sha256": "4d72502042f6f1d1e91ea088b8e29a95929ef8f80a60d9e45185d6ee83c3122f",
285
+ "start_s": 0,
286
  "source_duration_s": 5.45
287
  },
288
  "grader_status": "success"
289
  }
290
  ],
291
+ "display_title": "Find the latest canceled order total",
292
+ "playback_rate": 1.0,
293
+ "shown_arm": "edited",
294
+ "task_start_s": 0,
295
+ "task_end_s": 5.215377749991603,
296
+ "result_still_start_s": 36.4,
297
+ "result_still_duration_s": 1.28,
298
+ "frame_clock": "capture timestamps"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
299
  },
300
  {
301
+ "number": 7,
302
+ "start_s": 37.68,
303
+ "duration_s": 34.52,
304
  "environment": "WebArena",
305
  "task": "189",
306
  "title": "Find the total of the latest pending order",
 
307
  "arms": [
308
  {
309
  "arm": "Stock Inkling 2 Mini",
 
315
  "recording": {
316
  "kind": "native browser recording",
317
  "source_sha256": "9d35421225e57628bada9376e92528ac7a677e5629efde93d4c44aa7661dd037",
318
+ "start_s": 0,
319
  "source_duration_s": 33.85
320
  },
321
  "grader_status": "success"
 
330
  "recording": {
331
  "kind": "native browser recording",
332
  "source_sha256": "4b91334778448d041a1b3f59fa2188cd2cc44dbb0c14d5f8dfce324e6a9c8072",
333
+ "start_s": 0,
334
  "source_duration_s": 33.45
335
  },
336
  "grader_status": "success"
337
  }
338
  ],
339
+ "display_title": "Find the latest pending order total",
340
+ "playback_rate": 1.0,
341
+ "shown_arm": "edited",
342
+ "task_start_s": 0,
343
+ "task_end_s": 33.2397506670095,
344
+ "result_still_start_s": 70.92,
345
+ "result_still_duration_s": 1.28,
346
+ "frame_clock": "capture timestamps"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
347
  },
348
  {
349
  "number": 8,
350
+ "start_s": 72.2,
351
+ "duration_s": 6.04,
352
  "environment": "WebArena",
353
  "task": "274",
354
  "title": "Search for a USB Wi-Fi adapter",
 
355
  "arms": [
356
  {
357
  "arm": "Stock Inkling 2 Mini",
 
363
  "recording": {
364
  "kind": "native browser recording",
365
  "source_sha256": "df3bab2792f9ec4726fd9a419418f76bfcb25aeea3d2c690b523bc34e26a9ea5",
366
+ "start_s": 0,
367
  "source_duration_s": 14.75
368
  },
369
  "grader_status": "success"
 
378
  "recording": {
379
  "kind": "native browser recording",
380
  "source_sha256": "bc53c19aaaf7f2477f261e63856cf1cc31ce09d01560717ffa08080d39d0e005",
381
+ "start_s": 0,
382
  "source_duration_s": 5.3
383
  },
384
  "grader_status": "success"
385
  }
386
  ],
387
+ "display_title": "Search for a USB Wi-Fi adapter",
388
+ "playback_rate": 1.0,
389
+ "shown_arm": "edited",
390
+ "task_start_s": 0,
391
+ "task_end_s": 4.72369825001806,
392
+ "result_still_start_s": 76.96,
393
+ "result_still_duration_s": 1.28,
394
+ "frame_clock": "capture timestamps"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
395
  },
396
  {
397
  "number": 9,
398
+ "start_s": 78.24,
399
+ "duration_s": 6.24,
400
  "environment": "WebArena",
401
  "task": "275",
402
  "title": "Search for Xbox",
 
403
  "arms": [
404
  {
405
  "arm": "Stock Inkling 2 Mini",
 
411
  "recording": {
412
  "kind": "native browser recording",
413
  "source_sha256": "e7a404ab8b53fe2ea72cf7c1e558149909bd42afbf57e486f01c6e342c409840",
414
+ "start_s": 0,
415
  "source_duration_s": 20.65
416
  },
417
  "grader_status": "success"
 
426
  "recording": {
427
  "kind": "native browser recording",
428
  "source_sha256": "7084446b535c1c10f48e0c70ec9b825b62d19fd07290cd8865872aaa31b2b212",
429
+ "start_s": 0,
430
  "source_duration_s": 5.65
431
  },
432
  "grader_status": "success"
433
  }
434
  ],
435
+ "display_title": "Search for Xbox",
436
+ "playback_rate": 1.0,
437
+ "shown_arm": "edited",
438
+ "task_start_s": 0,
439
+ "task_end_s": 4.95540237496607,
440
+ "result_still_start_s": 83.2,
441
+ "result_still_duration_s": 1.28,
442
+ "frame_clock": "capture timestamps"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
443
  },
444
  {
445
  "number": 10,
446
+ "start_s": 84.48,
447
+ "duration_s": 6.64,
448
  "environment": "WebArena",
449
  "task": "276",
450
  "title": "Search for Switch accessories",
 
451
  "arms": [
452
  {
453
  "arm": "Stock Inkling 2 Mini",
 
459
  "recording": {
460
  "kind": "native browser recording",
461
  "source_sha256": "00d41ba656ae96772e431aed9aa84fdc9ddd8e57a3cedec0a3347eb37e73b1e9",
462
+ "start_s": 0,
463
  "source_duration_s": 13.1
464
  },
465
  "grader_status": "success"
 
474
  "recording": {
475
  "kind": "native browser recording",
476
  "source_sha256": "b6aaf024685e66ba3d9d34a3b80f4ea03006abda6c5b3c715c7d3dc49dca19ce",
477
+ "start_s": 0,
478
  "source_duration_s": 5.9
479
  },
480
  "grader_status": "success"
481
  }
482
  ],
483
+ "display_title": "Search for Switch accessories",
484
+ "playback_rate": 1.0,
485
+ "shown_arm": "edited",
486
+ "task_start_s": 0,
487
+ "task_end_s": 5.341348875081167,
488
+ "result_still_start_s": 89.84,
489
+ "result_still_duration_s": 1.28,
490
+ "frame_clock": "capture timestamps"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
491
  },
492
  {
493
  "number": 11,
494
+ "start_s": 91.12,
495
+ "duration_s": 7.12,
496
  "environment": "WebArena",
497
  "task": "277",
498
  "title": "Search for iPhone 13 batteries",
 
499
  "arms": [
500
  {
501
  "arm": "Stock Inkling 2 Mini",
 
507
  "recording": {
508
  "kind": "native browser recording",
509
  "source_sha256": "1ae3bb1db3e4d47e53d8b06684843b2aef8ce876e6becf70b0d9184665605b13",
510
+ "start_s": 0,
511
  "source_duration_s": 17.15
512
  },
513
  "grader_status": "success"
 
522
  "recording": {
523
  "kind": "native browser recording",
524
  "source_sha256": "1f34cfc44b67f4ae188868d7fa95b2a92af7abe7f2519e71cf57ff4c9c160417",
525
+ "start_s": 0,
526
  "source_duration_s": 6.35
527
  },
528
  "grader_status": "success"
529
  }
530
  ],
531
+ "display_title": "Search for iPhone 13 batteries",
532
+ "playback_rate": 1.0,
533
+ "shown_arm": "edited",
534
+ "task_start_s": 0,
535
+ "task_end_s": 5.821562124998309,
536
+ "result_still_start_s": 96.96,
537
+ "result_still_duration_s": 1.28,
538
+ "frame_clock": "capture timestamps"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
539
  },
540
  {
541
  "number": 12,
542
+ "start_s": 98.24,
543
+ "duration_s": 8.92,
544
  "environment": "WebArena",
545
  "task": "278",
546
  "title": "Search for green tea bags",
 
547
  "arms": [
548
  {
549
  "arm": "Stock Inkling 2 Mini",
 
555
  "recording": {
556
  "kind": "native browser recording",
557
  "source_sha256": "b4597dc162e0d24dd8a47b9236efd3498e95a803905150d3626b8df2d4e612d6",
558
+ "start_s": 0,
559
  "source_duration_s": 19.05
560
  },
561
  "grader_status": "success"
 
570
  "recording": {
571
  "kind": "native browser recording",
572
  "source_sha256": "97ccca271791b8e5eb58a6aaba1e836be5f34b51055a8d07fa394d7c9ca94016",
573
+ "start_s": 0,
574
  "source_duration_s": 8.1
575
  },
576
  "grader_status": "success"
577
  }
578
  ],
579
+ "display_title": "Search for green tea bags",
580
+ "playback_rate": 1.0,
581
+ "shown_arm": "edited",
582
+ "task_start_s": 0,
583
+ "task_end_s": 7.614025874994695,
584
+ "result_still_start_s": 105.88,
585
+ "result_still_duration_s": 1.28,
586
+ "frame_clock": "capture timestamps"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
587
  },
588
  {
589
  "number": 13,
590
+ "start_s": 107.16,
591
+ "duration_s": 21.84,
592
  "environment": "WebArena",
593
  "task": "630",
594
  "title": "Ask for headphone recommendations",
 
595
  "arms": [
596
  {
597
  "arm": "Original",
 
603
  "recording": {
604
  "kind": "native browser recording",
605
  "source_sha256": "39f0add873acca0cc39b1f0928ef80902590f07023e4dfdf829706f2a267f1c4",
606
+ "start_s": 0,
607
  "source_duration_s": 24.5
608
  }
609
  },
 
617
  "recording": {
618
  "kind": "native browser recording",
619
  "source_sha256": "ab7eaa11036cd84ee646ad4c74113973420e28b0dadeaa5e07350deee879bb1b",
620
+ "start_s": 0,
621
  "source_duration_s": 20.55
622
  }
623
  }
624
  ],
625
+ "display_title": "Post a headphone recommendation request",
626
+ "playback_rate": 1.0,
627
+ "shown_arm": "edited",
628
+ "task_start_s": 0,
629
+ "task_end_s": 20.557142707984895,
630
+ "result_still_start_s": 127.72,
631
+ "result_still_duration_s": 1.28,
632
+ "frame_clock": "capture timestamps"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
633
  }
634
  ],
635
+ "embedded_chapters": 14
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
636
  }
benchmark/speedups-film-receipts.zip CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:78c81e18eb406cd62a10af1677ffa7a0f976f0c92dd249ecfa4ebc9fa8bda2d9
3
- size 48881
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ec15cf77a2719470f416ea24aa3e6d0a0ff630016bf81a99a76bd89083cb03e
3
+ size 43502
benchmark/speedups-timings.json CHANGED
@@ -1,12 +1,12 @@
1
  {
2
- "selection": "Same 13 passing comparisons. Browser excerpts are aligned to each recorded action, with an annotated still at the target and a hold on its result. Displayed run times are full task times; the closing mean uses the 13 chapters and excludes the separate opener.",
3
  "tasks": 13,
4
- "mean_stock_s": 21.736033676317536,
5
- "mean_edit_s": 9.64190112815525,
6
- "time_speedup": 2.254330695514631,
7
- "time_saved_pct": 55.640935822341085,
8
- "film_sha256": "220a64db8806755e3f9c57a422e0f12c67d8759fe21ffa3df55d38416569efb9",
9
- "duration_s": 84.35,
10
  "release_checkpoint_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1",
11
  "expanded_rtx": {
12
  "tasks": [
 
1
  {
2
+ "selection": "One uninterrupted edited run per task at capture speed, followed by a labeled result still. Timings use the paired passing runs. The completed-order lookup uses the previously featured fastest take and appears once.",
3
  "tasks": 13,
4
+ "mean_stock_s": 21.76567169232294,
5
+ "mean_edit_s": 8.62629472107913,
6
+ "time_speedup": 2.5231773775519812,
7
+ "time_saved_pct": 60.36743160046034,
8
+ "film_sha256": "46730653d4b3e253f90fef69e8bc02de6733a2ec83451ea36edd664192b0c182",
9
+ "duration_s": 132.6,
10
  "release_checkpoint_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1",
11
  "expanded_rtx": {
12
  "tasks": [