husky8 commited on
Commit
f86b26e
·
verified ·
1 Parent(s): ebb17cf

Show the edited model full width at its recorded speed

Browse files
benchmark/browser-speedups.mp4 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:46730653d4b3e253f90fef69e8bc02de6733a2ec83451ea36edd664192b0c182
3
- size 4363224
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4918da3d8e665b12719749670f14ed21511ca1511fea261070292e6937f9c800
3
+ size 3365575
benchmark/speedups-chapters.json CHANGED
@@ -1,19 +1,19 @@
1
  {
2
- "sha256": "46730653d4b3e253f90fef69e8bc02de6733a2ec83451ea36edd664192b0c182",
3
  "fps": 25,
4
  "resolution": [
5
  1200,
6
  868
7
  ],
8
- "duration_s": 132.6,
9
- "task_count": 13,
10
- "closing_start_s": 129.0,
11
  "playback_rate": 1,
12
  "display_labels": [
13
  "Original weights",
14
  "Edited weights"
15
  ],
16
- "selection": "One uninterrupted edited run per task at capture speed, followed by a labeled result still. Timings use the paired passing runs. The completed-order lookup uses the previously featured fastest take and appears once.",
17
  "chapters": [
18
  {
19
  "number": 1,
@@ -300,54 +300,6 @@
300
  {
301
  "number": 7,
302
  "start_s": 37.68,
303
- "duration_s": 34.52,
304
- "environment": "WebArena",
305
- "task": "189",
306
- "title": "Find the total of the latest pending order",
307
- "arms": [
308
- {
309
- "arm": "Stock Inkling 2 Mini",
310
- "elapsed_s": 33.850683916010894,
311
- "passed": true,
312
- "device": "M5 Max MacBook Pro",
313
- "receipt_sha256": "9dd3fed47db798b1f67d44f794379eda89f3c949e8a10c0a65ced54dc05f5512",
314
- "receipt": "189-original.json",
315
- "recording": {
316
- "kind": "native browser recording",
317
- "source_sha256": "9d35421225e57628bada9376e92528ac7a677e5629efde93d4c44aa7661dd037",
318
- "start_s": 0,
319
- "source_duration_s": 33.85
320
- },
321
- "grader_status": "success"
322
- },
323
- {
324
- "arm": "Published stop-row weights",
325
- "elapsed_s": 33.2397506670095,
326
- "passed": true,
327
- "device": "M5 Max MacBook Pro",
328
- "receipt_sha256": "e26130900a6a30683ec77ad0f32b975f5f6f4be9b25df86c9e5caf1c800f16c1",
329
- "receipt": "189-edited.json",
330
- "recording": {
331
- "kind": "native browser recording",
332
- "source_sha256": "4b91334778448d041a1b3f59fa2188cd2cc44dbb0c14d5f8dfce324e6a9c8072",
333
- "start_s": 0,
334
- "source_duration_s": 33.45
335
- },
336
- "grader_status": "success"
337
- }
338
- ],
339
- "display_title": "Find the latest pending order total",
340
- "playback_rate": 1.0,
341
- "shown_arm": "edited",
342
- "task_start_s": 0,
343
- "task_end_s": 33.2397506670095,
344
- "result_still_start_s": 70.92,
345
- "result_still_duration_s": 1.28,
346
- "frame_clock": "capture timestamps"
347
- },
348
- {
349
- "number": 8,
350
- "start_s": 72.2,
351
  "duration_s": 6.04,
352
  "environment": "WebArena",
353
  "task": "274",
@@ -389,13 +341,13 @@
389
  "shown_arm": "edited",
390
  "task_start_s": 0,
391
  "task_end_s": 4.72369825001806,
392
- "result_still_start_s": 76.96,
393
  "result_still_duration_s": 1.28,
394
  "frame_clock": "capture timestamps"
395
  },
396
  {
397
- "number": 9,
398
- "start_s": 78.24,
399
  "duration_s": 6.24,
400
  "environment": "WebArena",
401
  "task": "275",
@@ -437,13 +389,13 @@
437
  "shown_arm": "edited",
438
  "task_start_s": 0,
439
  "task_end_s": 4.95540237496607,
440
- "result_still_start_s": 83.2,
441
  "result_still_duration_s": 1.28,
442
  "frame_clock": "capture timestamps"
443
  },
444
  {
445
- "number": 10,
446
- "start_s": 84.48,
447
  "duration_s": 6.64,
448
  "environment": "WebArena",
449
  "task": "276",
@@ -485,13 +437,13 @@
485
  "shown_arm": "edited",
486
  "task_start_s": 0,
487
  "task_end_s": 5.341348875081167,
488
- "result_still_start_s": 89.84,
489
  "result_still_duration_s": 1.28,
490
  "frame_clock": "capture timestamps"
491
  },
492
  {
493
- "number": 11,
494
- "start_s": 91.12,
495
  "duration_s": 7.12,
496
  "environment": "WebArena",
497
  "task": "277",
@@ -533,13 +485,13 @@
533
  "shown_arm": "edited",
534
  "task_start_s": 0,
535
  "task_end_s": 5.821562124998309,
536
- "result_still_start_s": 96.96,
537
  "result_still_duration_s": 1.28,
538
  "frame_clock": "capture timestamps"
539
  },
540
  {
541
- "number": 12,
542
- "start_s": 98.24,
543
  "duration_s": 8.92,
544
  "environment": "WebArena",
545
  "task": "278",
@@ -581,13 +533,13 @@
581
  "shown_arm": "edited",
582
  "task_start_s": 0,
583
  "task_end_s": 7.614025874994695,
584
- "result_still_start_s": 105.88,
585
  "result_still_duration_s": 1.28,
586
  "frame_clock": "capture timestamps"
587
  },
588
  {
589
- "number": 13,
590
- "start_s": 107.16,
591
  "duration_s": 21.84,
592
  "environment": "WebArena",
593
  "task": "630",
@@ -627,10 +579,10 @@
627
  "shown_arm": "edited",
628
  "task_start_s": 0,
629
  "task_end_s": 20.557142707984895,
630
- "result_still_start_s": 127.72,
631
  "result_still_duration_s": 1.28,
632
  "frame_clock": "capture timestamps"
633
  }
634
  ],
635
- "embedded_chapters": 14
636
  }
 
1
  {
2
+ "sha256": "4918da3d8e665b12719749670f14ed21511ca1511fea261070292e6937f9c800",
3
  "fps": 25,
4
  "resolution": [
5
  1200,
6
  868
7
  ],
8
+ "duration_s": 98.08,
9
+ "task_count": 12,
10
+ "closing_start_s": 94.48,
11
  "playback_rate": 1,
12
  "display_labels": [
13
  "Original weights",
14
  "Edited weights"
15
  ],
16
+ "selection": "Selected demonstration tasks. One uninterrupted edited run per task at capture speed, followed by a labeled result still. Timings use the paired passing runs. The completed-order lookup uses the previously featured fastest take and appears once.",
17
  "chapters": [
18
  {
19
  "number": 1,
 
300
  {
301
  "number": 7,
302
  "start_s": 37.68,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
303
  "duration_s": 6.04,
304
  "environment": "WebArena",
305
  "task": "274",
 
341
  "shown_arm": "edited",
342
  "task_start_s": 0,
343
  "task_end_s": 4.72369825001806,
344
+ "result_still_start_s": 42.44,
345
  "result_still_duration_s": 1.28,
346
  "frame_clock": "capture timestamps"
347
  },
348
  {
349
+ "number": 8,
350
+ "start_s": 43.72,
351
  "duration_s": 6.24,
352
  "environment": "WebArena",
353
  "task": "275",
 
389
  "shown_arm": "edited",
390
  "task_start_s": 0,
391
  "task_end_s": 4.95540237496607,
392
+ "result_still_start_s": 48.68,
393
  "result_still_duration_s": 1.28,
394
  "frame_clock": "capture timestamps"
395
  },
396
  {
397
+ "number": 9,
398
+ "start_s": 49.96,
399
  "duration_s": 6.64,
400
  "environment": "WebArena",
401
  "task": "276",
 
437
  "shown_arm": "edited",
438
  "task_start_s": 0,
439
  "task_end_s": 5.341348875081167,
440
+ "result_still_start_s": 55.32,
441
  "result_still_duration_s": 1.28,
442
  "frame_clock": "capture timestamps"
443
  },
444
  {
445
+ "number": 10,
446
+ "start_s": 56.6,
447
  "duration_s": 7.12,
448
  "environment": "WebArena",
449
  "task": "277",
 
485
  "shown_arm": "edited",
486
  "task_start_s": 0,
487
  "task_end_s": 5.821562124998309,
488
+ "result_still_start_s": 62.44,
489
  "result_still_duration_s": 1.28,
490
  "frame_clock": "capture timestamps"
491
  },
492
  {
493
+ "number": 11,
494
+ "start_s": 63.72,
495
  "duration_s": 8.92,
496
  "environment": "WebArena",
497
  "task": "278",
 
533
  "shown_arm": "edited",
534
  "task_start_s": 0,
535
  "task_end_s": 7.614025874994695,
536
+ "result_still_start_s": 71.36,
537
  "result_still_duration_s": 1.28,
538
  "frame_clock": "capture timestamps"
539
  },
540
  {
541
+ "number": 12,
542
+ "start_s": 72.64,
543
  "duration_s": 21.84,
544
  "environment": "WebArena",
545
  "task": "630",
 
579
  "shown_arm": "edited",
580
  "task_start_s": 0,
581
  "task_end_s": 20.557142707984895,
582
+ "result_still_start_s": 93.2,
583
  "result_still_duration_s": 1.28,
584
  "frame_clock": "capture timestamps"
585
  }
586
  ],
587
+ "embedded_chapters": 13
588
  }
benchmark/speedups-film-receipts.zip CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6ec15cf77a2719470f416ea24aa3e6d0a0ff630016bf81a99a76bd89083cb03e
3
- size 43502
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e4042a68277a2344001c1094c9ffbb582b90b815d665aab4f31204b5981fd05e
3
+ size 38497
benchmark/speedups-timings.json CHANGED
@@ -1,18 +1,17 @@
1
  {
2
- "selection": "One uninterrupted edited run per task at capture speed, followed by a labeled result still. Timings use the paired passing runs. The completed-order lookup uses the previously featured fastest take and appears once.",
3
- "tasks": 13,
4
- "mean_stock_s": 21.76567169232294,
5
- "mean_edit_s": 8.62629472107913,
6
- "time_speedup": 2.5231773775519812,
7
- "time_saved_pct": 60.36743160046034,
8
- "film_sha256": "46730653d4b3e253f90fef69e8bc02de6733a2ec83451ea36edd664192b0c182",
9
- "duration_s": 132.6,
10
  "release_checkpoint_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1",
11
  "expanded_rtx": {
12
  "tasks": [
13
  275,
14
- 190,
15
- 189
16
  ],
17
  "pairs": [
18
  {
@@ -111,104 +110,6 @@
111
  "recording_sha256": "993a915623f7851110f0178c347bd5ae29ad90318f9e9c1c9ace7c0bf53aff2a"
112
  }
113
  },
114
- {
115
- "checkpoint": "search-submit-stop",
116
- "task": 189,
117
- "repeat": 1,
118
- "order": [
119
- "edited",
120
- "original"
121
- ],
122
- "site": "shopping",
123
- "device": "RTX 5090",
124
- "baseline": "Upstream stock",
125
- "comparable": true,
126
- "ratio": 1.962606911217544,
127
- "original": {
128
- "seconds": 6.104229875025339,
129
- "score": 1.0,
130
- "grader_status": "success",
131
- "device": "RTX 5090",
132
- "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
133
- "commands": [
134
- "C1",
135
- "C28",
136
- "ANSWER [754.99]"
137
- ],
138
- "generated": 620,
139
- "thinking": 578,
140
- "inference_seconds": 4.274760375148617,
141
- "receipt_sha256": "c01c6c8aa792e1ae16983c2ad474b966f6c0fcf56558c5ff21edd1113d224e5e",
142
- "recording_sha256": "fc244225b27a04d3f4141fc7041bad661ed2fc4d470cf21c1be67c2399a13d1e"
143
- },
144
- "edited": {
145
- "seconds": 3.1102661669719964,
146
- "score": 1.0,
147
- "grader_status": "success",
148
- "device": "RTX 5090",
149
- "shard_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1",
150
- "commands": [
151
- "C1",
152
- "C28",
153
- "ANSWER [754.99]"
154
- ],
155
- "generated": 22,
156
- "thinking": 0,
157
- "inference_seconds": 0.9284818329615518,
158
- "receipt_sha256": "fbed2ebc713c45514796162f43130b9a237af33a75186cd5b7cc78c07c4974dc",
159
- "recording_sha256": "8e630b90a7db5c4ce8fdd5a116c8f15edc558818d3768b3a1918e5ea093f8ed3"
160
- }
161
- },
162
- {
163
- "checkpoint": "search-submit-stop",
164
- "task": 189,
165
- "repeat": 2,
166
- "order": [
167
- "original",
168
- "edited"
169
- ],
170
- "site": "shopping",
171
- "device": "RTX 5090",
172
- "baseline": "Upstream stock",
173
- "comparable": true,
174
- "ratio": 0.905024135016417,
175
- "original": {
176
- "seconds": 6.321594374952838,
177
- "score": 1.0,
178
- "grader_status": "success",
179
- "device": "RTX 5090",
180
- "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
181
- "commands": [
182
- "C1",
183
- "C28",
184
- "ANSWER [754.99]"
185
- ],
186
- "generated": 620,
187
- "thinking": 578,
188
- "inference_seconds": 4.289602998876944,
189
- "receipt_sha256": "dba8c17ed02c10d7e6039ed0c0c57bf2a4e140fa88433e6fefbfbf18ac996bb4",
190
- "recording_sha256": "4fc5007af77648d75306ee701cf48b69edc2f07e8903e8a6fba553ff1c3e39c7"
191
- },
192
- "edited": {
193
- "seconds": 6.985000874963589,
194
- "score": 1.0,
195
- "grader_status": "success",
196
- "device": "RTX 5090",
197
- "shard_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1",
198
- "commands": [
199
- "C1",
200
- "C1",
201
- "C28",
202
- "",
203
- "ANSWER [754.99]"
204
- ],
205
- "generated": 543,
206
- "thinking": 510,
207
- "inference_seconds": 4.45032141613774,
208
- "receipt_sha256": "04b717751d83fc591366f766bd3cf853a8d254f7212f48b4ea00b0644247b5eb",
209
- "recording_sha256": "3f88cf0b7c3cb68b8f77cbb4cad6eacecaa26d443eca928d72b4ec2ac4245485"
210
- }
211
- },
212
  {
213
  "checkpoint": "search-submit-stop",
214
  "task": 190,
@@ -307,9 +208,9 @@
307
  }
308
  }
309
  ],
310
- "mean_stock_s": 6.136755437512572,
311
- "mean_edit_s": 3.664298895668859,
312
- "time_speedup": 1.6747420481353514
313
  },
314
  "mac_xbox": [
315
  {
@@ -413,5 +314,5 @@
413
  }
414
  }
415
  ],
416
- "all_run_evidence": "https://github.com/jtc268/inkling-2-fast-computer-use/blob/b2a91855302aeae8a595708efccaaab45d243ffe/benchmark/WEBARENA-ORDER-CHECKPOINTS.md"
417
  }
 
1
  {
2
+ "selection": "Selected demonstration tasks. One uninterrupted edited run per task at capture speed, followed by a labeled result still. Timings use the paired passing runs. The completed-order lookup uses the previously featured fastest take and appears once.",
3
+ "tasks": 12,
4
+ "mean_stock_s": 20.758587340348942,
5
+ "mean_edit_s": 6.5751733922516,
6
+ "time_speedup": 3.1571163377701255,
7
+ "time_saved_pct": 68.32552579591346,
8
+ "film_sha256": "4918da3d8e665b12719749670f14ed21511ca1511fea261070292e6937f9c800",
9
+ "duration_s": 98.08,
10
  "release_checkpoint_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1",
11
  "expanded_rtx": {
12
  "tasks": [
13
  275,
14
+ 190
 
15
  ],
16
  "pairs": [
17
  {
 
110
  "recording_sha256": "993a915623f7851110f0178c347bd5ae29ad90318f9e9c1c9ace7c0bf53aff2a"
111
  }
112
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
113
  {
114
  "checkpoint": "search-submit-stop",
115
  "task": 190,
 
208
  }
209
  }
210
  ],
211
+ "mean_stock_s": 6.098677093774313,
212
+ "mean_edit_s": 2.9726315830193926,
213
+ "time_speedup": 2.0516087928998252
214
  },
215
  "mac_xbox": [
216
  {
 
314
  }
315
  }
316
  ],
317
+ "all_run_evidence": "https://github.com/jtc268/inkling-2-fast-computer-use/blob/main/benchmark/WEBARENA-ORDER-CHECKPOINTS.md"
318
  }