husky8's picture
Update the archive film, chapter index and selected timing summary
c4fc9be verified
Raw History Blame Contribute Delete
6.71 kB
{
"film_revision": "57e635d4bbb026b1c54017a41425ac06bd9500f6",
"film_sha256": "719c3a13b5812dc9886e828e4ffefb2f7b3242565917a6ff6eae2cd2e601ffc2",
"duration_s": 246.05,
"chapters": 40,
"webarena_ids": 33,
"grades_revision": "1bde02a4c7addbbe3776762d164f57b30dcf9f23",
"aggregate_definition": "Sum stock elapsed seconds divided by sum edited elapsed seconds. Each included comparison must pass in both arms.",
"film_main_takes": {
"selection": "One scheduled main comparison per chapter against upstream stock. Includes joint passes across several edits. Excludes additional takes and the published-search baseline.",
"summary": {
"pairs": 11,
"stock_total_s": 224.2025544170756,
"edited_total_s": 71.54782129102387,
"mean_stock_s": 20.382050401552327,
"mean_edited_s": 6.504347390093079,
"time_speedup": 3.1336042156353296,
"time_reduction_pct": 68.08786524442262,
"arithmetic_mean_pair_speedup": 3.363958270690793,
"geometric_mean_pair_speedup": 3.2281833349070617
},
"pairs": [
{
"chapter": 1,
"title": "Fill and submit a form",
"task": "Case 02",
"environment": "Selenium",
"device": "M5 Max MacBook Pro",
"checkpoint": "Two-row swap",
"stock_s": 8.643281833035871,
"edited_s": 3.055238249944523
},
{
"chapter": 2,
"title": "Set quantity to two and add the dress",
"task": "Case 03",
"environment": "Selenium + Automation Exercise",
"device": "M5 Max MacBook Pro",
"checkpoint": "Two-row swap",
"stock_s": 16.958890917012468,
"edited_s": 7.152266958029941
},
{
"chapter": 3,
"title": "Complete an autocomplete field",
"task": "Seed 103",
"environment": "BrowserGym · MiniWoB++",
"device": "M5 Max MacBook Pro",
"checkpoint": "Two-row swap",
"stock_s": 15.463160666055046,
"edited_s": 3.3551289580063894
},
{
"chapter": 4,
"title": "Reply to an email",
"task": "Seed 104",
"environment": "BrowserGym · MiniWoB++",
"device": "M5 Max MacBook Pro",
"checkpoint": "Two-row swap",
"stock_s": 26.014819292002358,
"edited_s": 5.98377462499775
},
{
"chapter": 14,
"title": "Find the total of the latest canceled order",
"task": "188",
"environment": "WebArena",
"device": "M5 Max MacBook Pro",
"checkpoint": "Stop-row edit",
"stock_s": 27.82845533301588,
"edited_s": 5.215377749991603
},
{
"chapter": 15,
"title": "Find the total of the latest completed order",
"task": "190",
"environment": "WebArena",
"device": "M5 Max MacBook Pro",
"checkpoint": "Frozen click and stop weights",
"stock_s": 45.501502291997895,
"edited_s": 18.32999724999536
},
{
"chapter": 21,
"title": "Search for a USB Wi-Fi adapter",
"task": "274",
"environment": "WebArena",
"device": "M5 Max MacBook Pro",
"checkpoint": "Published stop-row weights",
"stock_s": 14.633767332998104,
"edited_s": 4.72369825001806
},
{
"chapter": 22,
"title": "Search for Xbox",
"task": "275",
"environment": "WebArena",
"device": "M5 Max MacBook Pro",
"checkpoint": "Click-row edit",
"stock_s": 20.42676358402241,
"edited_s": 4.95540237496607
},
{
"chapter": 23,
"title": "Search for Switch accessories",
"task": "276",
"environment": "WebArena",
"device": "M5 Max MacBook Pro",
"checkpoint": "Published stop-row weights",
"stock_s": 12.93084349995479,
"edited_s": 5.341348875081167
},
{
"chapter": 24,
"title": "Search for iPhone 13 batteries",
"task": "277",
"environment": "WebArena",
"device": "M5 Max MacBook Pro",
"checkpoint": "Published stop-row weights",
"stock_s": 16.96133550000377,
"edited_s": 5.821562124998309
},
{
"chapter": 25,
"title": "Search for green tea bags",
"task": "278",
"environment": "WebArena",
"device": "M5 Max MacBook Pro",
"checkpoint": "Click-row edit",
"stock_s": 18.839734166976996,
"edited_s": 7.614025874994695
}
]
},
"released_checkpoint_rtx": {
"selection": "All four reversed pairs on the Xbox search and completed-order lookup at this release checkpoint.",
"summary": {
"pairs": 4,
"stock_total_s": 24.394708375097252,
"edited_total_s": 11.89052633207757,
"mean_stock_s": 6.098677093774313,
"mean_edited_s": 2.9726315830193926,
"time_speedup": 2.0516087928998252,
"time_reduction_pct": 51.25776398206207,
"arithmetic_mean_pair_speedup": 2.08091726101326,
"geometric_mean_pair_speedup": 1.9743400899908492
},
"pairs": [
{
"task": 275,
"repeat": 1,
"order": [
"edited",
"original"
],
"stock_s": 4.781691708951257,
"edited_s": 3.290172165958211
},
{
"task": 275,
"repeat": 2,
"order": [
"original",
"edited"
],
"stock_s": 4.751382500049658,
"edited_s": 3.002433125046082
},
{
"task": 190,
"repeat": 1,
"order": [
"edited",
"original"
],
"stock_s": 9.295178166008554,
"edited_s": 2.8483604160137475
},
{
"task": 190,
"repeat": 2,
"order": [
"original",
"edited"
],
"stock_s": 5.566456000087783,
"edited_s": 2.74956062505953
}
]
},
"headline_weights": {
"revision": "686faa4d1748f0120008ec007dc97677678e1643",
"branch": "browser-mac-order-trial-3a62b5a9",
"url": "https://huggingface.co/husky8/Inkling-2-Fast-Computer-Use/tree/686faa4d1748f0120008ec007dc97677678e1643",
"shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889",
"auxiliary_files_verified": 11
},
"search_submit_weights": {
"revision": "89ab0e2a2e260fdb9252702a6509b8a4490884cd",
"branch": "browser-search-submit-bbc639c4",
"url": "https://huggingface.co/husky8/Inkling-2-Fast-Computer-Use/tree/89ab0e2a2e260fdb9252702a6509b8a4490884cd",
"shard_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1",
"auxiliary_files_verified": 11
}
}