{ "sha256": "719c3a13b5812dc9886e828e4ffefb2f7b3242565917a6ff6eae2cd2e601ffc2", "fps": 20, "resolution": [ 1200, 900 ], "duration_s": 246.05, "task_count": 40, "opening_s": 6.15, "closing_start_s": 242.55, "selection": "Selected recorded browser tasks in benchmark order. Task times cover the full run. The film includes passing and failing grades.", "chapters": [ { "number": 1, "start_s": 6.15, "duration_s": 4.25, "environment": "Selenium", "task": "Case 02", "title": "Fill and submit a form", "shared_elapsed_start_s": 0, "arms": [ { "arm": "Original", "elapsed_s": 8.643281833035871, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "a1e00b60607d136ddeca47529834e417bdb2b17e1266bafca808ee1ee4f96e36", "receipt": "02-original.json", "recording": { "kind": "native browser recording", "source_sha256": "c094391f4bdc6545669730fd7dc5dfe30137e5ef2b63c178921d783a4de7f8bc", "start_s": 0.04, "source_duration_s": 15.44, "available_frames": 85 } }, { "arm": "Two-row swap", "elapsed_s": 3.055238249944523, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "6d5e5c9ac411ce3464e3c94b5149c1d633abd315acb6a7c33402604da815182c", "receipt": "02-release_swap.json", "recording": { "kind": "native browser recording", "source_sha256": "f80ae6b4df0da070fa403b8b00a77542577ae73977a4a16fa5645f6f977dcc65", "start_s": 0.0, "source_duration_s": 8.84, "available_frames": 63 } } ] }, { "number": 2, "start_s": 10.4, "duration_s": 4.25, "environment": "Selenium + Automation Exercise", "task": "Case 03", "title": "Set quantity to two and add the dress", "shared_elapsed_start_s": 3.602266958029941, "arms": [ { "arm": "Original", "elapsed_s": 16.958890917012468, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "40ccf6dc2ebf5387bf9bf807a770404c3ac5ea6808df9e157f9f4ce664321291", "receipt": "03-original.json", "recording": { "kind": "native browser recording", "source_sha256": "cb68912b3f5d8c99bbca757b933501f4e4978a9de8b4c47966faa5418ca5394f", "start_s": 3.642266958029941, "source_duration_s": 18.44, "available_frames": 85 } }, { "arm": "Two-row swap", "elapsed_s": 7.152266958029941, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "54049cfc3bade3cc994902ea470824f215579aca05ddb7b4acc5f0e72de6b21d", "receipt": "03-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "7da366d3d7889ee801626bf6c4a6f38104490ccd51dab650ebda4f28d44b1c4d", "start_s": 3.602266958029941, "source_duration_s": 8.12, "available_frames": 73 } } ] }, { "number": 3, "start_s": 14.65, "duration_s": 4.75, "environment": "BrowserGym · MiniWoB++", "task": "Seed 103", "title": "Complete an autocomplete field", "shared_elapsed_start_s": 0, "arms": [ { "arm": "Original", "elapsed_s": 15.463160666055046, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "ad8aa2d68a7798effdfa3d80104409c76e4d4317f4aef7a43cc28f6085c269dc", "receipt": "use-autocomplete-103-original.json", "recording": { "kind": "timestamped browser captures", "start_s": 0, "source_sha256": "3768c12d71ff98aafc5104cb764816e43a53371c8a5b5532d9a33df538c04d0f", "timing_manifest_sha256": "08490f9e128d0d41dc1da13f69388e39b2888f9acc2a2802fb83ece1f6bca894", "frame_count": 123 } }, { "arm": "Two-row swap", "elapsed_s": 3.3551289580063894, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "9d89431a3f116bc20716074fb85f73dc22f9b8059bbf516cb9a3db1393c608ba", "receipt": "use-autocomplete-103-edited.json", "recording": { "kind": "timestamped browser captures", "start_s": 0, "source_sha256": "f01a27f87a8e46e5921345e4e56d68636eca684525547d67abd81371958968a2", "timing_manifest_sha256": "3a2e2d6a4a2f3fc2ec4ad2de24126f5c20c9091ccd3089312d6b5d7f3f13a5d7", "frame_count": 28 } } ] }, { "number": 4, "start_s": 19.4, "duration_s": 4.75, "environment": "BrowserGym · MiniWoB++", "task": "Seed 104", "title": "Reply to an email", "shared_elapsed_start_s": 1.9337746249977499, "arms": [ { "arm": "Original", "elapsed_s": 26.014819292002358, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "1e180cd54134623afe13834a76e83bfc5367d8ed17b4ebc6c60b0f7f502584a1", "receipt": "email-inbox-104-original.json", "recording": { "kind": "timestamped browser captures", "start_s": 1.9337746249977499, "source_sha256": "1b8cda1812e06a017e5b2dc72a7e25fdefb16d89e0cfffb25ee081a9e38878f7", "timing_manifest_sha256": "27eb1ae06a81a62873e5e94ddfc5c3eef534ddaf7d76624fb5ae93557707ab60", "frame_count": 206 } }, { "arm": "Two-row swap", "elapsed_s": 5.98377462499775, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "e4c9072125bf488bd278eea39bb069d4144453dd3dec8aafca42fe65c26b3526", "receipt": "email-inbox-104-edited.json", "recording": { "kind": "timestamped browser captures", "start_s": 1.9337746249977499, "source_sha256": "ed5227ad3c7a9a44c11b380bd85960c691ee8cc46fa1be754a45061d0dacf3b1", "timing_manifest_sha256": "c0284e5f6372e4d63e1a835bb30390cca3ed62ec86514b1602a44b366e990b5c", "frame_count": 49 } } ] }, { "number": 5, "start_s": 24.15, "duration_s": 2.95, "environment": "BrowserGym · MiniWoB++", "task": "Seed 101", "title": "Like every post from one account", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 20.16481412493158, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "4c14d15d582ab4266aab07f068e91f1585ea48eb03d8bf543c13b0dfd1c530e3", "receipt": "social-media-all-101-original.json", "recording": { "kind": "timestamped browser captures", "start_s": 17.214814124931582, "source_sha256": "5a6d0777378409f6eef724dcc619730a421370a5c1e477ff1a45ac13026e7544", "timing_manifest_sha256": "726a597396101b4d3412f6e0631b4dd9dab77ddd379a54077b2029b7021db09e", "frame_count": 160 } }, { "arm": "Two-row swap", "elapsed_s": 5.829392791958526, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "9c72c15bd32d0d89bd95bf1ef85d9b880d506629e8be014c8f718a1b3e97d5eb", "receipt": "social-media-all-101-edited.json", "recording": { "kind": "timestamped browser captures", "start_s": 2.8793927919585256, "source_sha256": "0faed0ad2dbd2480afdc77f1b1844ea53750f69d3560f4eed356991e223b63a2", "timing_manifest_sha256": "83c29e31ae88d6a20a5fe55cca737ed51ccd2f541057d788296b0f1ae7164407", "frame_count": 47 } } ] }, { "number": 6, "start_s": 27.1, "duration_s": 2.95, "environment": "BrowserGym · MiniWoB++", "task": "Seed 101", "title": "Pick a date", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 21.568157291971147, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "ac9c8f8ca21273947820aab7456d0e1cfa79a420d4d8a0f7d90daee765d0a063", "receipt": "choose-date-101-original.json", "recording": { "kind": "timestamped browser captures", "start_s": 18.618157291971148, "source_sha256": "b37c67102485f4ac79f9b399599b0cc6e9d60496fe7262e0874d744040b9bef4", "timing_manifest_sha256": "6a2c829c951ea6e6c771defe86e7e0a2213e314e26cc88c8d884deed80ae6b77", "frame_count": 172 } }, { "arm": "Two-row swap", "elapsed_s": 16.94887112500146, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "5d88b3b5b2b118224bd0980c69baefa7d21898228eb94f923340d02ded3c361f", "receipt": "choose-date-101-edited.json", "recording": { "kind": "timestamped browser captures", "start_s": 13.998871125001461, "source_sha256": "ff657f909934323620646be6a81041a502f8925f67b1a80d9c89f0f0c803cd1a", "timing_manifest_sha256": "27ea078b875eaab02210909e61032b7fc6597c577002938dbd03e4e2092c5897", "frame_count": 135 } } ] }, { "number": 7, "start_s": 30.05, "duration_s": 2.95, "environment": "BrowserGym · MiniWoB++", "task": "Seed 101", "title": "Book the shortest one-way flight", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 63.24608595808968, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "f5af0d7228f868bdb87cd815c0833364e82608948c40f9adb747acfda59a858c", "receipt": "book-flight-101-original.json", "recording": { "kind": "timestamped browser captures", "start_s": 60.29608595808968, "source_sha256": "a54ebcf05f3b95ff7395113d60fa3e737eb433ba2455d97b5203361c50d5320e", "timing_manifest_sha256": "31fbe4c309f615aa635d72f095479e35e6178af934223f7e849fb2a509a53b1c", "frame_count": 499 } }, { "arm": "Two-row swap", "elapsed_s": 90.67623720807023, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "3f5b947b0621d1c2bff09cdda3639257642bf703a6720d0ad96cfc69727fe7e9", "receipt": "book-flight-101-edited.json", "recording": { "kind": "timestamped browser captures", "start_s": 87.72623720807023, "source_sha256": "94d7378ee86e6796df7005545270edf539d713b1447a4f14a06574d9bad8587b", "timing_manifest_sha256": "0664f31709cf1cf0b2082180e305690e8a405d509985c83b0101819869d701a8", "frame_count": 713 } } ] }, { "number": 8, "start_s": 33.0, "duration_s": 2.2, "environment": "WebArena", "task": "21", "title": "Read headphone reviews", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 14.290919666993432, "passed": false, "device": "RTX 5090", "receipt_sha256": "67dfd0d778b9e046f1a0a35fb2c3f3b83fb3b4467be291c3cbf141b133aa0309", "receipt": "021-original.json", "recording": { "kind": "native browser recording", "source_sha256": "52f058d3faeb02f519bde468ee6997b91bf1e3e6d2dd4f02673b10e51edf314d", "start_s": 13.05, "source_duration_s": 15.25, "available_frames": 44 } }, { "arm": "Selective edit", "elapsed_s": 9.804292374989018, "passed": false, "device": "RTX 5090", "receipt_sha256": "06634e5687f057641486f0c070e283d7e6d989105a403675b980c0d69f7055c3", "receipt": "021-selective.json", "recording": { "kind": "native browser recording", "source_sha256": "97ec666ed9f1215a484e727965837f815666ba603261804e293fe021e9d8e781", "start_s": 8.600000000000001, "source_duration_s": 10.8, "available_frames": 44 } } ] }, { "number": 9, "start_s": 35.2, "duration_s": 2.2, "environment": "WebArena", "task": "27", "title": "Read the latest finance discussion", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 12.017803041962907, "passed": false, "device": "RTX 5090", "receipt_sha256": "8885db630d211e440d3d723c64d32d493627a16d92c0c7e8949beae97403b97f", "receipt": "027-original.json", "recording": { "kind": "native browser recording", "source_sha256": "bc559b70f6a20fb9f944f9cdfe41a9c80e6dd5ec7e6bac332a88f1c5223fb455", "start_s": 9.8, "source_duration_s": 12.0, "available_frames": 44 } }, { "arm": "Selective edit", "elapsed_s": 1.0350043339421973, "passed": false, "device": "RTX 5090", "receipt_sha256": "ee0cb22aba3180b8e63b9751be0e6f078b6e294a82c93ec56edcd5c9dd15eb90", "receipt": "027-selective.json", "recording": { "kind": "native browser recording", "source_sha256": "b7ebacb3fbbf9aa66c05120beaaa082afc6e8e6e475f533290bd9b451ed32604", "start_s": 0, "source_duration_s": 2.05, "available_frames": 41 } } ] }, { "number": 10, "start_s": 37.4, "duration_s": 2.2, "environment": "WebArena", "task": "62", "title": "Find the customer with the most completed orders", "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 75.4982749170158, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "3e081086af99c49c181ece856f66992d59331e4332aadf1d33fa4d54a51b7f73", "receipt": "062-original.json", "recording": { "kind": "native browser recording", "source_sha256": "8b121887d4ef926468b58cc5873d2a2a737180e5e237be6aef8d813dc6353387", "start_s": 74.6984193749493, "source_duration_s": 77.9, "available_frames": 44 }, "grader_status": "failure" }, { "arm": "Frozen click and stop weights", "elapsed_s": 0.7192069169832394, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "b1e4632555bd70819b5b05162b66bb945dcc83480914ac92f79ed9945dd0df90", "receipt": "062-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "916a14f93614ba687b3cb27bd38f4b31af8cb6ecdc6ef1762e6774af7e68cc51", "start_s": 0, "source_duration_s": 1.75, "available_frames": 17 }, "grader_status": "failure" } ] }, { "number": 11, "start_s": 39.6, "duration_s": 2.2, "environment": "WebArena", "task": "66", "title": "Find book recommendations", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 93.16534770908765, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "2ef0a76cbe8fed96735175027e4db53a6414138afc6caf3f105572ded0a2aa4d", "receipt": "066-original.json", "recording": { "kind": "native browser recording", "source_sha256": "4ed4ce9f2135395cccaa6ddbde34f2de413ad1c619c5455edbdf3800ba25b3b1", "start_s": 91.95, "source_duration_s": 94.15, "available_frames": 44 } }, { "arm": "Selective edit", "elapsed_s": 32.15284504101146, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "c6db2eef52a9b7b7d671a810907bcbf995566a4e233f288125811776cd399b35", "receipt": "066-selective.json", "recording": { "kind": "native browser recording", "source_sha256": "d750b2db988c7a8ef4a824e7a4d8926433565a4b24275b2aba1b8e65402b95ee", "start_s": 30.95, "source_duration_s": 33.15, "available_frames": 44 } } ] }, { "number": 12, "start_s": 41.8, "duration_s": 13.35, "environment": "WebArena", "task": "96", "title": "Track the latest order", "shared_elapsed_start_s": 2.4432596249505876, "arms": [ { "arm": "Published search weights", "elapsed_s": 46.71270216605626, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "ba4155116b598270399c505a9bafef59c4d5f18e5eeec94b4164d41603cbfe35", "receipt": "096-original.json", "recording": { "kind": "native browser recording", "source_sha256": "a5eec98a9d3b8b1d7fd392442b2921648e280d51bde5595b9ca7cd5b97e95660", "start_s": 2.4439413339830933, "source_duration_s": 46.7, "available_frames": 103 }, "grader_status": "success" }, { "arm": "Stop-row edit", "elapsed_s": 6.893259624950588, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "e7cdc0cc51e500151f87c21e2f2ed0c0b1cb255531f0b38d9d286d7f494eb0c2", "receipt": "096-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "703012eb551efa4e4581cd97e1d0f6ef118cc9bb5048b8eac514c75da78cba0f", "start_s": 2.444003125024028, "source_duration_s": 6.9, "available_frames": 89 }, "grader_status": "success" } ], "additional_takes": [ { "start_s": 46.95, "duration_s": 2.55, "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 65.1210219169734, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "134d2ad8ea2350d8fa12c6e90b062df2294a77c17c2ea04eee43ff9946855caa", "receipt": "096-original.json", "recording": { "kind": "native browser recording", "source_sha256": "8619fafe57ce53b71b85965a667805388cccb7abd4cff7119f5898b552606f13", "start_s": 62.57169170798734, "source_duration_s": 66.1, "available_frames": 51 }, "grader_status": "failure" }, { "arm": "Native click-row edit", "elapsed_s": 50.31726612499915, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "2197d30cf11c20449f856fcaf2068d650f7655eb69c77bc3abc1ea4053e0a190", "receipt": "096-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "5006de2337810b8f4747b92fe490f828415061e0127a24d4e5e23a53550021f4", "start_s": 47.75, "source_duration_s": 50.3, "available_frames": 51 }, "grader_status": "success" } ] }, { "start_s": 49.5, "duration_s": 2.75, "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 26.085084333084524, "passed": false, "device": "RTX 5090", "receipt_sha256": "64e17086125e8b7415df7d67bc3c392bf5d9e0607b0fa01e2d3adca800e08bd0", "receipt": "096-original.json", "recording": { "kind": "native browser recording", "source_sha256": "e9ac1aae1feb9aed31aec0cf50ffcef31b83672800d669620bd6c35252939b2a", "start_s": 23.3, "source_duration_s": 26.05, "available_frames": 55 }, "grader_status": "failure" }, { "arm": "Stop-row edit", "elapsed_s": 12.522111167083494, "passed": true, "device": "RTX 5090", "receipt_sha256": "ca17c3b57b21d5ba173a873789c99f78ffc47b5b00c754c206b55518396063eb", "receipt": "096-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "78fd4bc93937f72e8c02cd87f2679dc34d0c0c5a6b056329be2cc067f71b3e4f", "start_s": 9.772704167058691, "source_duration_s": 13.2, "available_frames": 55 }, "grader_status": "success" } ] }, { "start_s": 52.25, "duration_s": 2.9, "shared_elapsed_start_s": 4.708922334085218, "arms": [ { "arm": "Published search weights", "elapsed_s": 48.335796666913666, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "f1c8df045f1cddb4282f5bbff2d96b64feb52187c56d4f156a241cc463597709", "receipt": "096-original.json", "recording": { "kind": "native browser recording", "source_sha256": "b913f6b44e7bc8a909937cb54cb7a111641675eca31c23373fec959ad21525af", "start_s": 4.709638125100173, "source_duration_s": 48.35, "available_frames": 58 }, "grader_status": "success" }, { "arm": "Cross-device order stop", "elapsed_s": 6.908922334085219, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "61f1d15c0471b7d9c1931eca1246f6b57dcd787f70767ffc8b182e59c288eb7a", "receipt": "096-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "b4b9cfb3f5cde9985ba1f8edeaee13168a9d1ba2d38d6cf6da88218cce5c2bcf", "start_s": 4.709668625099584, "source_duration_s": 6.9, "available_frames": 43 }, "grader_status": "success" } ] } ] }, { "number": 13, "start_s": 55.15, "duration_s": 2.2, "environment": "WebArena", "task": "163", "title": "Find low-star review titles", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 2.5146072920178995, "passed": false, "device": "RTX 5090", "receipt_sha256": "d6b0f1ad89ad5dbbe2af4de8261561c090531ae4717185725bb83d723e80cbb4", "receipt": "163-original.json", "recording": { "kind": "native browser recording", "source_sha256": "37b42a6f830145e8b01e6cec8c6d28fb89aa8fe801abe9919e3fe76d5ddfaeba", "start_s": 1.2999999999999998, "source_duration_s": 3.5, "available_frames": 44 } }, { "arm": "Selective edit", "elapsed_s": 0.4777584580006078, "passed": false, "device": "RTX 5090", "receipt_sha256": "3cde6bdc0aa98f6ce25ad143de37b65b65e52f515f84b6c172be435634481cd8", "receipt": "163-selective.json", "recording": { "kind": "native browser recording", "source_sha256": "0c611de0ed923a1e4783be502e71797e3651f18b222148e5b80586284ef23572", "start_s": 0, "source_duration_s": 1.5, "available_frames": 30 } } ] }, { "number": 14, "start_s": 57.35, "duration_s": 17.75, "environment": "WebArena", "task": "188", "title": "Find the total of the latest canceled order", "shared_elapsed_start_s": 1.8153777499916035, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 27.82845533301588, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "e8d4bf968632a8cef22b38ac5064ef06f481f8f575b43c325cc54dea1df69ad5", "receipt": "188-original.json", "recording": { "kind": "native browser recording", "source_sha256": "ec642f8c46a74743cac1669a727e102b9abc253393196d9deb9ad44b050d6577", "start_s": 1.8161042080260816, "source_duration_s": 27.8, "available_frames": 82 }, "grader_status": "success" }, { "arm": "Stop-row edit", "elapsed_s": 5.215377749991603, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "8fbb44b3a0bce442b32c1723080e9bfa2b2f358175f3d6aea8b902e7ba5d32ca", "receipt": "188-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "4d72502042f6f1d1e91ea088b8e29a95929ef8f80a60d9e45185d6ee83c3122f", "start_s": 1.8161005830857906, "source_duration_s": 5.45, "available_frames": 69 }, "grader_status": "success" } ], "additional_takes": [ { "start_s": 61.45, "duration_s": 2.55, "shared_elapsed_start_s": 4.4611123749986294, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 6.455109708942473, "passed": true, "device": "RTX 5090", "receipt_sha256": "eb515e24359a996a59494fbecf121524067d73640157d5f4bfd748738df8a64f", "receipt": "188-original.json", "recording": { "kind": "native browser recording", "source_sha256": "e422aff860a9fc557cfeb046b11dcef4cb58e50dfd28e03725f43cfdd17b3e40", "start_s": 4.462007375061512, "source_duration_s": 6.5, "available_frames": 40 }, "grader_status": "success" }, { "arm": "Stop-row edit", "elapsed_s": 6.311112374998629, "passed": true, "device": "RTX 5090", "receipt_sha256": "b9d6575e06a3459f14d064e922fb9b89d6ac607e2fc258d5ce03f163c855f157", "receipt": "188-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "70a0c736d6425b339e374a6f4463fdbf459dcf597c0c5fb38d27de840cab74c6", "start_s": 4.461929707997479, "source_duration_s": 7.1, "available_frames": 38 }, "grader_status": "success" } ] }, { "start_s": 64.0, "duration_s": 2.65, "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 9.634521208005026, "passed": true, "device": "RTX 5090", "receipt_sha256": "9c7e5a8cf0b3b67cd41703cde5d602223e99dccf5e561fa4e798189c560a5a0f", "receipt": "188-original.json", "recording": { "kind": "native browser recording", "source_sha256": "b4de1dbecbcf5d668e75ad47e9e47890390be59d66a813e988bdb450a033a35f", "start_s": 6.985210167034529, "source_duration_s": 9.65, "available_frames": 53 }, "grader_status": "success" }, { "arm": "Order click edit", "elapsed_s": 6.733829957898706, "passed": false, "device": "RTX 5090", "receipt_sha256": "4d39fdeeb7dabf9ac09ff8bee394d4d4b95b32025309bf43f9fcf30108b48610", "receipt": "188-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "db00d2cd44f079b54492c7e7f1e7a7a34a4b91f5bb61b64385ab805a5fd4c846", "start_s": 4.084594207955524, "source_duration_s": 7.5, "available_frames": 53 }, "grader_status": "failure" } ] }, { "start_s": 66.65, "duration_s": 2.55, "shared_elapsed_start_s": 0.7962742909789087, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 6.857609499944374, "passed": true, "device": "RTX 5090", "receipt_sha256": "c7cec829eb022d0eedbe230b27a36d6ee5346ccd33ac284c0703eed777f3f584", "receipt": "188-original.json", "recording": { "kind": "native browser recording", "source_sha256": "fb5ff40e56daa8f3b1bb3ec7cf75c4737be808a7451d370246bb6cb44779c593", "start_s": 0.7970619160681964, "source_duration_s": 6.85, "available_frames": 51 }, "grader_status": "success" }, { "arm": "Order boundary edit", "elapsed_s": 2.6462742909789085, "passed": true, "device": "RTX 5090", "receipt_sha256": "f28317df706ae824db182c96082d262e272b6c3c545cf23251c189b8db5fe92a", "receipt": "188-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "dfd9521b812a2461962c05936ce0bf25ea5c2f998355315dc07d7ca122247807", "start_s": 0.7968766659963877, "source_duration_s": 3.5, "available_frames": 38 }, "grader_status": "success" } ] }, { "start_s": 69.2, "duration_s": 2.95, "shared_elapsed_start_s": 4.191279500024393, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 45.752225124975666, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "9eab04dc7cff27e3a5092aab8a335560ba1b4f005dd0447e515a22df8198aef8", "receipt": "188-original.json", "recording": { "kind": "native browser recording", "source_sha256": "441458b2d3e51e08b564656b24c2c6e591ce02670f9b444904aa0af865c9e537", "start_s": 4.192172999959439, "source_duration_s": 45.75, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Completed-order stop edit", "elapsed_s": 6.441279500024393, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "6ca9c942603d1cb1047e49b31e961627fa3276c2ee639fa397f45203b2d61380", "receipt": "188-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "5b8f672631de970383c6adcfe4e3995a2cb1ddb5422085d72fc3656c5b2073f6", "start_s": 4.19245058298111, "source_duration_s": 6.65, "available_frames": 46 }, "grader_status": "success" } ] }, { "start_s": 72.15, "duration_s": 2.95, "shared_elapsed_start_s": 0.4969808330060912, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 9.687184332986362, "passed": true, "device": "RTX 5090", "receipt_sha256": "cdc35d8bbdee31f92e5940f640e5d65584e4fa5f1d7dfa23d48c722cc3aa7506", "receipt": "188-original.json", "recording": { "kind": "native browser recording", "source_sha256": "408df306a04a38c47902fd374b9fea8cea2a379b66a0fae370f18059428a7d3f", "start_s": 0.4976228330051524, "source_duration_s": 9.65, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Order-history routing trial", "elapsed_s": 2.7469808330060914, "passed": true, "device": "RTX 5090", "receipt_sha256": "b8fa575755735d911468f7fe35395a5c56e0954cb79df2458b9f415b252433fe", "receipt": "188-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "da565e7526d5613ac7b4a6405214591cd8b2b706fd9bc474830627469f8fce1d", "start_s": 0.49799595808144637, "source_duration_s": 3.35, "available_frames": 46 }, "grader_status": "success" } ] } ] }, { "number": 15, "start_s": 75.1, "duration_s": 33.15, "environment": "WebArena", "task": "190", "title": "Find the total of the latest completed order", "shared_elapsed_start_s": 15.379997249995357, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 45.501502291997895, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "0c5039a54ece418d72927e94ef46bf347afc2f7c4328924b409d0bd8029f30b9", "receipt": "190-original.json", "recording": { "kind": "native browser recording", "source_sha256": "aef068d2419ff13564764c7d28509ff34185b09fa4e7c68e4365eba05171aa2f", "start_s": 15.380713749979622, "source_duration_s": 45.5, "available_frames": 73 }, "grader_status": "success" }, { "arm": "Frozen click and stop weights", "elapsed_s": 18.32999724999536, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "c7266a5d572d38b90d3949ed3958f7a547c3e495cd2d2d13089c111af9e022d9", "receipt": "190-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "e2ecd40026fc920fedc2bd53366aed4636b1e97fc769c771694a3f1516581c5a", "start_s": 15.380719750025309, "source_duration_s": 18.55, "available_frames": 61 }, "grader_status": "success" } ], "additional_takes": [ { "start_s": 78.75, "duration_s": 2.95, "shared_elapsed_start_s": 2.940150791895576, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 45.33590604097117, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "b944a6bd099412a0bad414e64b4990c7b99d8d9c25c191cd040399511f710f3e", "receipt": "190-original.json", "recording": { "kind": "native browser recording", "source_sha256": "942526c0551c329e1ac489749aad074e1ef028e893eb28801167a6512d218e66", "start_s": 2.940871750935912, "source_duration_s": 45.35, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Completed-order stop edit", "elapsed_s": 5.190150791895576, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "84cf20808be895d2e05f3534cac15b07c016dc676049a987a70ae08f0f108be2", "receipt": "190-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "7832813623fc7a3cf56fe92630aa5d7346ee07478b94d24bafffad56ced8b3fe", "start_s": 2.9408789589069784, "source_duration_s": 5.4, "available_frames": 46 }, "grader_status": "success" } ] }, { "start_s": 81.7, "duration_s": 2.95, "shared_elapsed_start_s": 6.681119083077647, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 12.153708999976516, "passed": true, "device": "RTX 5090", "receipt_sha256": "2c7a4e2807c73cd87a088b3d20e2d958db61fed9c14b8317980503920b3be701", "receipt": "190-original.json", "recording": { "kind": "native browser recording", "source_sha256": "24915a2e17337a0b0d87f7cfcdc83f58cd02e4c8d9b541a6811426a06e940a17", "start_s": 6.681789208087139, "source_duration_s": 12.1, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Completed-order stop edit", "elapsed_s": 8.931119083077647, "passed": true, "device": "RTX 5090", "receipt_sha256": "72fc0400d00b8206a2d5ed97e3056e362a4f7c0d05a0e932d414b6ae97611a00", "receipt": "190-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "bc1d171a325af34b00b81ba751c1d2f23b8a0a8b8ac5d6496264456aedbe319c", "start_s": 6.681948500103317, "source_duration_s": 9.8, "available_frames": 46 }, "grader_status": "success" } ] }, { "start_s": 84.65, "duration_s": 2.95, "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 9.333173334016465, "passed": true, "device": "RTX 5090", "receipt_sha256": "583e40d83a2dae419c463e1a58a2f47de180396ad80763f32dbbbf3a55d52fe7", "receipt": "190-original.json", "recording": { "kind": "native browser recording", "source_sha256": "ca72633366aff0727d18d68f0fdb00af50b53630d994725e56f2055f9006fbb9", "start_s": 6.3500000000000005, "source_duration_s": 9.3, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Cross-device order stop", "elapsed_s": 3.2936403329949826, "passed": false, "device": "RTX 5090", "receipt_sha256": "c9c7fe2cf441982418f578852a90317ed712a0ff675a75864085f40825ccd304", "receipt": "190-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "8d5ee987796266a46304fc15dffd7e24e748c345cc6dae4bf3904712d795bba6", "start_s": 0.34454195802099985, "source_duration_s": 4.15, "available_frames": 59 }, "grader_status": "failure" } ] }, { "start_s": 87.6, "duration_s": 2.95, "shared_elapsed_start_s": 2.8771139580057934, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 45.88679650006816, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "db7cc933937f077603d615ef906b8ccd107f4b80f9e4303af200341f55a6e809", "receipt": "190-original.json", "recording": { "kind": "native browser recording", "source_sha256": "001a63b86d63e645ef77f990d03c3f2b17ea4dc17c1d6a34cc91e3b4126bca25", "start_s": 2.8778829999500886, "source_duration_s": 45.9, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Cross-device order stop", "elapsed_s": 5.127113958005793, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "de99cd3b9ffdab3dcde0a6ba3b7ab53c4f78c893b5e04a1a42cd729a32ca8816", "receipt": "190-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "1c0b7325f4c3119182342a5017acce4187ed18802c46dddd919c151140565054", "start_s": 2.8778513750294223, "source_duration_s": 5.35, "available_frames": 46 }, "grader_status": "success" } ] }, { "start_s": 90.55, "duration_s": 2.95, "shared_elapsed_start_s": 1.092322333017364, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 10.326355000026524, "passed": true, "device": "RTX 5090", "receipt_sha256": "1c087c5814f1b002e1ea9377eda5ae25767e0362adbf324db50698ae35aa14bd", "receipt": "190-original.json", "recording": { "kind": "native browser recording", "source_sha256": "75627feca13b116dd5c8711c21396b0ad8ba92904f6d6985cfdcdff2e39128dd", "start_s": 1.093227708013728, "source_duration_s": 12.9, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Guarded reasoning trial", "elapsed_s": 3.342322333017364, "passed": true, "device": "RTX 5090", "receipt_sha256": "57474389b1de11eecd972d4151cf8e8887e673c33bb669705efa6b8966067709", "receipt": "190-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "05bf4f59c5b86c85d42dd5b4919c9a63d0d52aea1c6bfbfb93ca0e322f79aee9", "start_s": 1.0937854580115525, "source_duration_s": 4.2, "available_frames": 46 }, "grader_status": "success" } ] }, { "start_s": 93.5, "duration_s": 2.95, "shared_elapsed_start_s": 2.38902004098054, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 10.092377167078666, "passed": true, "device": "RTX 5090", "receipt_sha256": "d80a54f157029a2fdf18c152cf4c9be428055a5574ecf70100a031d63422eb85", "receipt": "190-original.json", "recording": { "kind": "native browser recording", "source_sha256": "10264730f938c44c0831997391e4afcfea057cba7c97f31043653d8e58856865", "start_s": 2.3896774579770863, "source_duration_s": 12.65, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Order-history routing trial", "elapsed_s": 4.63902004098054, "passed": true, "device": "RTX 5090", "receipt_sha256": "0c0427e0d66d742d46459dde8b9b9cbeeb7744057b831a49ef28fe71d636c16a", "receipt": "190-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "34d4f7f84ec4a464b2cf9ad45dc74950ecf4177b43691f46aa0d71bdc3512397", "start_s": 2.3896975410170853, "source_duration_s": 5.5, "available_frames": 46 }, "grader_status": "success" } ] }, { "start_s": 96.45, "duration_s": 2.95, "shared_elapsed_start_s": 0.5983604160137472, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 9.295178166008554, "passed": true, "device": "RTX 5090", "receipt_sha256": "10e113f940c9e247fd8334c79dbad2180d4601a67a2dc440581a33b0dd942ed8", "receipt": "190-original.json", "recording": { "kind": "native browser recording", "source_sha256": "3cbd6436db955b45efb52b2295633470ed96ded06e28bfaa7a7d9a23acadee81", "start_s": 0.599131125025451, "source_duration_s": 9.25, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Search submit stop", "elapsed_s": 2.8483604160137475, "passed": true, "device": "RTX 5090", "receipt_sha256": "2944f50cd972d2d335dead10baae03824517bccd51345bb5254e87b397ecbb44", "receipt": "190-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "b6aa451ecac294a4142363a3ebec5555bda2325574d81e31a24ab89d071345d0", "start_s": 0.59967129107099, "source_duration_s": 3.45, "available_frames": 46 }, "grader_status": "success" } ] }, { "start_s": 99.4, "duration_s": 2.95, "shared_elapsed_start_s": 2.878219416947104, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 44.78415783296805, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "81671d792843cc52402bf24f0314e1cb015a940fa171d07cb0354021766938d1", "receipt": "190-original.json", "recording": { "kind": "native browser recording", "source_sha256": "f2dfbabfe77f82e4515c5d0a302ae94ead3fa360478d4c05c5b87932ba21c770", "start_s": 2.8790539589244872, "source_duration_s": 44.75, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Search submit stop", "elapsed_s": 5.128219416947104, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "c411be851762f8cd21854df7d9fee8b198d0d7096d8087eae99c342405e246bd", "receipt": "190-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "f3872bae954176fc4003c34437964000593da0774029c6f29e051f1f06e44a2f", "start_s": 2.8790000419830903, "source_duration_s": 5.35, "available_frames": 46 }, "grader_status": "success" } ] }, { "start_s": 102.35, "duration_s": 2.95, "shared_elapsed_start_s": 3.5121810829732567, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 47.179218083037995, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "99f4d7ef78dd466de0df077ee455cba9f3e8cd3517bc0a94ef33ff6f8aa51cc8", "receipt": "190-original.json", "recording": { "kind": "native browser recording", "source_sha256": "4974c4c2a4452b1874bee80a103ad38b77124bc03ddd05c1cacd41603c8e6258", "start_s": 3.512865124968812, "source_duration_s": 47.15, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Order-history routing trial", "elapsed_s": 5.762181082973257, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "4ee31a78c56726cb5e325af372acb7cee4ba4d7894ed7ae0d72e06ab645423e1", "receipt": "190-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "638a31430f6f8abb2169b4b5b087cdefe93d800f4cde76836b2df959cb35f3cc", "start_s": 3.513225583010353, "source_duration_s": 5.95, "available_frames": 46 }, "grader_status": "success" } ] }, { "start_s": 105.3, "duration_s": 2.95, "shared_elapsed_start_s": 3.061038124957122, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 45.43856291694101, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "45df24edf60219d0e61d64a02c69e35e048d11f7d491ee3b2109cfd85cf2cc71", "receipt": "190-original.json", "recording": { "kind": "native browser recording", "source_sha256": "ffbcc9cad73b37a6350063524e03cd3e3643732816e9f4e15dfbf481238e3b0b", "start_s": 3.061735332943499, "source_duration_s": 45.4, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Product-link selection edit", "elapsed_s": 5.311038124957122, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "c465c7b586943e052d903a67139c94bf11c27aaeb1af0762f721614009925fb2", "receipt": "190-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "5727b53a048fbdc1e198bb17f97cbbb3a63ca7f899f2c118d85bbe06914221ba", "start_s": 3.0618657909799367, "source_duration_s": 5.55, "available_frames": 46 }, "grader_status": "success" } ] } ] }, { "number": 16, "start_s": 108.25, "duration_s": 8.1, "environment": "WebArena", "task": "191", "title": "Find the total of the latest processing order", "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 72.89141737495083, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "1b0f760548d1a346c64962ec9d28e96599619d967e6a45ed79596b7c9ecaf44b", "receipt": "191-original.json", "recording": { "kind": "native browser recording", "source_sha256": "68c336263c3ddb7e33bb2c2397001df3c1f9d3ac0e8a732c9dd444252a24dbdd", "start_s": 70.69231779193505, "source_duration_s": 73.85, "available_frames": 44 }, "grader_status": "failure" }, { "arm": "Completed-order stop edit", "elapsed_s": 19.83579604199622, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "65a1fffe7d9a22163195d72cc9badbacc70adc6fea9b93c0d1db398a66491975", "receipt": "191-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "6d96505a8709636d495217bf4faea1f0a9324809b7892cd05e5a601e0d947d82", "start_s": 17.6, "source_duration_s": 19.8, "available_frames": 44 }, "grader_status": "error" } ], "additional_takes": [ { "start_s": 110.45, "duration_s": 2.95, "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 11.728927165968344, "passed": false, "device": "RTX 5090", "receipt_sha256": "a50856d15bb6364982e3dee40947266669790d0829db39fb9c3e0d07533d3603", "receipt": "191-original.json", "recording": { "kind": "native browser recording", "source_sha256": "cb08030636da33fcc3620fc94101b555aa61fc8f832e2382cacf13b30e318041", "start_s": 8.779709249921144, "source_duration_s": 12.75, "available_frames": 59 }, "grader_status": "failure" }, { "arm": "Completed-order stop edit", "elapsed_s": 3.0059442919446155, "passed": false, "device": "RTX 5090", "receipt_sha256": "742867ddf6f8b75053dfe21966d372805bb4310d938dfd5828f9d8f41216d479", "receipt": "191-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "cbd08257ce72a112f677b7486daefa30e143b6a8c91b166a5e0f127f6ec55549", "start_s": 0.06729824999347311, "source_duration_s": 3.85, "available_frames": 59 }, "grader_status": "error" } ] }, { "start_s": 113.4, "duration_s": 2.95, "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 8.418479042011313, "passed": false, "device": "RTX 5090", "receipt_sha256": "7e8cbc36e5095c7df6a6b54ff0165fb76a83d8909bc1074d4beb222f537788c9", "receipt": "191-original.json", "recording": { "kind": "native browser recording", "source_sha256": "b5e52b0d037f58be3f3f5d3e33fa4c61577a57ccf71cb8486209c076e4e95a69", "start_s": 5.469136499986052, "source_duration_s": 9.4, "available_frames": 59 }, "grader_status": "failure" }, { "arm": "Order-history routing trial", "elapsed_s": 4.19265154202003, "passed": false, "device": "RTX 5090", "receipt_sha256": "f515e9de11c72ea33e3bbd84e2f2acc402998968fee8eaf8d35462f8924a6ad7", "receipt": "191-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "e947f45bcea752e6b8ac792fe0b8e9b1a188311839fcd822088f4524f5a4810f", "start_s": 1.2433369999751447, "source_duration_s": 5.45, "available_frames": 59 }, "grader_status": "error" } ] } ] }, { "number": 17, "start_s": 116.35, "duration_s": 5.15, "environment": "WebArena", "task": "231", "title": "Find a canceled order number", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 6.6563728749752045, "passed": false, "device": "RTX 5090", "receipt_sha256": "dba0db06cdd24a364d3d7ed703e8aa221607112e484ed065bfeae794a7720775", "receipt": "231-original.json", "recording": { "kind": "native browser recording", "source_sha256": "7b3fa7670690b658d28ceb15bcd79caa6f9b0807a87040418099f3c18070528d", "start_s": 5.45, "source_duration_s": 7.65, "available_frames": 44 } }, { "arm": "Selective edit", "elapsed_s": 4.067517541931011, "passed": false, "device": "RTX 5090", "receipt_sha256": "6ff7b40f9f37b75708af6b516d99bc3310d4c17014d14f3978026b3e3cb2d298", "receipt": "231-selective.json", "recording": { "kind": "native browser recording", "source_sha256": "ff31faeef21323e7c9b0a1ef1256e39febbc70ce304011fca60c4e7f3693e3ba", "start_s": 2.3, "source_duration_s": 4.5, "available_frames": 44 } } ], "additional_takes": [ { "start_s": 118.55, "duration_s": 2.95, "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 31.377146000042558, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "6ab7b6b4825e98503f76ac54ea4da6d6f19389788486ba6778faef2bcb8fd294", "receipt": "231-original.json", "recording": { "kind": "native browser recording", "source_sha256": "3aea1b0be0d4e8bd1125a74b743474e3f4723109d8f6ecb2b6767bc86ce17ee2", "start_s": 28.42790495797526, "source_duration_s": 32.35, "available_frames": 59 }, "grader_status": "failure" }, { "arm": "Completed-order stop edit", "elapsed_s": 5.121967457933351, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "c88c14e8d706e1ec5007960ac30dd3c5214714a865525ccea79b6f5463d05cbc", "receipt": "231-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "fd230a0bf607e34cda779e74694430fb50e01c81b4fea13b87cecdac971afb6b", "start_s": 2.172678832965903, "source_duration_s": 5.3, "available_frames": 59 }, "grader_status": "success" } ] } ] }, { "number": 18, "start_s": 121.5, "duration_s": 2.2, "environment": "WebArena", "task": "260", "title": "Browse video games", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 1.7300323750823736, "passed": false, "device": "RTX 5090", "receipt_sha256": "6572f6525b5cd53e9209f7b4ad47d496ef0127bfa07be3bc4d6d45001ae4ff4b", "receipt": "260-original.json", "recording": { "kind": "native browser recording", "source_sha256": "f843acca5acb2c45a0c0fa7c270f9c158fc40a9af337df8c37595fccba26a992", "start_s": 0.5, "source_duration_s": 2.7, "available_frames": 44 } }, { "arm": "Selective edit", "elapsed_s": 2.5460750829661265, "passed": false, "device": "RTX 5090", "receipt_sha256": "2448812891647952494e84f31962d367ab1e8933593250ab74a665e94e86392a", "receipt": "260-selective.json", "recording": { "kind": "native browser recording", "source_sha256": "d656fafbd06df21ba0aa012ede65d77d0d8a7270ba3ef4e9f17638f8576202ab", "start_s": 1.3499999999999996, "source_duration_s": 3.55, "available_frames": 44 } } ] }, { "number": 19, "start_s": 123.7, "duration_s": 2.2, "environment": "WebArena", "task": "269", "title": "Filter shoes under $25", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 31.290332584059797, "passed": false, "device": "RTX 5090", "receipt_sha256": "f203578690aa898f2da05005183436661df266655bef4e9472afe6ad17e19c80", "receipt": "269-original.json", "recording": { "kind": "native browser recording", "source_sha256": "ca9f0cb2585200e98edefc752db33ef399456af00fab2d676711003ce1446176", "start_s": 29.091356208967046, "source_duration_s": 32.2, "available_frames": 44 } }, { "arm": "Click / stop edit", "elapsed_s": 27.34156583400909, "passed": false, "device": "RTX 5090", "receipt_sha256": "264558df5dc080389b3efd246022b9c54d8e09b040578c63845e6c06f46da6d2", "receipt": "269-candidate.json", "recording": { "kind": "native browser recording", "source_sha256": "a4da8be857d5ac2a6e75856e63069d36dc9d5db4022e54437951be198e17965c", "start_s": 25.14343662497122, "source_duration_s": 28.3, "available_frames": 44 } } ] }, { "number": 20, "start_s": 125.9, "duration_s": 2.2, "environment": "WebArena", "task": "270", "title": "Filter men’s shoes below $30", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 26.58710758399684, "passed": false, "device": "RTX 5090", "receipt_sha256": "5abd4dea89adb3781d0f9aeb538b334ab7e99b4c56830f20b883cc50546ac042", "receipt": "270-original.json", "recording": { "kind": "native browser recording", "source_sha256": "83196e62b497e1cc7418b0ef1e0389b36d1a66194397b96be5c61a1f19446c4c", "start_s": 24.38832854195498, "source_duration_s": 27.55, "available_frames": 44 } }, { "arm": "Four-row edit", "elapsed_s": 7.783542124903761, "passed": false, "device": "RTX 5090", "receipt_sha256": "14886741a83d0cbdbf099501a961f49516b00773a124f83968d52027eed48b54", "receipt": "270-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "73288faace2fb00a3fcb19608f983bdebdaf5516b679cb546dba76bd715816fe", "start_s": 5.5843622498912735, "source_duration_s": 8.6, "available_frames": 44 } } ] }, { "number": 21, "start_s": 128.1, "duration_s": 7.7, "environment": "WebArena", "task": "274", "title": "Search for a USB Wi-Fi adapter", "shared_elapsed_start_s": 0.6736982500180602, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 14.633767332998104, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "a0030ee6bdaf34797101c31d7d61a9558428d07858421eca72c80c65bcd5b5ad", "receipt": "274-original.json", "recording": { "kind": "native browser recording", "source_sha256": "df3bab2792f9ec4726fd9a419418f76bfcb25aeea3d2c690b523bc34e26a9ea5", "start_s": 0.6743999580387026, "source_duration_s": 14.75, "available_frames": 95 }, "grader_status": "success" }, { "arm": "Published stop-row weights", "elapsed_s": 4.72369825001806, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "69d948230e4f42f059e25fc52fa817a440d771f99f2e8709cec53d1b289244f7", "receipt": "274-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "bc53c19aaaf7f2477f261e63856cf1cc31ce09d01560717ffa08080d39d0e005", "start_s": 0.6744516670471057, "source_duration_s": 5.3, "available_frames": 82 }, "grader_status": "success" } ], "additional_takes": [ { "start_s": 132.85, "duration_s": 2.95, "shared_elapsed_start_s": 3.1353435839992017, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 14.349906249903142, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "8e75836126e49acae38b153d83368055a0d49906c9f839a229005e60e8338762", "receipt": "274-original.json", "recording": { "kind": "native browser recording", "source_sha256": "af12cc9808d1e46d8a9c2c52e2e5a547b9e1345a484eec137aa3090d7aa8f212", "start_s": 3.1360501260496676, "source_duration_s": 14.6, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Completed-order stop edit", "elapsed_s": 5.385343583999202, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "1bbcfec71ea23fc04d20d9a67f569cf0acfa7a95e356897b655dc885a09a780e", "receipt": "274-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "e897298b69c97db8c118e4c2317b0c5f90298da7dc204658cf5e7d67cf008e52", "start_s": 3.1360644170781597, "source_duration_s": 6.05, "available_frames": 46 }, "grader_status": "success" } ] } ] }, { "number": 22, "start_s": 135.8, "duration_s": 23.75, "environment": "WebArena", "task": "275", "title": "Search for Xbox", "shared_elapsed_start_s": 0, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 20.42676358402241, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "09459a34d723b4cc3e9a7246bb6f759ffc0d7ae82d18f9977455c66299968f0a", "receipt": "275-original.json", "recording": { "kind": "native browser recording", "source_sha256": "e7a404ab8b53fe2ea72cf7c1e558149909bd42afbf57e486f01c6e342c409840", "start_s": 0.0013574579497799277, "source_duration_s": 20.65, "available_frames": 121 }, "grader_status": "success" }, { "arm": "Click-row edit", "elapsed_s": 4.95540237496607, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "16a454dcfa6db09b222d7891d85c7f559c938b9a2382e9de4e2101ddd505c718", "receipt": "275-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "7084446b535c1c10f48e0c70ec9b825b62d19fd07290cd8865872aaa31b2b212", "start_s": 0.0006438749842345715, "source_duration_s": 5.65, "available_frames": 101 }, "grader_status": "success" } ], "additional_takes": [ { "start_s": 141.85, "duration_s": 2.95, "shared_elapsed_start_s": 2.7589218330103904, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 5.00892183301039, "passed": true, "device": "RTX 5090", "receipt_sha256": "e2ddcc46e4c061c79c9d2669ecbff0f6ccd069a3fa895273916cdefa9a5dd614", "receipt": "275-original.json", "recording": { "kind": "native browser recording", "source_sha256": "90909532e5e7a03ef11e431f670b650c64adc4b87ed6173b91b4474a451e325d", "start_s": 2.7596860420890152, "source_duration_s": 5.3, "available_frames": 46 }, "grader_status": "success" }, { "arm": "Completed-order stop edit", "elapsed_s": 6.081362874945626, "passed": true, "device": "RTX 5090", "receipt_sha256": "3de8aefd1e2fcce1fec46d94a4e3498444550b0c77062e94d219e6db5f1902d4", "receipt": "275-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "d126217723fea7e12c378ec0e931a570d40178157550c328b307bbb8b58a935e", "start_s": 2.7594927080208436, "source_duration_s": 7.1, "available_frames": 59 }, "grader_status": "success" } ] }, { "start_s": 144.8, "duration_s": 2.95, "shared_elapsed_start_s": 2.770188458962366, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 5.020188458962366, "passed": true, "device": "RTX 5090", "receipt_sha256": "61fb1b50bc29906ad350fd24a1b1e15ebed4a36b679e61d1f06a5f89ed9c963b", "receipt": "275-original.json", "recording": { "kind": "native browser recording", "source_sha256": "14429436b695064bb95a4db5ad46cd2f1c92e6a2283c2aa9175d0cdb19be67fd", "start_s": 2.771443958976306, "source_duration_s": 5.05, "available_frames": 45 }, "grader_status": "success" }, { "arm": "Order-history routing trial", "elapsed_s": 6.352745249983855, "passed": true, "device": "RTX 5090", "receipt_sha256": "8ef5a9dda6679658b3549adf62a04655d89ca128503e539deab907b65d214294", "receipt": "275-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "524f90fa5fa0edb6001e43e3e61ed5b73c053ba1139d7b494a0ff7eb185c340e", "start_s": 2.7714176670415327, "source_duration_s": 7.0, "available_frames": 59 }, "grader_status": "success" } ] }, { "start_s": 147.75, "duration_s": 2.95, "shared_elapsed_start_s": 0.7524331250460816, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 4.751382500049658, "passed": true, "device": "RTX 5090", "receipt_sha256": "1ed81a2acf8fc1c1f74a90b325579f1692c73339bcc2581c8f8484c2de7c0887", "receipt": "275-original.json", "recording": { "kind": "native browser recording", "source_sha256": "3d8f81550117b42ee36ada4ccd70dff5808fd4c6c9bf9685eb305daa60e4ffa7", "start_s": 0.7534510829718781, "source_duration_s": 4.95, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Search submit stop", "elapsed_s": 3.002433125046082, "passed": true, "device": "RTX 5090", "receipt_sha256": "05fbd793a01633e453f840b53c32ce89d0b97515fcd68aced3b9bdf24de01554", "receipt": "275-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "993a915623f7851110f0178c347bd5ae29ad90318f9e9c1c9ace7c0bf53aff2a", "start_s": 0.7531419170554725, "source_duration_s": 4.1, "available_frames": 46 }, "grader_status": "success" } ] }, { "start_s": 150.7, "duration_s": 2.95, "shared_elapsed_start_s": 2.7378833330003545, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 18.785939250024967, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "09ef4ded08f0db1e74a46302902dfb168fe246285d9b04cfd369299b0295b51a", "receipt": "275-original.json", "recording": { "kind": "native browser recording", "source_sha256": "dd891d17e6c96baf3fa4ce46698790adacd99ed449f0fbef594d347192a8703e", "start_s": 2.7385854159947485, "source_duration_s": 18.95, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Search submit stop", "elapsed_s": 4.9878833330003545, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "833b8f3c747ffc8120093122d485f7fc663f123a1da4f1b0f46c6c90ed0411ff", "receipt": "275-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "1a651a7722aff065265042313363f0d86e9dd5b1078bd0141355601cca71a3e4", "start_s": 2.738696665968746, "source_duration_s": 5.8, "available_frames": 46 }, "grader_status": "success" } ] }, { "start_s": 153.65, "duration_s": 2.95, "shared_elapsed_start_s": 2.8694580829469487, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 20.91780145803932, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "bace17fc01e37ed306f4e64981bf49c67bb9a6b95100b1895f955c2e54fd1c54", "receipt": "275-original.json", "recording": { "kind": "native browser recording", "source_sha256": "00c066740a18093c3dab6ab4a99c066338f6c2e60a0d3675214d560b8a216371", "start_s": 2.870172541937791, "source_duration_s": 21.05, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Completed-order stop edit", "elapsed_s": 5.119458082946949, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "ecec834e9b1f89a8d4a50df566ab0db1caea8bdaa500e96d765805ac755e99f0", "receipt": "275-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "c3836f0939f129721c89b1c4493bc3339533ffd48694876bc341252d06cbca13", "start_s": 2.8701312079792842, "source_duration_s": 5.6, "available_frames": 46 }, "grader_status": "success" } ] }, { "start_s": 156.6, "duration_s": 2.95, "shared_elapsed_start_s": 3.021435666945763, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 19.23649004206527, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "b1661afd4c77b723a5770615088a5b6825721c8fafc3dd409cfa136405873f5d", "receipt": "275-original.json", "recording": { "kind": "native browser recording", "source_sha256": "cff138a1b744586afb236948d3794e48f440edf63ae0526523e7476211b05019", "start_s": 3.022113249870017, "source_duration_s": 19.25, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Product-link selection edit", "elapsed_s": 5.271435666945763, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "74d1c41ba0b33fc4c739af5b06c9b26266834783fa088edbdd95563dee27d5aa", "receipt": "275-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "846cac34d51a1b98428873e4aa4387ccc195acb6ee1dac258398ee84b0454e21", "start_s": 3.022121167043224, "source_duration_s": 5.8, "available_frames": 46 }, "grader_status": "success" } ] } ] }, { "number": 23, "start_s": 159.55, "duration_s": 10.75, "environment": "WebArena", "task": "276", "title": "Search for Switch accessories", "shared_elapsed_start_s": 1.2913488750811666, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 12.93084349995479, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "deb7733e8c1cdd4c0f5ac722d54636ee2672f96a1149b3d693436b42fb25ba9c", "receipt": "276-original.json", "recording": { "kind": "native browser recording", "source_sha256": "00d41ba656ae96772e431aed9aa84fdc9ddd8e57a3cedec0a3347eb37e73b1e9", "start_s": 1.2919892081292346, "source_duration_s": 13.1, "available_frames": 95 }, "grader_status": "success" }, { "arm": "Published stop-row weights", "elapsed_s": 5.341348875081167, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "4fdcd3249c01cd5b857fc3bb1ab5c20cb53eb0873d775ec8bad3410e2d2ff02f", "receipt": "276-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "b6aaf024685e66ba3d9d34a3b80f4ea03006abda6c5b3c715c7d3dc49dca19ce", "start_s": 1.291956916032359, "source_duration_s": 5.9, "available_frames": 82 }, "grader_status": "success" } ], "additional_takes": [ { "start_s": 164.3, "duration_s": 3.05, "shared_elapsed_start_s": 1.3485021249856801, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 5.319939290988259, "passed": true, "device": "RTX 5090", "receipt_sha256": "73d8c217bdc2e7fedf38a37fc5d3088aa7e3100c8088a05b5e1fb4210cef57a9", "receipt": "276-original.json", "recording": { "kind": "native browser recording", "source_sha256": "3981b646f3ef012a477b5324d53c26779b4d267bd4328d4668097d98fd59b05a", "start_s": 1.349146250030026, "source_duration_s": 5.55, "available_frames": 61 } }, { "arm": "Query-transfer edit", "elapsed_s": 3.69850212498568, "passed": true, "device": "RTX 5090", "receipt_sha256": "0ff0fbe4a87cb9cf4ab50f3b769f871255fbee0d5de6ad688e0b6de970e3ec9f", "receipt": "276-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "eb3e5fcb71a4735f73bf19c46aea6f60f27fd7f6df72236f44b18d133cb47950", "start_s": 1.3491312080062927, "source_duration_s": 4.7, "available_frames": 48 } } ] }, { "start_s": 167.35, "duration_s": 2.95, "shared_elapsed_start_s": 3.2479067909298465, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 14.731332250055857, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "71905bf063f752c654a13cbfa2f216e892d4c18809658f74f083f4f4e39d1598", "receipt": "276-original.json", "recording": { "kind": "native browser recording", "source_sha256": "18ca2607d91d54a0a6b66b8f3c13e657a3f1eaa1078ec0d2f15e48e8f2a415f1", "start_s": 3.2486487078713253, "source_duration_s": 14.95, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Completed-order stop edit", "elapsed_s": 5.4979067909298465, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "1382dbbfddbf5afaa24370bcf99eb3c6605aa18db4b9ba05d74e5a9259b1d686", "receipt": "276-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "920d1fdd4076374344a35630e6bd209a8a639523021e90f779d93b62ba109726", "start_s": 3.2486720830202103, "source_duration_s": 6.0, "available_frames": 46 }, "grader_status": "success" } ] } ] }, { "number": 24, "start_s": 170.3, "duration_s": 7.7, "environment": "WebArena", "task": "277", "title": "Search for iPhone 13 batteries", "shared_elapsed_start_s": 1.7715621249983087, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 16.96133550000377, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "988018672f26ffb16fe637c9f6473b257ba11914d25bd7359401539c81762a89", "receipt": "277-original.json", "recording": { "kind": "native browser recording", "source_sha256": "1ae3bb1db3e4d47e53d8b06684843b2aef8ce876e6becf70b0d9184665605b13", "start_s": 1.7721921670483425, "source_duration_s": 17.15, "available_frames": 95 }, "grader_status": "success" }, { "arm": "Published stop-row weights", "elapsed_s": 5.821562124998309, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "96b8493af4c46f44008da5e2735f4ec75ade194abba6b5dddf9d6f95091f6e3a", "receipt": "277-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "1f34cfc44b67f4ae188868d7fa95b2a92af7abe7f2519e71cf57ff4c9c160417", "start_s": 1.7724100000225007, "source_duration_s": 6.35, "available_frames": 82 }, "grader_status": "success" } ], "additional_takes": [ { "start_s": 175.05, "duration_s": 2.95, "shared_elapsed_start_s": 3.629690458998084, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 19.38110354100354, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "cedcfe5b1a31810a6b057846f778a6502ee0c60f84ceb6dc279763555b30696b", "receipt": "277-original.json", "recording": { "kind": "native browser recording", "source_sha256": "9c0d7164e632a65f82c2ab8b3ce320b5521b47404d104ca09247aa8050537c6c", "start_s": 3.630406250944361, "source_duration_s": 19.6, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Completed-order stop edit", "elapsed_s": 5.879690458998084, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "c20a66a68f4fade474ce0021165497d211331c8281dc3136b93555e663b9c150", "receipt": "277-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "372124866542f39ea4e2625481c057e19b69efe4e4ba55b88451f58e191a5c76", "start_s": 3.630373541964218, "source_duration_s": 6.35, "available_frames": 46 }, "grader_status": "success" } ] } ] }, { "number": 25, "start_s": 178.0, "duration_s": 13.3, "environment": "WebArena", "task": "278", "title": "Search for green tea bags", "shared_elapsed_start_s": 3.5640258749946954, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 18.839734166976996, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "aa960fdc043ebfbfccb8de8d78549b30d4e197fff970e23bf023f616ce57cc82", "receipt": "278-original.json", "recording": { "kind": "native browser recording", "source_sha256": "b4597dc162e0d24dd8a47b9236efd3498e95a803905150d3626b8df2d4e612d6", "start_s": 3.5725090419175105, "source_duration_s": 19.05, "available_frames": 95 }, "grader_status": "success" }, { "arm": "Click-row edit", "elapsed_s": 7.614025874994695, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "07a00d4de61caccda2b817330c07556c62f979844be1e1835bcf85e238f2c8c2", "receipt": "278-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "97ccca271791b8e5eb58a6aaba1e836be5f34b51055a8d07fa394d7c9ca94016", "start_s": 3.568142666993663, "source_duration_s": 8.1, "available_frames": 82 }, "grader_status": "success" } ], "additional_takes": [ { "start_s": 182.75, "duration_s": 3.05, "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 6.485668874927796, "passed": true, "device": "RTX 5090", "receipt_sha256": "c68b607a3e30001624940c47a6fbce8667aef0690c10449ca119bb02aa898ae3", "receipt": "278-original.json", "recording": { "kind": "native browser recording", "source_sha256": "f423ea9fbe914caf95fdc105bfa61b5e149ea5fa133a62e4f2e237ea379dbd16", "start_s": 3.436384665942751, "source_duration_s": 6.8, "available_frames": 61 }, "grader_status": "success" }, { "arm": "Click-row edit", "elapsed_s": 7.744358584051952, "passed": false, "device": "RTX 5090", "receipt_sha256": "e773087b73962946c4bacc9d899d930c7f5a651c6cb5f0a04f197c8457f9791b", "receipt": "278-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "eb1cf4fc23270a3feec6d23e585c4a73145a9f8523e5cefdbb502aca786ef1ff", "start_s": 4.695583625044674, "source_duration_s": 8.9, "available_frames": 61 }, "grader_status": "failure" } ] }, { "start_s": 185.8, "duration_s": 2.55, "shared_elapsed_start_s": 3.232748709013686, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 6.580361375003122, "passed": true, "device": "RTX 5090", "receipt_sha256": "3537a8a13d22f1cfb7f2a3f8aa66f1c34faba2d79826bb57683348c828b1379f", "receipt": "278-original.json", "recording": { "kind": "native browser recording", "source_sha256": "5e45633c2050d9e466cdcb577e9634c375e5a9101adf4396227b4a25b2cdadbc", "start_s": 3.2335233340039853, "source_duration_s": 6.9, "available_frames": 51 }, "grader_status": "success" }, { "arm": "Native click-row edit", "elapsed_s": 5.082748709013686, "passed": true, "device": "RTX 5090", "receipt_sha256": "b701018c0ada5e0cc94f79c0c1597e8dc0c6a266c42a883090d204dee8cf0255", "receipt": "278-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "31cb1968b84848c6b8b2e56aa449d19f4025e658cc7c10bc840a9f95a9d89fde", "start_s": 3.2357539169956, "source_duration_s": 6.25, "available_frames": 38 }, "grader_status": "success" } ] }, { "start_s": 188.35, "duration_s": 2.95, "shared_elapsed_start_s": 3.86688183399383, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 24.29026104195509, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "f54ed2ed164381f7fce725a451b1243c220bd58f2cf76e6864a5b4ce14c7363d", "receipt": "278-original.json", "recording": { "kind": "native browser recording", "source_sha256": "051a34b4ceeff354add69fd699507064e20260cddc69d406284ec3e69a89005f", "start_s": 3.8675468750298023, "source_duration_s": 24.55, "available_frames": 59 }, "grader_status": "success" }, { "arm": "Completed-order stop edit", "elapsed_s": 6.11688183399383, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "5ad1b4a1fa25da2ebe1e5ed4f13976b9d52457987b254a55e09912f3d1ded67a", "receipt": "278-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "1fc55b1c968704e5e58ae61587eaedc473f3bf7e4e69926c42efa56eca019525", "start_s": 3.867606583982706, "source_duration_s": 6.6, "available_frames": 46 }, "grader_status": "success" } ] } ] }, { "number": 26, "start_s": 191.3, "duration_s": 2.2, "environment": "WebArena", "task": "279", "title": "Find Sony headphone models and prices", "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 58.91331287508365, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "5c91309b26f2443c46ae131c96253075b5373b4c4d587024433fc58dba500840", "receipt": "279-original.json", "recording": { "kind": "native browser recording", "source_sha256": "cfa6a48076e0eba32d9684104b654c6dfb6957941f3418a3f6ab616504e89b43", "start_s": 56.714017208036964, "source_duration_s": 59.9, "available_frames": 44 }, "grader_status": "failure" }, { "arm": "Click-row edit", "elapsed_s": 8.496458709007129, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "079b85c3cf1856e430fc6af64fc3384f277f3ceb0f58545c2d6d77453dbb0c38", "receipt": "279-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "a9630aa6046b952cd1574d022e4fe1fe3f8967bd2719753e8f71cfdd93556c82", "start_s": 6.297402042034082, "source_duration_s": 8.7, "available_frames": 44 }, "grader_status": "failure" } ] }, { "number": 27, "start_s": 193.5, "duration_s": 14.0, "environment": "WebArena", "task": "283", "title": "Find Xbox controllers released in 2020–2021", "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 81.33533662499394, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "c0668e2011b93eb9c2e80746d17db3de4c0a07df6aee1b2dc76e65a918dbe99d", "receipt": "283-original.json", "recording": { "kind": "native browser recording", "source_sha256": "fe0be6adb38961edcb5c4e5e48798336e93ebd4d962ea6db10fb51ecac152e67", "start_s": 79.13605354093015, "source_duration_s": 83.0, "available_frames": 44 }, "grader_status": "failure" }, { "arm": "Completed-order stop edit", "elapsed_s": 12.874797667027451, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "522a89222aa4484cb9a5801275aa38ea28c6451b336fe6327eb8ec1714590ff2", "receipt": "283-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "a2c70c7239ec95f2be1d56b722cf1d51d7fddca0c772d48f9e9dd0146b9c01e9", "start_s": 10.675659332959913, "source_duration_s": 13.85, "available_frames": 44 }, "grader_status": "failure" } ], "additional_takes": [ { "start_s": 195.7, "duration_s": 2.95, "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 87.93015045800712, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "2c450dd02c237f79c8c5162b8916d7a597dd137a771a903101909ff3ff4dba83", "receipt": "283-original.json", "recording": { "kind": "native browser recording", "source_sha256": "e5e09a023862edf3033be65c4a8c22306acd87f4881cf6076507ae184f138943", "start_s": 84.98082500004675, "source_duration_s": 89.6, "available_frames": 59 }, "grader_status": "failure" }, { "arm": "Focused search-field edit", "elapsed_s": 18.76859758410137, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "5ddc2381679f9010a0cbe1497c08631418a24110efeba721a68207316efb1100", "receipt": "283-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "75373c42203df67877ff8e69cb6111aa5b86498715a0a2d75a1c035b6e242d12", "start_s": 15.819353209063411, "source_duration_s": 19.75, "available_frames": 59 }, "grader_status": "failure" } ] }, { "start_s": 198.65, "duration_s": 2.95, "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 83.21020058402792, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "ee4856c05d0ff4d0770283c305adda801af5687b4b832386d59b49b5b08d1d47", "receipt": "283-original.json", "recording": { "kind": "native browser recording", "source_sha256": "7e5531f2ff6b397792d722269290804638dd8557eca6c3ff55419a5579ecf7cf", "start_s": 80.26086333401035, "source_duration_s": 84.85, "available_frames": 59 }, "grader_status": "failure" }, { "arm": "Search field-state edit", "elapsed_s": 35.93294025002979, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "f407596f9683e3096d782dce5c1efd674c319e14bb841f544b1f681f2b5369e7", "receipt": "283-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "3be4a09bcb12e77dfbdc95c936122164911447bd4d32d10485dd05cca6023587", "start_s": 32.98367970897816, "source_duration_s": 36.9, "available_frames": 59 }, "grader_status": "failure" } ] }, { "start_s": 201.6, "duration_s": 2.95, "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 87.58633945893962, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "9b65b62e25ac44058102f06c2d942d6126a77631c3ce56fbbd251db947e7ee9b", "receipt": "283-original.json", "recording": { "kind": "native browser recording", "source_sha256": "34cbd3e8dbd7a2e06aba936385ed06dc098f249606cf3dbc4abc27cebab3d2dc", "start_s": 84.63732720895204, "source_duration_s": 89.2, "available_frames": 59 }, "grader_status": "failure" }, { "arm": "Guarded product navigation", "elapsed_s": 7.486511458992027, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "ecb4338098a011c1e62da6714b102055d2cb79f940f14c814134c395fa0410ea", "receipt": "283-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "a9a065d9141c9c0330e4508a9b97e3394ffb043b9740630670f2ae6dea8b0ba2", "start_s": 4.537230958999134, "source_duration_s": 7.6, "available_frames": 59 }, "grader_status": "failure" } ] }, { "start_s": 204.55, "duration_s": 2.95, "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 83.67859258397948, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "caa357c2cfbe6aa3e5071ea4911e793ae704149abec00613e955397f1b3fd9f0", "receipt": "283-original.json", "recording": { "kind": "native browser recording", "source_sha256": "3c41d6f534fe265e807a15743f36af902a3ef846a34cb201b30f4e935276b024", "start_s": 80.72925879196264, "source_duration_s": 85.35, "available_frames": 59 }, "grader_status": "failure" }, { "arm": "Product-link selection edit", "elapsed_s": 9.97932625003159, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "815093b92848d5256a73998a7ea848405874b2fe8342deb4191a9d7ca1cf367b", "receipt": "283-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "3dc5df389d5f36faecc16700018661c6aa29f198d75d5951ae0e1d1712aa5f31", "start_s": 7.030045042000711, "source_duration_s": 10.3, "available_frames": 59 }, "grader_status": "success" } ] } ] }, { "number": 28, "start_s": 207.5, "duration_s": 2.2, "environment": "WebArena", "task": "313", "title": "Find customer service", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 15.730594541062601, "passed": false, "device": "RTX 5090", "receipt_sha256": "a2242a6d96326dcc4520e061a9f588be4bc43c8ca398ba867ca608e6d0dca11f", "receipt": "313-original.json", "recording": { "kind": "native browser recording", "source_sha256": "17ae6a74b8bedfac5aff872ff339927313b46ab01d199651778c2cb866a14a2d", "start_s": 13.53175816598814, "source_duration_s": 16.65, "available_frames": 44 } }, { "arm": "Click / stop edit", "elapsed_s": 17.031620875000954, "passed": false, "device": "RTX 5090", "receipt_sha256": "7846098a3172221821062a368f9b743129b459e6d57cb7f52472dabb58199113", "receipt": "313-candidate.json", "recording": { "kind": "native browser recording", "source_sha256": "7bd25009f7c92e838fd64ef0a10b474dec0958de1a28d33bdc68020fac472b4c", "start_s": 14.832806040951983, "source_duration_s": 17.95, "available_frames": 44 } } ] }, { "number": 29, "start_s": 209.7, "duration_s": 2.2, "environment": "WebArena", "task": "324", "title": "Sort chairs by price", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 30.0253297080053, "passed": false, "device": "RTX 5090", "receipt_sha256": "9c1c269ec32abe8e43ea1ed29980507159d696673b9319a590705c4b3284d076", "receipt": "324-original.json", "recording": { "kind": "native browser recording", "source_sha256": "8df44618553af86755eaea7d406159d7f1a8001f403a1fa4c88810e42a89bad1", "start_s": 28.8, "source_duration_s": 31.0, "available_frames": 44 } }, { "arm": "Selective edit", "elapsed_s": 11.177542124991305, "passed": false, "device": "RTX 5090", "receipt_sha256": "7c163e07d043bbe0bcfdcc3f349239eebdf7f5d922c3ccc8ec21fb37a8ec36f0", "receipt": "324-selective.json", "recording": { "kind": "native browser recording", "source_sha256": "3332a57a3f93fade94bf96f303a192bf42cb9909394e68571b0bc5b92e2c1d80", "start_s": 9.95, "source_duration_s": 12.15, "available_frames": 44 } } ] }, { "number": 30, "start_s": 211.9, "duration_s": 2.2, "environment": "WebArena", "task": "358", "title": "Check how an order shipped", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 9.758979709004052, "passed": false, "device": "RTX 5090", "receipt_sha256": "0f3ba69b2c7781c67776b658e1c0f8b0391943dadb8cad018db2a6c14c3244e5", "receipt": "358-original.json", "recording": { "kind": "native browser recording", "source_sha256": "f9d8b8b9a5131fc4c85ec7664d7ede173ebb8e9fd6177e509eb48666ebd3a16e", "start_s": 8.55, "source_duration_s": 10.75, "available_frames": 44 } }, { "arm": "Selective edit", "elapsed_s": 18.59243716695346, "passed": false, "device": "RTX 5090", "receipt_sha256": "c84986785157b659a33094a501aede077ad601848b3d85021345476c6c7f21df", "receipt": "358-selective.json", "recording": { "kind": "native browser recording", "source_sha256": "c14e04dc376b26d51c8b837cefac39b518050f0a9aee522d06da15810b5f2d8e", "start_s": 17.05, "source_duration_s": 19.25, "available_frames": 44 } } ] }, { "number": 31, "start_s": 214.1, "duration_s": 2.2, "environment": "WebArena", "task": "399", "title": "Edit a forum profile", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 5.167680417071097, "passed": false, "device": "RTX 5090", "receipt_sha256": "0d8533467c8c95298d7682544aceede284ecba7b2aacc3c187f95099af55a55f", "receipt": "399-original.json", "recording": { "kind": "native browser recording", "source_sha256": "c421512ee0caefe06383ce23cd66006a0278751b9fc012a80e5a4b2602c577c8", "start_s": 2.9688355420483274, "source_duration_s": 6.15, "available_frames": 44 } }, { "arm": "Click / stop edit", "elapsed_s": 5.198532000067644, "passed": false, "device": "RTX 5090", "receipt_sha256": "c2482360d48af58ca2f1237fe8a07816ad10b35a85b597b768f6c2088c608748", "receipt": "399-candidate.json", "recording": { "kind": "native browser recording", "source_sha256": "1a92c89e4977d6ddbdf56eee56a09de4a34e5aa0acedaf607ed7e1e6a76de618", "start_s": 2.999260583030991, "source_duration_s": 6.2, "available_frames": 44 } } ] }, { "number": 32, "start_s": 216.3, "duration_s": 2.2, "environment": "WebArena", "task": "404", "title": "Upvote the newest book post", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 29.582460999954492, "passed": false, "device": "RTX 5090", "receipt_sha256": "6e72f32dfb9c5f481dd939462073c9c1a82dbaf09a50c09d82ceb92765e6b6ed", "receipt": "404-original.json", "recording": { "kind": "native browser recording", "source_sha256": "c43a5ef62a55c1c8fc868f9754533a8149ef7544a65bfddbb4984a98191fb3de", "start_s": 27.38365874993615, "source_duration_s": 30.55, "available_frames": 44 } }, { "arm": "Click / stop edit", "elapsed_s": 27.15264012501575, "passed": false, "device": "RTX 5090", "receipt_sha256": "8dd38d2e92c4e17b6eaa092d8590bd641dbf9faed0f8a16a770f7475afe8d6de", "receipt": "404-candidate.json", "recording": { "kind": "native browser recording", "source_sha256": "448b5eba3e4b9d4ca33eea80aff9090f4778514ed46dd3507bc6a544650aa0fe", "start_s": 24.953498625033536, "source_duration_s": 28.15, "available_frames": 44 } } ] }, { "number": 33, "start_s": 218.5, "duration_s": 2.2, "environment": "WebArena", "task": "431", "title": "Add the lowest-unit-price item to the cart", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 15.824151332955807, "passed": false, "device": "RTX 5090", "receipt_sha256": "b9771c2bd3ce8b881ea44786e2b3b8595c3aeda4278b7f39c7cf4e4ebc06d8f1", "receipt": "431-original.json", "recording": { "kind": "native active-tab browser recordings", "source_sha256": "ac933af0e705b13aa7e615b9a036e3e31ab55c84e6d1a4419ea4f2a3d05184d6", "start_s": 13.624151332955808, "all_tabs": [ { "kind": "native browser recording", "source_sha256": "ac933af0e705b13aa7e615b9a036e3e31ab55c84e6d1a4419ea4f2a3d05184d6", "start_s": 13.63598233293742, "source_duration_s": 16.8, "available_frames": 44 }, { "kind": "native browser recording", "source_sha256": "e1dd39080ca5ee956574164559ddb60546e4fcd9e03d7a98bbc084c7eb717998", "start_s": 13.631874207942747, "source_duration_s": 17.55, "available_frames": 44 }, { "kind": "native browser recording", "source_sha256": "9f46a0294c151dfcb49714524f9fc6b194e08c363cde39db43fe5aac30bed6a4", "start_s": 13.627635499997997, "source_duration_s": 16.05, "available_frames": 44 } ] } }, { "arm": "Four-row edit", "elapsed_s": 8.88824487500824, "passed": false, "device": "RTX 5090", "receipt_sha256": "aab35c7429eb82c8cd23aa64dac96f55a80a1fece1737f72012c9cb4ddedaf02", "receipt": "431-swap_click_stop.json", "recording": { "kind": "native active-tab browser recordings", "source_sha256": "6840a0a06d22b82cc663d1a57d4beb17b7a47a3a31d31020651e5f393f0bbcc7", "start_s": 6.68824487500824, "all_tabs": [ { "kind": "native browser recording", "source_sha256": "6840a0a06d22b82cc663d1a57d4beb17b7a47a3a31d31020651e5f393f0bbcc7", "start_s": 6.699231291026808, "source_duration_s": 9.9, "available_frames": 44 }, { "kind": "native browser recording", "source_sha256": "9a462e83d00da06a03a3fd33dd7ec8506b5dbc45083b8190cc7bcee3d2f84ac1", "start_s": 6.69557141601108, "source_duration_s": 10.05, "available_frames": 44 }, { "kind": "native browser recording", "source_sha256": "1e9869e86e78e9bbe47f3accd26010a85bcd07f20c24f57c42fbf9a0b4ce39d8", "start_s": 6.693636208027601, "source_duration_s": 9.1, "available_frames": 44 } ] } } ] }, { "number": 34, "start_s": 220.7, "duration_s": 3.55, "environment": "WebArena", "task": "432", "title": "Compare two drinks and add the cheaper one per unit", "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 34.84325033298228, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "a3b25743627fbfa772b9b4601a3b0a94a0e0750ac417424ce0e78d531e21d68e", "receipt": "432-original.json", "recording": { "kind": "native active-tab browser recordings", "source_sha256": "cdf08dda9f4162aceb0dfee3e06d679af589c71b5dd3d7d4e503a4bb99e20fc2", "start_s": 31.29325033298228, "all_tabs": [ { "kind": "native browser recording", "source_sha256": "cdf08dda9f4162aceb0dfee3e06d679af589c71b5dd3d7d4e503a4bb99e20fc2", "start_s": 31.295433457987382, "source_duration_s": 35.8, "available_frames": 71 }, { "kind": "native browser recording", "source_sha256": "e772943ab672588e4fdcf0b611b54534ea715fe0ef731a8cd13b51f4d4dbf8cf", "start_s": 31.294645499950274, "source_duration_s": 37.95, "available_frames": 71 } ] }, "grader_status": "failure" }, { "arm": "Stop-row edit", "elapsed_s": 11.119863915955648, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "3f5122698ea14fba6f4e02f2924d0b86811abca0b15e937fbf1f1ccdea1f4116", "receipt": "432-edited.json", "recording": { "kind": "native active-tab browser recordings", "source_sha256": "d243903dbad81ad0b5faf960807157607ee4d31858b96b9a4749b4c5c3b1df2e", "start_s": 7.569863915955648, "all_tabs": [ { "kind": "native browser recording", "source_sha256": "d243903dbad81ad0b5faf960807157607ee4d31858b96b9a4749b4c5c3b1df2e", "start_s": 7.573782415990718, "source_duration_s": 12.1, "available_frames": 71 }, { "kind": "native browser recording", "source_sha256": "75b0e72010224450821ee486cf22ea4635386dab7f70f2977d9c3ee5b18a28cf", "start_s": 7.572015582979657, "source_duration_s": 12.75, "available_frames": 71 } ] }, "grader_status": "failure" } ] }, { "number": 35, "start_s": 224.25, "duration_s": 2.2, "environment": "WebArena", "task": "516", "title": "Save a product to a wishlist", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 24.595935750054196, "passed": false, "device": "RTX 5090", "receipt_sha256": "c071fee49c1e6d2ad97dd61f8d03bc48861454eb290b4a2da944dde98c23c69f", "receipt": "516-original.json", "recording": { "kind": "native browser recording", "source_sha256": "30aff6e294222ff9bc1a7589b0903d1afa41123ceb5f8f758ade44a34b405678", "start_s": 22.406587709044107, "source_duration_s": 25.55, "available_frames": 44 } }, { "arm": "Click / stop edit", "elapsed_s": 3.0620824169600382, "passed": true, "device": "RTX 5090", "receipt_sha256": "dbfaf59c5a2efdbf8991efb36ff34d5f53b285bda64883ed0000a0dd3c6adf0e", "receipt": "516-candidate.json", "recording": { "kind": "native browser recording", "source_sha256": "5069a4f745e6b6d6a36587b8120290dbb4cb3a3c9902ca5a7be5d6919b5e4e9e", "start_s": 0.8499999999999996, "source_duration_s": 3.05, "available_frames": 44 } } ] }, { "number": 36, "start_s": 226.45, "duration_s": 2.2, "environment": "WebArena", "task": "595", "title": "Follow the space forum", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 24.92897675000131, "passed": false, "device": "RTX 5090", "receipt_sha256": "711dea08e414b30ea1bfc6101ee6dd07e642d658f7e6dcd1ce3b2e3e0f3d451d", "receipt": "595-original.json", "recording": { "kind": "native browser recording", "source_sha256": "6e93f8dae0dc059414d525340b5f5a77ea4d6e17692115788f0c56c51c54d3cd", "start_s": 22.729442916996778, "source_duration_s": 25.85, "available_frames": 44 } }, { "arm": "Click / stop edit", "elapsed_s": 24.435653541935608, "passed": false, "device": "RTX 5090", "receipt_sha256": "b569365ae257d926110b84a4751e1e4b30f393fc4094d50630ef13d05dd00ab9", "receipt": "595-candidate.json", "recording": { "kind": "native browser recording", "source_sha256": "1e88ed1d1ab78121c61f8f56a8ea446d81f4d064d9301d63ec154a1148579b05", "start_s": 22.23621233291924, "source_duration_s": 25.4, "available_frames": 44 } } ] }, { "number": 37, "start_s": 228.65, "duration_s": 2.2, "environment": "WebArena", "task": "600", "title": "Ask which console to buy", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 15.429348958074115, "passed": false, "device": "RTX 5090", "receipt_sha256": "2428dc87eec9914bb25ebb00152acc605b667b7255345f421ffbc5593c294786", "receipt": "600-original.json", "recording": { "kind": "native browser recording", "source_sha256": "a1677419be2c9a47adf1c0c4d7bdad6f482137a66cd9da5ef1dc99dbe9f6bd60", "start_s": 14.150000000000002, "source_duration_s": 16.35, "available_frames": 44 } } ] }, { "number": 38, "start_s": 230.85, "duration_s": 7.3, "environment": "WebArena", "task": "630", "title": "Ask for headphone recommendations", "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 95.00853933300823, "passed": false, "device": "M5 Max MacBook Pro", "receipt_sha256": "4cb3199a3fe86dbea6a1fa82743d1038699e3a0360a6967e72a247d2a69e913a", "receipt": "630-original.json", "recording": { "kind": "native browser recording", "source_sha256": "5ad3614b2ce1d886f35c3e6f2aa7f406cc90a201690fa25a91255108587035e0", "start_s": 92.80955016701482, "source_duration_s": 95.95, "available_frames": 44 }, "grader_status": "failure" }, { "arm": "Cached-submit edit", "elapsed_s": 20.946383042028174, "passed": true, "device": "M5 Max MacBook Pro", "receipt_sha256": "cbc711eb057d563054baaad79c58b12558d097df178a2ad2086ba58c6d7ce4f4", "receipt": "630-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "57222361251c34526466020f73d93916616fc724aae7f4f0e95622c4c816889f", "start_s": 18.747268416988664, "source_duration_s": 20.95, "available_frames": 44 }, "grader_status": "success" } ], "additional_takes": [ { "start_s": 233.05, "duration_s": 2.55, "shared_elapsed_start_s": 18.707142707984893, "arms": [ { "arm": "Original", "elapsed_s": 24.51519945904147, "passed": true, "device": "RTX 5090", "receipt_sha256": "064d4dfe613e98720dc39b55360befa8c023da78c29af94ade63068cd1f37c61", "receipt": "630-original.json", "recording": { "kind": "native browser recording", "source_sha256": "39f0add873acca0cc39b1f0928ef80902590f07023e4dfdf829706f2a267f1c4", "start_s": 18.708387083001433, "source_duration_s": 24.5, "available_frames": 51 } }, { "arm": "One-row edit", "elapsed_s": 20.557142707984895, "passed": true, "device": "RTX 5090", "receipt_sha256": "d00823bc4c06b54af3b06ab08ac977bbf40005175c62396ac38e5ad4b442d2dc", "receipt": "630-close_strong.json", "recording": { "kind": "native browser recording", "source_sha256": "ab7eaa11036cd84ee646ad4c74113973420e28b0dadeaa5e07350deee879bb1b", "start_s": 18.708449874981305, "source_duration_s": 20.55, "available_frames": 36 } } ] }, { "start_s": 235.6, "duration_s": 2.55, "shared_elapsed_start_s": null, "arms": [ { "arm": "Stock Inkling 2 Mini", "elapsed_s": 25.25740695791319, "passed": true, "device": "RTX 5090", "receipt_sha256": "1d5cb4e084092ab42062795c3389bd13a24a9e3f8483bf3344d0fe94b85f9048", "receipt": "630-original.json", "recording": { "kind": "native browser recording", "source_sha256": "16828a5f2648215220dff2c7e61ed42c9e37a3e92cb75cd7245a25c1b023c86f", "start_s": 22.7, "source_duration_s": 25.25, "available_frames": 51 }, "grader_status": "success" }, { "arm": "Cached-submit edit", "elapsed_s": 12.70991133293137, "passed": false, "device": "RTX 5090", "receipt_sha256": "a3d5da6fcd63d35f95790e06c0bb53d798fea6aee7d9e0d5958602b2c45c3caa", "receipt": "630-edited.json", "recording": { "kind": "native browser recording", "source_sha256": "e30f1737068683f8b380feefa4053b30e2c0f9b0cd05ebe183e3c74f11d771c7", "start_s": 10.160904916981234, "source_duration_s": 13.7, "available_frames": 51 }, "grader_status": "failure" } ] } ] }, { "number": 39, "start_s": 238.15, "duration_s": 2.2, "environment": "WebArena", "task": "653", "title": "Fill a refund request form", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 26.86761945905164, "passed": false, "device": "RTX 5090", "receipt_sha256": "84dabcfebfdd1f15906299ed7b26f2f00695280fe24553873f127be7f7d4eb8a", "receipt": "653-original.json", "recording": { "kind": "native browser recording", "source_sha256": "7eece87539ab508547114925371c060edfe992f2d4bddd14c85f0380ceaaf3f5", "start_s": 24.6689379170537, "source_duration_s": 27.85, "available_frames": 44 } }, { "arm": "Four-row edit", "elapsed_s": 4.754835375002585, "passed": false, "device": "RTX 5090", "receipt_sha256": "c3c6804487f380b08ec044fe53163b2ff9af0aa669920a5a160883816c3ad800", "receipt": "653-swap_click_stop.json", "recording": { "kind": "native browser recording", "source_sha256": "5cb519977a7dd560a4df0d10c67059ecfbeded79d8cf134ca68e13a285c72f73", "start_s": 2.555689707933925, "source_duration_s": 5.75, "available_frames": 44 } } ] }, { "number": 40, "start_s": 240.35, "duration_s": 2.2, "environment": "WebArena", "task": "689", "title": "Prepare a coupon request", "shared_elapsed_start_s": null, "arms": [ { "arm": "Original", "elapsed_s": 17.615055207978003, "passed": false, "device": "RTX 5090", "receipt_sha256": "56ec91b4fd23c439c77b59c524ee8f6e340b7c5452dd7948bc5a7bf315ecd050", "receipt": "689-original.json", "recording": { "kind": "native browser recording", "source_sha256": "ccd71c8a30d6426c0ec49aec48a3d86ab7a3188f543c8ac9b5e5e1995ad358d6", "start_s": 16.400000000000002, "source_duration_s": 18.6, "available_frames": 44 } } ] } ], "opening_race": { "task": "190", "device": "M5 Max MacBook Pro", "playback_rate": 1, "shared_start_s": 0, "duration_s": 6.15, "arms": [ { "name": "Stock Inkling 2 Mini", "seconds": 45.88679650006816, "receipt_sha256": "db7cc933937f077603d615ef906b8ccd107f4b80f9e4303af200341f55a6e809", "recording_sha256": "001a63b86d63e645ef77f990d03c3f2b17ea4dc17c1d6a34cc91e3b4126bca25" }, { "name": "Cross-device order stop", "seconds": 5.127113958005793, "receipt_sha256": "de99cd3b9ffdab3dcde0a6ba3b7ab53c4f78c893b5e04a1a42cd729a32ca8816", "recording_sha256": "1c0b7325f4c3119182342a5017acce4187ed18802c46dddd919c151140565054" } ] }, "embedded_chapters": 42 }