husky8's picture
Publish the ordered browser film, verified timings and standalone checkpoint links
a55a3c9 verified
Raw History Blame Contribute Delete
157 kB
{ "pairs": [ { "checkpoint": "order-boundary", "task": 188, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": false, "ratio": null, "original": { "seconds": 5.450880583026446, "score": 0.0, "grader_status": "failure", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "FAIL RETRIEVE UNKNOWN_ERROR The site metis.lti.cs.cmu.edu is unreachable (ERR_FAILED), so no order data can be retrieved." ], "generated": 639, "thinking": 546, "inference_seconds": 4.341966916923411, "receipt_sha256": "957b7bbfe248a0d71c07dfe9f6143283ca95163ce64e0c4d27d3d58a78c7e0f8", "recording_sha256": "9e49c54ad4cf8133a89475b938369b28d94fe9e23950def74d4329db26adb377" }, "edited": { "seconds": 3.161069875000976, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d", "commands": [ "C1", "C28", "ANSWER [365.42]" ], "generated": 22, "thinking": 0, "inference_seconds": 0.9746392919914797, "receipt_sha256": "0f6c92e5cec25fca09c73407032489ee4f57b9fdb24eec2c419388557ab3ddf0", "recording_sha256": "c576fd15d4c275fa3a4268d21074c49f47bf719c5e9853432f816bf289247064" } }, { "checkpoint": "order-boundary", "task": 188, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 2.591420520284619, "original": { "seconds": 6.857609499944374, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C28", "ANSWER [365.42]" ], "generated": 636, "thinking": 601, "inference_seconds": 4.802369457902387, "receipt_sha256": "c7cec829eb022d0eedbe230b27a36d6ee5346ccd33ac284c0703eed777f3f584", "recording_sha256": "fb5ff40e56daa8f3b1bb3ec7cf75c4737be808a7451d370246bb6cb44779c593" }, "edited": { "seconds": 2.6462742909789085, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d", "commands": [ "C1", "C28", "ANSWER [365.42]" ], "generated": 22, "thinking": 0, "inference_seconds": 0.6069882510928437, "receipt_sha256": "f28317df706ae824db182c96082d262e272b6c3c545cf23251c189b8db5fe92a", "recording_sha256": "dfd9521b812a2461962c05936ce0bf25ea5c2f998355315dc07d7ca122247807" } }, { "checkpoint": "order-boundary", "task": 188, "repeat": 3, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 2.072767228160558, "original": { "seconds": 6.469065667013638, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C28", "ANSWER [365.42]" ], "generated": 636, "thinking": 601, "inference_seconds": 4.4553279160754755, "receipt_sha256": "5ef518cc46d164ffeb4d3a3a80a4dc5563fb2f7f96daae5999c4534847fc483a", "recording_sha256": "958c9ea2f0a1b32fc75186e207337bc871abf869e87d165836072698ac9dba36" }, "edited": { "seconds": 3.120980291045271, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d", "commands": [ "C1", "C28", "ANSWER [365.42]" ], "generated": 22, "thinking": 0, "inference_seconds": 0.9501105819363147, "receipt_sha256": "0357e8ffe6af7f1701f92db3a70b9078d706683ea6677b571462b4e0eabbc5e9", "recording_sha256": "93072ab561becd3e4a598612f93a1332211a98caa87d7bd86ebe1d32301ccde3" } }, { "checkpoint": "order-boundary", "task": 188, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 5.359326661640119, "original": { "seconds": 27.987604541005567, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C 1", "C 1", "C 27", "ANSWER [365.42]" ], "generated": 919, "thinking": 871, "inference_seconds": 25.7202133750543, "receipt_sha256": "128fc030b4598015a85db7f0dd018c2b1bc84b5ae6843d0f058cf3f6c0367569", "recording_sha256": "4bd9540d4d612fb6907b69fadec3f04118f14317e5798648d8e8b5585da28496" }, "edited": { "seconds": 5.222224041935988, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d", "commands": [ "C1", "C1", "C27", "ANSWER [365.42]" ], "generated": 28, "thinking": 0, "inference_seconds": 2.9366160839563236, "receipt_sha256": "3ec9fe5c258b22955ee880afed5c775409b04fc431e3c8fbb8ea288ea60b93e4", "recording_sha256": "8c8aabd21f93162e25a87524faf4ded911d24b5e672e7da3fb182fb0477d12eb" } }, { "checkpoint": "order-boundary", "task": 188, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 3.819419314247951, "original": { "seconds": 29.266462499974295, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C 1", "C 1", "C 27", "ANSWER [365.42]" ], "generated": 919, "thinking": 871, "inference_seconds": 26.097251708968543, "receipt_sha256": "cb77da3b789910b351884448ed9b081ae0865494650b55c0f08b16c2cdfa1ce8", "recording_sha256": "451bc4eb12375e54d238548134731d7ed6703e51064ec823413f158f330c1c18" }, "edited": { "seconds": 7.66254241601564, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d", "commands": [ "C1", "C1", "C27", "ANSWER [365.42]" ], "generated": 28, "thinking": 0, "inference_seconds": 3.8887990009970963, "receipt_sha256": "357383bc371d424f36a29926a4f863957d26184ff921180f62ceada031526d24", "recording_sha256": "6e894e4f5b3790fc3bf50bf1dad31c972e2aeb573d78dce400acb289720b8121" } }, { "checkpoint": "order-boundary", "task": 274, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 2.2733463979361197, "original": { "seconds": 15.23012145794928, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 usb wifi", "C9", "DONE navigate" ], "generated": 377, "thinking": 342, "inference_seconds": 12.311935833073221, "receipt_sha256": "dbbbe5106ffa51e7a6820ca560359732ece5934ae28e52a06dc56445c11aba4e", "recording_sha256": "ab4548bbbc6c0d5b41c9c0875ee51573688e82031acc0bf40ccca070b6dc94b2" }, "edited": { "seconds": 6.699428416090086, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d", "commands": [ "F7 usb wifi", "C 9", "DONE navigate" ], "generated": 24, "thinking": 0, "inference_seconds": 2.4915796670829877, "receipt_sha256": "6631149152033fe004eb356f17393e8d0a14314c3344c57b561f1ee9edd8d456", "recording_sha256": "fa7bd6a481e34444be8687e5991964fbe2486d077c277594b4f5acd833df530d" } }, { "checkpoint": "order-boundary", "task": 274, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 1.6974869808068649, "original": { "seconds": 14.144440833013505, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 usb wifi", "C9", "DONE navigate" ], "generated": 377, "thinking": 342, "inference_seconds": 11.703376956982538, "receipt_sha256": "d49cbe46168654cc31315a957c0b496c597b90502e31be1de3dc6882c9ef3b01", "recording_sha256": "f78c57a3d107f7f2711d06a6d4ba6ef8f498970baa21809cbc7a828149a3fec1" }, "edited": { "seconds": 8.332576916902326, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d", "commands": [ "F7 usb wifi", "C 9", "DONE navigate" ], "generated": 24, "thinking": 0, "inference_seconds": 3.6981667910004035, "receipt_sha256": "161a1f02a25a507280279a2e2f36a7472cf4d86ef135eb5535e77d63605e8809", "recording_sha256": "7d4c872f1668ad34517d3431ca292f8f79327c514043209de8f7a7d11714e7f8" } }, { "checkpoint": "pending-native-stop", "task": 189, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 4.690249713838864, "original": { "seconds": 24.013557916972786, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C27", "ANSWER [754.99]" ], "generated": 758, "thinking": 701, "inference_seconds": 21.782493917038664, "receipt_sha256": "92cded3eaeb1c6ac345ee7153dc235b48713bb1848157613b2a7a43cc61a2cca", "recording_sha256": "1337f1b87fd8979bef3d2cdd755bf87c1334d23427d0f358a9728d715b836cc8" }, "edited": { "seconds": 5.119888999965042, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "C1", "C1", "C27", "ANSWER [754.99]" ], "generated": 28, "thinking": 0, "inference_seconds": 2.8962276668753475, "receipt_sha256": "20f5c7b9017be392ef4924dab5e91f762102f6d254f80148817a8d3b10ddba76", "recording_sha256": "8244b9f143231b5b92e7ba0d60f39146d8e831f4d4ed68f5f7bc8e57c8c6d399" } }, { "checkpoint": "pending-native-stop", "task": 189, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 5.741061375550865, "original": { "seconds": 23.85580195800867, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C27", "ANSWER [754.99]" ], "generated": 758, "thinking": 701, "inference_seconds": 21.622225041966885, "receipt_sha256": "b1634b0f8243545d4097ffe4ca933354bde835338404e687caf3aae5c2018efc", "recording_sha256": "1ee1ac955ae3bda5d4734587196fc9c44130579aa394ed8db92f8f008b490ab0" }, "edited": { "seconds": 4.1552947090240195, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "C1", "C27", "ANSWER [754.99]" ], "generated": 22, "thinking": 0, "inference_seconds": 2.4584740839200094, "receipt_sha256": "e6c644893422e06b9a286c7b2a9ddf26a1cde522b86c712bbf9ca2de49158152", "recording_sha256": "3cfe1ad7af674a69fbeca895f5c882f54b60d2cf811bcb95d1bb15552a817703" } }, { "checkpoint": "pending-native-stop", "task": 189, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 3.4908572236245425, "original": { "seconds": 12.520012957975268, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C27", "", "", "ANSWER [754.99]" ], "generated": 1562, "thinking": 1516, "inference_seconds": 10.342188334092498, "receipt_sha256": "637ddc2bcc9efed7959e36faf0409d31339d847d40b7cdc22e4a90a383178681", "recording_sha256": "d6dc8ea68b740d7ee8708a1a761b95a3f56f6a5ea020f8bdecde4fb3ba82bf4b" }, "edited": { "seconds": 3.5865153330378234, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "C1", "C28", "ANSWER [754.99]" ], "generated": 27, "thinking": 0, "inference_seconds": 1.3370232919696718, "receipt_sha256": "ad87f98adb0c4f50778203c7ef93c0da8ce1340f16a480be81ab0a965fd84639", "recording_sha256": "7dac24884bef50b5afebd54a69aeeedd5fdd67abbad3e484f7ddc8e978b055ef" } }, { "checkpoint": "pending-native-stop", "task": 189, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 2.3868772001649847, "original": { "seconds": 6.852668249979615, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C28", "ANSWER [754.99]" ], "generated": 620, "thinking": 578, "inference_seconds": 4.467606668011285, "receipt_sha256": "5ca42820d967775d2c77ca87c7a07b09ff805ad2ac827cc99774ce01c169be7e", "recording_sha256": "462fde75ead0791de0fbf04f15c0283f3052fd9bc88a6bdce55eb671940dc451" }, "edited": { "seconds": 2.8709764580707997, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "C1", "C28", "ANSWER [754.99]" ], "generated": 27, "thinking": 0, "inference_seconds": 0.8831789589021355, "receipt_sha256": "5ea4e294f8dcf1b0dbdc9462cd11262af9c7803bd16fda7540e5455c723c4c46", "recording_sha256": "bf4d4b52fe1fcf00e99fccfef11a2cc7844367cba01340f7ddd991101877c2ba" } }, { "checkpoint": "pending-native-stop", "task": 188, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 5.4233442163780445, "original": { "seconds": 27.98192616691813, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C 1", "C 1", "C 27", "ANSWER [365.42]" ], "generated": 919, "thinking": 871, "inference_seconds": 25.61580516619142, "receipt_sha256": "c4e77e913a2700d41fc3c2c0f92210fa9984f150e34db5108682efb9aa5d9d69", "recording_sha256": "091e6e77a2fa5a8ee9236e8596ba70a97c218ae4adbf2002289786c8c1b615aa" }, "edited": { "seconds": 5.15953350008931, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "C1", "C1", "C27", "ANSWER [365.42]" ], "generated": 28, "thinking": 0, "inference_seconds": 2.9151421249844134, "receipt_sha256": "1a5fd921f02fcda29dda6d58913f0b3b9928f292b380cac33fbc33499bf1d2db", "recording_sha256": "a4a1cb13c081ca25ec98305179606451fd1c39bd2d855bf5672a99cecc63d367" } }, { "checkpoint": "pending-native-stop", "task": 188, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 5.317121801071281, "original": { "seconds": 28.35290774994064, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C 1", "C 1", "C 27", "ANSWER [365.42]" ], "generated": 919, "thinking": 871, "inference_seconds": 26.04253954195883, "receipt_sha256": "f041e43abedf42b3a82505b3103d2ab475a775b192763acf9a0ca7950e5ec37a", "recording_sha256": "f583df51c82b5316d8bec41dee8f2381a9521b08bb8694502721793998870939" }, "edited": { "seconds": 5.332378834020346, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "C1", "C1", "C27", "ANSWER [365.42]" ], "generated": 28, "thinking": 0, "inference_seconds": 2.835695124929771, "receipt_sha256": "a2cd8e1e72203e060e25b4fec5e3cf3ace430637e579fc2b81298b43ebcac803", "recording_sha256": "03b6a501240822c7bbae639704f26618abac051a77a66ccbf60bb6f23901dfd9" } }, { "checkpoint": "pending-native-stop", "task": 274, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 2.576897133969056, "original": { "seconds": 16.203930207993835, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 usb wifi", "C9", "DONE navigate" ], "generated": 377, "thinking": 342, "inference_seconds": 12.098144791903906, "receipt_sha256": "2b71849b36d362f701517b915550525910f4afc5b426a88c137842bd1f0382a2", "recording_sha256": "409331b716e8b290701a23c1ae30d86dc28155b45374d723101e2b11ca268daf" }, "edited": { "seconds": 6.288155624992214, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "F7 usb wifi", "C 9", "DONE navigate" ], "generated": 24, "thinking": 0, "inference_seconds": 2.6552863750839606, "receipt_sha256": "e66a2f773bb3f091e5c4fe01c5e5a2043d91bb62c87a94468df370c5d522539f", "recording_sha256": "185965f2d023ea1c681268a9fa6fee53ceef4f6fb489641f19153b4612366180" } }, { "checkpoint": "pending-native-stop", "task": 274, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 2.1599602468069996, "original": { "seconds": 15.693350290996023, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 usb wifi", "C9", "DONE navigate" ], "generated": 377, "thinking": 342, "inference_seconds": 12.281953208963387, "receipt_sha256": "571feff368482ac1bdd966a55a7768822bb8c0e57ba34e82459620d22edf9918", "recording_sha256": "d9228f92519c624ffbff61dd48a540c7b9d3925d8e4606f7ce5aec4db2e73d53" }, "edited": { "seconds": 7.265573667013086, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "F7 usb wifi", "C 9", "DONE navigate" ], "generated": 24, "thinking": 0, "inference_seconds": 3.17822329106275, "receipt_sha256": "395b7ef79d73fc39daa7011fffd20d8d2deaa21d28620b8fe81a402625776e0c", "recording_sha256": "390bae81b54061e1ea797bcfd61659b866b9aeb246b78d3a9a4a887769dddb6e" } }, { "checkpoint": "pending-native-stop", "task": 275, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 3.869762236782253, "original": { "seconds": 18.401632540975697, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F 7 xbox", "K", "K", "C 9", "DONE navigate" ], "generated": 491, "thinking": 448, "inference_seconds": 15.962464833981358, "receipt_sha256": "08e061c6ed6db2fd62fe4d7fe05eb57385c78e21d90b3bc5c760c317c6a7cf91", "recording_sha256": "d40c572948a3ba59457b824f18a11a7081018f8096fec4a2815686aaa5c4bd92" }, "edited": { "seconds": 4.755235958960839, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "F7 xbox", "C 9", "DONE navigate" ], "generated": 22, "thinking": 0, "inference_seconds": 2.4998124580597505, "receipt_sha256": "0b66fecbb366ec5510b95a03b1d8e4181e0c17a123653d7212f2f28f78349965", "recording_sha256": "39c761e1df8fd1eb594145cbddff0874c2f2ed93b03096a1dd4556edcbb2d808" } }, { "checkpoint": "pending-native-stop", "task": 275, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 3.435848931056179, "original": { "seconds": 18.26574841607362, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F 7 xbox", "K", "K", "C 9", "DONE navigate" ], "generated": 491, "thinking": 448, "inference_seconds": 16.133834291016683, "receipt_sha256": "d32ba301e36399f71bb0cc6f49e6856ec4285c774feb1bae99df4c4b56b5f142", "recording_sha256": "853754b6e2dc851c85ed8ccde96ff32d47cf398721d8787ebfd379632baa0e55" }, "edited": { "seconds": 5.316225707996637, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "F7 xbox", "C 9", "DONE navigate" ], "generated": 22, "thinking": 0, "inference_seconds": 2.954169916920364, "receipt_sha256": "9b1fe463d73c3cc4610ed0338e7c18b4f1809e9e2136e24ed609a91d908698cc", "recording_sha256": "9429d85b909bc2fab4f79b702330b717dd7aa7ede2867576b45451df54019cd4" } }, { "checkpoint": "pending-native-stop", "task": 276, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 2.343943583427812, "original": { "seconds": 14.041020957985893, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 switch accessories", "C9", "DONE navigate" ], "generated": 312, "thinking": 284, "inference_seconds": 10.495542207965627, "receipt_sha256": "541e5d568123893c38017af7ab728bfb236491636ecd9f40a02ef83e99e7242e", "recording_sha256": "dea2bff742159d7ae154651cc1aa822c225bdb2f020b04ede0565ef74b5c5aeb" }, "edited": { "seconds": 5.990340832970105, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "F7 switch accessories", "C 9", "DONE navigate" ], "generated": 23, "thinking": 0, "inference_seconds": 2.522448667092249, "receipt_sha256": "3f4a1ffd9aa8a109bb9c40e9f630b4f32c8671a6242fdb01acc1dfa761b25949", "recording_sha256": "4454ed999d13e56ca986b6b419f4e858f8a15a8471a2cb62e5b331df542911b4" } }, { "checkpoint": "pending-native-stop", "task": 276, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 1.7859771140842748, "original": { "seconds": 14.029689416987821, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 switch accessories", "C9", "DONE navigate" ], "generated": 312, "thinking": 284, "inference_seconds": 10.5524590840796, "receipt_sha256": "d4ff5a29f8e72c1f4a3a41943448262c3d5e1856aab0a683d048b28be4c47a55", "recording_sha256": "77aa1ee806f8f440553efe8a2251e417034ac21a69f297115e31ca8f0a26c835" }, "edited": { "seconds": 7.855469874921255, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "F7 switch accessories", "C 9", "DONE navigate" ], "generated": 23, "thinking": 0, "inference_seconds": 5.144786624936387, "receipt_sha256": "aad2e34d010b71a07c4d5c43bf72601d30f2d7b6ab780834f8e08c1ddfe3b800", "recording_sha256": "6b947bf405c7d7325d07d1ebfed1a3f9684d648f7d3eb03e1e0467e173084ff3" } }, { "checkpoint": "pending-native-stop", "task": 96, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Published search weights", "comparable": true, "ratio": 6.102700053998085, "original": { "seconds": 46.78624800010584, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "d75b5709ac7a94fde651571ab7b810147f1c9ea43009a62bccfffbe4e3eab44c", "commands": [ "C1", "C1", "C27", "C21", "ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]" ], "generated": 1568, "thinking": 1201, "inference_seconds": 43.03139620705042, "receipt_sha256": "eab3fd9452d84e7f88f60e7af2a9b4a3c8eba1c0fd06069a0e6c357ad2d15719", "recording_sha256": "a1e0deaee168cf97f2aa2bcab75637c68ec0e1c6e73142dd696f6b3b4d4baef7" }, "edited": { "seconds": 7.6664832920068875, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "C1", "C1", "C27", "C21", "ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]" ], "generated": 50, "thinking": 0, "inference_seconds": 4.144598416867666, "receipt_sha256": "a318d072c734c8025be9ca65e1504e4874a5917b414f237521c2ce94cc3ab6ce", "recording_sha256": "f6a10adb1c0b5bcd988acf0727a23515287d7d145b8940d932ef4cd6cf57ede0" } }, { "checkpoint": "pending-native-stop", "task": 96, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Published search weights", "comparable": true, "ratio": 6.567726605205213, "original": { "seconds": 48.26774325000588, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "d75b5709ac7a94fde651571ab7b810147f1c9ea43009a62bccfffbe4e3eab44c", "commands": [ "C1", "C1", "C27", "C21", "ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]" ], "generated": 1568, "thinking": 1201, "inference_seconds": 45.08624020800926, "receipt_sha256": "aeca3c77f135fa9e7b9c3726bb5835d903ab635ed4fd8875179be1909a5725ae", "recording_sha256": "3aa13a46f9003e40ed8bd3da88f9e84967b1574b051eb9fb7861742539c5f7e8" }, "edited": { "seconds": 7.349231499945745, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "C1", "C1", "C27", "C21", "ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]" ], "generated": 50, "thinking": 0, "inference_seconds": 4.145376333151944, "receipt_sha256": "492b2c7a22b0db8499facb0841cfe1c2f39a7f2116635dfe99f35fdf0091d837", "recording_sha256": "43269aea0aeac812951a512bc814013ef28e34c1808f3d8f450f608e724b9d20" } }, { "checkpoint": "pending-native-stop", "task": 188, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 1.943401366683929, "original": { "seconds": 5.755649958970025, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C27", "ANSWER [365.42]" ], "generated": 494, "thinking": 459, "inference_seconds": 3.584349957993254, "receipt_sha256": "f94f9adc964d4670e727d8c593afef0545590296ffb1b86fa6e9c0e9d4b44811", "recording_sha256": "efc55d108ab168208d79ba827228dfa840759a17189ebeb0a72bc3f0bfffbf87" }, "edited": { "seconds": 2.961637291009538, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "C1", "C28", "ANSWER [365.42]" ], "generated": 22, "thinking": 0, "inference_seconds": 0.8650916670449078, "receipt_sha256": "0303bcaf5546b57453f9b40851dc3537df8534a7ab5bf9fa361e4dd2dc3dd9f8", "recording_sha256": "026734ff64cf3e9dba913d682f960ce050139fc4a4690fda5fe809aca755dced" } }, { "checkpoint": "pending-native-stop", "task": 188, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 2.428293647989783, "original": { "seconds": 6.62423158402089, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C28", "ANSWER [365.42]" ], "generated": 636, "thinking": 601, "inference_seconds": 4.467334917979315, "receipt_sha256": "32c46688488dfe923d9402a00d6402b3451d010dded23453b6c4db0408364eaa", "recording_sha256": "6d5e63865be0dd998908296402ca9c213474022bdb02655f9e802dba4f37df35" }, "edited": { "seconds": 2.727936791954562, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "C1", "C28", "ANSWER [365.42]" ], "generated": 22, "thinking": 0, "inference_seconds": 0.6054706659633666, "receipt_sha256": "e86f58d22dccb23b78a4eef217a99ce6d4854034e87b1d20e34f2d650045efa0", "recording_sha256": "af6b0db45fb22c880537000deba98bb7b8cedff4edafd6dd081ce0d6d8f78803" } }, { "checkpoint": "pending-native-stop", "task": 190, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 2.482351834068574, "original": { "seconds": 45.501502291997895, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C27", "", "ANSWER [65.32]" ], "generated": 1555, "thinking": 1509, "inference_seconds": 42.78597154200543, "receipt_sha256": "0c5039a54ece418d72927e94ef46bf347afc2f7c4328924b409d0bd8029f30b9", "recording_sha256": "aef068d2419ff13564764c7d28509ff34185b09fa4e7c68e4365eba05171aa2f" }, "edited": { "seconds": 18.32999724999536, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "C1", "C1", "C27", "ANSWER [65.32]" ], "generated": 534, "thinking": 343, "inference_seconds": 16.06127941503655, "receipt_sha256": "c7266a5d572d38b90d3949ed3958f7a547c3e495cd2d2d13089c111af9e022d9", "recording_sha256": "e2ecd40026fc920fedc2bd53366aed4636b1e97fc769c771694a3f1516581c5a" } }, { "checkpoint": "pending-native-stop", "task": 190, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 2.4815721427158386, "original": { "seconds": 49.00226112501696, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C27", "", "", "ANSWER [65.32]" ], "generated": 1720, "thinking": 1676, "inference_seconds": 47.02861291600857, "receipt_sha256": "9d2dc23993320ee07ee8a433c42f2d2f8213ea8325fda157996c97121c1d1745", "recording_sha256": "4c8f95c65e3c46768337005a263e32b21eee32912970bdb400ae3c851b9885b5" }, "edited": { "seconds": 19.746458417037502, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "C1", "C1", "C27", "ANSWER [65.32]" ], "generated": 534, "thinking": 343, "inference_seconds": 17.46305316616781, "receipt_sha256": "5d95a3825cd56c0f25059384003386af36d72f7ddc18939485fd981846662cd3", "recording_sha256": "92bcb1b32fc35ffaeac839747167f0b96cf755dfcb02b2a2c55928629f71d88a" } }, { "checkpoint": "pending-native-stop", "task": 62, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping_admin", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 75.4982749170158, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "5", "C5", "C5", "C5", "C5", "C5", "C5", "C5", "C6", "C8", "C24", "F17 2020-01-01" ], "generated": 2379, "thinking": 2161, "inference_seconds": 68.99048875004519, "receipt_sha256": "3e081086af99c49c181ece856f66992d59331e4332aadf1d33fa4d54a51b7f73", "recording_sha256": "8b121887d4ef926468b58cc5873d2a2a737180e5e237be6aef8d813dc6353387" }, "edited": { "seconds": 0.7192069169832394, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "DONE navigate" ], "generated": 7, "thinking": 0, "inference_seconds": 0.7151282919803634, "receipt_sha256": "b1e4632555bd70819b5b05162b66bb945dcc83480914ac92f79ed9945dd0df90", "recording_sha256": "916a14f93614ba687b3cb27bd38f4b31af8cb6ecdc6ef1762e6774af7e68cc51" } }, { "checkpoint": "pending-native-stop", "task": 62, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping_admin", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 76.96602091705427, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "5", "C5", "C5", "C5", "C5", "C5", "C5", "C5", "C6", "C8", "C24", "F17 2020-01-01" ], "generated": 2379, "thinking": 2161, "inference_seconds": 70.3279061667854, "receipt_sha256": "0a3e878d66f833d6be4e41d7b163ee16a444ec91c9b088d6b90250fc8c0cabad", "recording_sha256": "1f998482e8b079e22bafd05f3fdfabdbdac79cad7e6c65e9dd23b40c23d7d2ed" }, "edited": { "seconds": 1.008508124970831, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c", "commands": [ "DONE navigate" ], "generated": 7, "thinking": 0, "inference_seconds": 1.005628832965158, "receipt_sha256": "1bea02762e10cecab560e7f1b1de4df9a9d7e92338cf00ef1c1a07e52e2e887c", "recording_sha256": "069d872e8fa980b43a8410fdcef1ed6a48b6f6d224b5e141659fee79719f7eb8" } }, { "checkpoint": "complete-stop", "task": 190, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 8.734988222647253, "original": { "seconds": 45.33590604097117, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C27", "", "ANSWER [65.32]" ], "generated": 1555, "thinking": 1509, "inference_seconds": 42.99088237492833, "receipt_sha256": "b944a6bd099412a0bad414e64b4990c7b99d8d9c25c191cd040399511f710f3e", "recording_sha256": "942526c0551c329e1ac489749aad074e1ef028e893eb28801167a6512d218e66" }, "edited": { "seconds": 5.190150791895576, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C1", "C27", "ANSWER [65.32]" ], "generated": 28, "thinking": 0, "inference_seconds": 2.92674979113508, "receipt_sha256": "84cf20808be895d2e05f3534cac15b07c016dc676049a987a70ae08f0f108be2", "recording_sha256": "7832813623fc7a3cf56fe92630aa5d7346ee07478b94d24bafffad56ced8b3fe" } }, { "checkpoint": "complete-stop", "task": 190, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 7.7460947913453335, "original": { "seconds": 46.58707808400504, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C27", "", "ANSWER [65.32]" ], "generated": 1555, "thinking": 1509, "inference_seconds": 44.305321957101114, "receipt_sha256": "9b47dcf6c674a568ceb0f8fbc4119dd16df576a41f3396b34dd249eb9b202307", "recording_sha256": "cec595452188d8350b71db44e21484f22217c4e84e955a3c27844b664b879dec" }, "edited": { "seconds": 6.014266457990743, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C1", "C27", "ANSWER [65.32]" ], "generated": 28, "thinking": 0, "inference_seconds": 3.309025958995335, "receipt_sha256": "8b729c4bcaea345f9744a11f42c1257d38ec3be17b1236f694e4de2bae22cc38", "recording_sha256": "ad265854077e737e48522c06297e43afc05be25661d9a37e4a29a0fd0eba844a" } }, { "checkpoint": "complete-stop", "task": 190, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 0.7768753233595105, "original": { "seconds": 7.21626258303877, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C28", "ANSWER [65.32]" ], "generated": 662, "thinking": 612, "inference_seconds": 4.76776066608727, "receipt_sha256": "8726d5aa918653cd9a022445555ba3f978836fefa0d160feac9b580ed0cd5372", "recording_sha256": "fbb66bed1ce5064b2bf276daf9e310be8e8cf82473c4397f03cd3e66eac6f37a" }, "edited": { "seconds": 9.288829708006233, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C28", "ANSWER [65.32]" ], "generated": 1034, "thinking": 767, "inference_seconds": 7.138767582830042, "receipt_sha256": "8837ce095ebbf98d48c0c2cf246993515e50042e85d13ced316f91a1c1cd3662", "recording_sha256": "b0e9e487c12576311698b8e8efb727745b3c5abb98b3dcb6004591524c4f4699" } }, { "checkpoint": "complete-stop", "task": 190, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 1.3608271132567153, "original": { "seconds": 12.153708999976516, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C28", "ANSWER [65.32]" ], "generated": 662, "thinking": 612, "inference_seconds": 4.9256642068503425, "receipt_sha256": "2c7a4e2807c73cd87a088b3d20e2d958db61fed9c14b8317980503920b3be701", "recording_sha256": "24915a2e17337a0b0d87f7cfcdc83f58cd02e4c8d9b541a6811426a06e940a17" }, "edited": { "seconds": 8.931119083077647, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C28", "ANSWER [65.32]" ], "generated": 1034, "thinking": 767, "inference_seconds": 6.83465154201258, "receipt_sha256": "72fc0400d00b8206a2d5ed97e3056e362a4f7c0d05a0e932d414b6ae97611a00", "recording_sha256": "bc1d171a325af34b00b81ba751c1d2f23b8a0a8b8ac5d6496264456aedbe319c" } }, { "checkpoint": "complete-stop", "task": 191, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 11.728927165968344, "score": 0.0, "grader_status": "failure", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "1", "1", "1", "1 a", "1 a", "1 a 1", "1 a 1", "1 a 1", "1 a", "C 1", "C 28", "D" ], "generated": 1240, "thinking": 1080, "inference_seconds": 9.507527501089498, "receipt_sha256": "a50856d15bb6364982e3dee40947266669790d0829db39fb9c3e0d07533d3603", "recording_sha256": "cb08030636da33fcc3620fc94101b555aa61fc8f832e2382cacf13b30e318041" }, "edited": { "seconds": 3.0059442919446155, "score": 0.0, "grader_status": "error", "device": "RTX 5090", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C28", "ANSWER [\"$365.42\"]" ], "generated": 30, "thinking": 0, "inference_seconds": 0.9276975840330124, "receipt_sha256": "742867ddf6f8b75053dfe21966d372805bb4310d938dfd5828f9d8f41216d479", "recording_sha256": "cbd08257ce72a112f677b7486daefa30e143b6a8c91b166a5e0f127f6ec55549" } }, { "checkpoint": "complete-stop", "task": 191, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 9.138379249954596, "score": 0.0, "grader_status": "failure", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "1", "1", "1", "1<click>", "1<click>", "1<click>", "1<click> 1", "1<click>", "1<click>", "1<click>", "1<click>", "1<click> 1" ], "generated": 1056, "thinking": 858, "inference_seconds": 8.987814538180828, "receipt_sha256": "37a6b8b838075a84424405af6f22d02d5f23e13b8f30b8afbf9b65e145da7029", "recording_sha256": "c6c80ac2530dd3a6f5556c5c1eda08687d602a388c1c4b179c295fe3a7ad5992" }, "edited": { "seconds": 2.7042575840605423, "score": 0.0, "grader_status": "error", "device": "RTX 5090", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C28", "ANSWER [\"$365.42\"]" ], "generated": 30, "thinking": 0, "inference_seconds": 0.6422569160349667, "receipt_sha256": "ed3a93933827ae679c54f7d586de41e62783e48599d4673cac1ef35254778c42", "recording_sha256": "4f7e01a2274723d5fd21a9642fffe945beff254d9fdf3a980b29b33ed3873561" } }, { "checkpoint": "complete-stop", "task": 275, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 0.7906818639505235, "original": { "seconds": 4.807198666036129, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 xbox", "C9", "DONE navigate" ], "generated": 315, "thinking": 288, "inference_seconds": 2.4517232079524547, "receipt_sha256": "bda878fa25ef3d346c5c56d56854475ec23c5176097057290769c47a615edca2", "recording_sha256": "1d2e68f8a990e8209ded2fc6c8b965a3b472fc3cdf32455e8d4620f0b8e5871a" }, "edited": { "seconds": 6.079814000055194, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "F 7 xbox", "C 9", "DONE navigate" ], "generated": 528, "thinking": 358, "inference_seconds": 3.751093417056836, "receipt_sha256": "b36a2386bd99b5bcce06fd963cd5e114add5e373323cb4948fff505f4311ddf2", "recording_sha256": "0c9e5d7de9282572e6d465d9cd9cf88f3ad32d5029665dd3e5ad3c250773a53c" } }, { "checkpoint": "complete-stop", "task": 275, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 0.8236512005633566, "original": { "seconds": 5.00892183301039, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 xbox", "C9", "DONE navigate" ], "generated": 315, "thinking": 288, "inference_seconds": 2.400264916010201, "receipt_sha256": "e2ddcc46e4c061c79c9d2669ecbff0f6ccd069a3fa895273916cdefa9a5dd614", "recording_sha256": "90909532e5e7a03ef11e431f670b650c64adc4b87ed6173b91b4474a451e325d" }, "edited": { "seconds": 6.081362874945626, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "F 7 xbox", "C 9", "DONE navigate" ], "generated": 528, "thinking": 358, "inference_seconds": 3.70308424893301, "receipt_sha256": "3de8aefd1e2fcce1fec46d94a4e3498444550b0c77062e94d219e6db5f1902d4", "recording_sha256": "d126217723fea7e12c378ec0e931a570d40178157550c328b307bbb8b58a935e" } }, { "checkpoint": "complete-cross", "task": 190, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 9.333173334016465, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C28", "", "ANSWER [65.32]" ], "generated": 1104, "thinking": 1060, "inference_seconds": 7.393800664925948, "receipt_sha256": "583e40d83a2dae419c463e1a58a2f47de180396ad80763f32dbbbf3a55d52fe7", "recording_sha256": "ca72633366aff0727d18d68f0fdb00af50b53630d994725e56f2055f9006fbb9" }, "edited": { "seconds": 3.2936403329949826, "score": 0.0, "grader_status": "failure", "device": "RTX 5090", "shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889", "commands": [ "C1", "C28", "DONE navigate" ], "generated": 19, "thinking": 0, "inference_seconds": 1.0277085829293355, "receipt_sha256": "c9c7fe2cf441982418f578852a90317ed712a0ff675a75864085f40825ccd304", "recording_sha256": "8d5ee987796266a46304fc15dffd7e24e748c345cc6dae4bf3904712d795bba6" } }, { "checkpoint": "complete-cross", "task": 190, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 12.414950583013706, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C28", "", "", "ANSWER [65.32]" ], "generated": 1545, "thinking": 1499, "inference_seconds": 10.28308187611401, "receipt_sha256": "28dfb5fe4aa6758b8ce48d99869af6bececc54431280a23781aa6bdbfbf42a78", "recording_sha256": "6cc9161e3fe28c6f4770eae3dd5bf7adf923cff8c8aa09e8912a1e377b51e79f" }, "edited": { "seconds": 3.1889395409962162, "score": 0.0, "grader_status": "failure", "device": "RTX 5090", "shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889", "commands": [ "C1", "C1", "C28", "ANSWER [762.18]" ], "generated": 28, "thinking": 0, "inference_seconds": 0.8131322080735117, "receipt_sha256": "82e5d2e9d7890ec8db155b8d402d9ea526073957c4b18cebd21a38c8211f48ac", "recording_sha256": "aefcbb784438eec64ab97f93fcb26c2fdc527ca9176d01a8a1c36aece794a043" } }, { "checkpoint": "order-short-margin", "task": 190, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 0.8425275440700961, "original": { "seconds": 5.717545500025153, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C27", "ANSWER [65.32]" ], "generated": 470, "thinking": 428, "inference_seconds": 3.4925888748839498, "receipt_sha256": "3d27fc260c6451f5bf4d262375982a9e3405da0f0e8f5101bd0d2e76e6e51450", "recording_sha256": "153ddba796ec0cea3348bdf7bd04db3f859fd799f9da48a1cc94fc3fc1efc293" }, "edited": { "seconds": 6.7861822919221595, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "cf2d48bcddb8952fbb415293b1155b17f2941562d664d290b2d7a0c75944422f", "commands": [ "C1", "C28", "", "DONE", "ANSWER [65.32]" ], "generated": 540, "thinking": 510, "inference_seconds": 4.444257041905075, "receipt_sha256": "0e182901fc852d33d93ece0a897f195fc1a606c6355dff4cf78cdd7ff0543324", "recording_sha256": "0ea83acd758f6e9fc7994e04609925d6a5d10954abfe3104107b66f6590eac2f" } }, { "checkpoint": "order-short-margin", "task": 190, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 3.0895748438194923, "original": { "seconds": 10.326355000026524, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C28", "", "ANSWER [65.32]" ], "generated": 1181, "thinking": 1137, "inference_seconds": 7.871952000074089, "receipt_sha256": "1c087c5814f1b002e1ea9377eda5ae25767e0362adbf324db50698ae35aa14bd", "recording_sha256": "75627feca13b116dd5c8711c21396b0ad8ba92904f6d6985cfdcdff2e39128dd" }, "edited": { "seconds": 3.342322333017364, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "cf2d48bcddb8952fbb415293b1155b17f2941562d664d290b2d7a0c75944422f", "commands": [ "C1", "C28", "ANSWER [65.32]" ], "generated": 22, "thinking": 0, "inference_seconds": 1.1054121260531247, "receipt_sha256": "57474389b1de11eecd972d4151cf8e8887e673c33bb669705efa6b8966067709", "recording_sha256": "05bf4f59c5b86c85d42dd5b4919c9a63d0d52aea1c6bfbfb93ca0e322f79aee9" } }, { "checkpoint": "order-history-route", "task": 190, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 2.0347174710697638, "original": { "seconds": 9.880964207928628, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C28", "", "ANSWER [65.32]" ], "generated": 1181, "thinking": 1137, "inference_seconds": 7.881787376129068, "receipt_sha256": "372d90d6141a4dda6107d86ae775038e2e0a64fd8ca2284a27a703c3a4916722", "recording_sha256": "f79074bb7f4cb544f66fa3f312794508f1c5f990eea5afa581dcbc188ea95735" }, "edited": { "seconds": 4.856184875010513, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073", "commands": [ "C1", "C28", "ANSWER [65.32]" ], "generated": 335, "thinking": 252, "inference_seconds": 2.784148833132349, "receipt_sha256": "18dc25332d1bccbbd06e9769cebcbc6f3a9d93369113dacfc94b3d63af488656", "recording_sha256": "ad50979b258cd8f5d3980dbe720beaa013787474a9b4c4968bcee86d6b1f9bd5" } }, { "checkpoint": "order-history-route", "task": 190, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 2.1755407560053266, "original": { "seconds": 10.092377167078666, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C28", "", "ANSWER [65.32]" ], "generated": 1181, "thinking": 1137, "inference_seconds": 7.956198376021348, "receipt_sha256": "d80a54f157029a2fdf18c152cf4c9be428055a5574ecf70100a031d63422eb85", "recording_sha256": "10264730f938c44c0831997391e4afcfea057cba7c97f31043653d8e58856865" }, "edited": { "seconds": 4.63902004098054, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073", "commands": [ "C1", "C28", "ANSWER [65.32]" ], "generated": 335, "thinking": 252, "inference_seconds": 2.5504851238802075, "receipt_sha256": "0c0427e0d66d742d46459dde8b9b9cbeeb7744057b831a49ef28fe71d636c16a", "recording_sha256": "34d4f7f84ec4a464b2cf9ad45dc74950ecf4177b43691f46aa0d71bdc3512397" } }, { "checkpoint": "order-history-route", "task": 189, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 2.1573877246999222, "original": { "seconds": 6.301418249960989, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C28", "ANSWER [754.99]" ], "generated": 620, "thinking": 578, "inference_seconds": 4.256730875116773, "receipt_sha256": "a931070b8d954ea26af76e8a472877b8810e7d4d165c4c9af63e8e16115f5cb1", "recording_sha256": "2038aa9c1016a75aef04d502a4f790f76259f27ebbffb86ddb532c755dc4b5cf" }, "edited": { "seconds": 2.9208557079546154, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073", "commands": [ "C1", "C28", "ANSWER [754.99]" ], "generated": 30, "thinking": 0, "inference_seconds": 0.7570810420438647, "receipt_sha256": "27ac1dc9318bc5e1026f478030af5934d3c8ebd9f3d274f6e895e847c254a409", "recording_sha256": "710d46d01844b0323c393730ca915aff375b00f8b7fd642fed6c303398e2b058" } }, { "checkpoint": "order-history-route", "task": 189, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 2.3613654577613685, "original": { "seconds": 6.77029658399988, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C28", "ANSWER [754.99]" ], "generated": 620, "thinking": 578, "inference_seconds": 4.633256333065219, "receipt_sha256": "0cea03e6cc7571e55dc7bf0fcfdae72c74474be28a708ed8c8a8ef89d405058b", "recording_sha256": "4a5813e8ccab420942171b54d76c3c8c6979f2238075a2f1615da63647d27d5f" }, "edited": { "seconds": 2.8671108750859275, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073", "commands": [ "C1", "C28", "ANSWER [754.99]" ], "generated": 30, "thinking": 0, "inference_seconds": 0.6372076660627499, "receipt_sha256": "38994d8c0ccc902ff2a20f85752d8996348439c8f2de2fed16ac66e0e130ccb8", "recording_sha256": "77e0dd320302f1be3b54c6052cb774f2f3cf67981508c2f3bb041138002d4d0c" } }, { "checkpoint": "order-history-route", "task": 188, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 2.2340988253506344, "original": { "seconds": 6.307632959098555, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C28", "ANSWER [365.42]" ], "generated": 636, "thinking": 601, "inference_seconds": 4.462099457974546, "receipt_sha256": "ece2e1ff126dfe2e79cc898f9f0abc9900ed6316e2361251acc06394b4c94712", "recording_sha256": "cbe8a8253a5e729703f3435b4472ae9bf8505dd6ba2c1d1a0440a70790115ef3" }, "edited": { "seconds": 2.823345542070456, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073", "commands": [ "C1", "C28", "ANSWER [365.42]" ], "generated": 22, "thinking": 0, "inference_seconds": 0.9109670828329399, "receipt_sha256": "cfd52c7f0d070ccf4bcdb41e61353929243173510fd17ef2816172a36587c3bc", "recording_sha256": "04ff5295e15c53889a00aee652cf12e12368d270a7419b1c03f86c7f2dfe4dcc" } }, { "checkpoint": "order-history-route", "task": 188, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 3.526484137272057, "original": { "seconds": 9.687184332986362, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C28", "ANSWER [365.42]" ], "generated": 1077, "thinking": 1026, "inference_seconds": 7.271809124969877, "receipt_sha256": "cdc35d8bbdee31f92e5940f640e5d65584e4fa5f1d7dfa23d48c722cc3aa7506", "recording_sha256": "408df306a04a38c47902fd374b9fea8cea2a379b66a0fae370f18059428a7d3f" }, "edited": { "seconds": 2.7469808330060914, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073", "commands": [ "C1", "C28", "ANSWER [365.42]" ], "generated": 22, "thinking": 0, "inference_seconds": 0.6333339589182287, "receipt_sha256": "b8fa575755735d911468f7fe35395a5c56e0954cb79df2458b9f415b252433fe", "recording_sha256": "da565e7526d5613ac7b4a6405214591cd8b2b706fd9bc474830627469f8fce1d" } }, { "checkpoint": "order-history-route", "task": 191, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 8.418479042011313, "score": 0.0, "grader_status": "failure", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "1", "1", "1", "1<click>", "1<click>", "1<click>", "1<click> 1", "1<click>", "1<click>", "1<click>", "1<click>", "1<click> 1" ], "generated": 1056, "thinking": 858, "inference_seconds": 8.246465002186596, "receipt_sha256": "7e8cbc36e5095c7df6a6b54ff0165fb76a83d8909bc1074d4beb222f537788c9", "recording_sha256": "b5e52b0d037f58be3f3f5d3e33fa4c61577a57ccf71cb8486209c076e4e95a69" }, "edited": { "seconds": 4.19265154202003, "score": 0.0, "grader_status": "error", "device": "RTX 5090", "shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073", "commands": [ "C1", "C1", "C28", "The page shows orders with statuses Canceled,", "ANSWER [\"$754.99\"]" ], "generated": 70, "thinking": 15, "inference_seconds": 1.5621499170083553, "receipt_sha256": "f515e9de11c72ea33e3bbd84e2f2acc402998968fee8eaf8d35462f8924a6ad7", "recording_sha256": "e947f45bcea752e6b8ac792fe0b8e9b1a188311839fcd822088f4524f5a4810f" } }, { "checkpoint": "order-history-route", "task": 191, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 12.566988083999604, "score": 0.0, "grader_status": "failure", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "1", "1", "1", "1 a", "1 a", "1 a 1", "1 a 1", "1 a 1", "1 a", "C 1", "C 28", "D" ], "generated": 1294, "thinking": 1134, "inference_seconds": 10.013543709064834, "receipt_sha256": "5610b4c1e255ec71dba6390a839e52aa53632ca3e5eb94e0a80acb4e2e749b57", "recording_sha256": "850cad11095323b921fe44fd8a50487cbbc3b1a51a312664f688d0dc073e0ab4" }, "edited": { "seconds": 11.337048665969633, "score": 0.0, "grader_status": "error", "device": "RTX 5090", "shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073", "commands": [ "C1", "C28", "C1", "C28", "DONE", "DONE", "ANSWER [\"$365.42\"]" ], "generated": 1076, "thinking": 569, "inference_seconds": 7.667066333931871, "receipt_sha256": "01307ed3f7cee0eed3d9d915eb039beba143a5f55f3b9a566210894b2ee37574", "recording_sha256": "bee98dbd6fa6fbfcf54d871edf97d374285878291a7c49d1420bada18ed0d17e" } }, { "checkpoint": "order-history-route", "task": 275, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 0.7902392212209556, "original": { "seconds": 5.020188458962366, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 xbox", "C9", "DONE navigate" ], "generated": 315, "thinking": 288, "inference_seconds": 2.458598250988871, "receipt_sha256": "61fb1b50bc29906ad350fd24a1b1e15ebed4a36b679e61d1f06a5f89ed9c963b", "recording_sha256": "14429436b695064bb95a4db5ad46cd2f1c92e6a2283c2aa9175d0cdb19be67fd" }, "edited": { "seconds": 6.352745249983855, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073", "commands": [ "F 7 xbox", "C 9", "DONE navigate" ], "generated": 528, "thinking": 388, "inference_seconds": 3.990014291019179, "receipt_sha256": "8ef5a9dda6679658b3549adf62a04655d89ca128503e539deab907b65d214294", "recording_sha256": "524f90fa5fa0edb6001e43e3e61ed5b73c053ba1139d7b494a0ff7eb185c340e" } }, { "checkpoint": "order-history-route", "task": 275, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 0.7715830968978724, "original": { "seconds": 4.637832708074711, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 xbox", "C9", "DONE navigate" ], "generated": 315, "thinking": 288, "inference_seconds": 2.3957099590916187, "receipt_sha256": "f131cf000fc72ff2f1759f348c05098a0ff22b4623c2cb559f9419f71a8c2aff", "recording_sha256": "3f1dae74ea90990353d351262ee52ad0bcb41d6554169763dc5d9f6490edcbeb" }, "edited": { "seconds": 6.010801333934069, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073", "commands": [ "F 7 xbox", "C 9", "DONE navigate" ], "generated": 528, "thinking": 388, "inference_seconds": 3.70716108311899, "receipt_sha256": "7915ce1de6a4f4d67a5e2d30d84fdf7ce9a03380f8c3d76cea2333f9222268fc", "recording_sha256": "805f68103cbd9dc63cc61d40fb3c97b2847d91174103f8585ca826bf630d5978" } }, { "checkpoint": "search-submit-stop", "task": 275, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 1.4533256825965108, "original": { "seconds": 4.781691708951257, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 xbox", "C9", "DONE navigate" ], "generated": 315, "thinking": 288, "inference_seconds": 2.4454436249798164, "receipt_sha256": "2cf58128d38a9db061d9749db7e3bd1811c0832b99b45036c81103a99ac695b1", "recording_sha256": "da5153ef7157b25aaccf66b9579b58c5a49882b70e08d00eb8c216b7543d09fa" }, "edited": { "seconds": 3.290172165958211, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1", "commands": [ "F 7 xbox", "C 9", "DONE navigate" ], "generated": 23, "thinking": 0, "inference_seconds": 0.8812786679482087, "receipt_sha256": "d2eeef9872f3686a34d77f90bc36122ff3c5c16d69817ff200bb894d0dad016f", "recording_sha256": "3c5b0de9c51ba3e024bbe3c6dd21227860b6feddfbb03d4e8de50d17e7e245ea" } }, { "checkpoint": "search-submit-stop", "task": 275, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 1.582510684555791, "original": { "seconds": 4.751382500049658, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 xbox", "C9", "DONE navigate" ], "generated": 315, "thinking": 288, "inference_seconds": 2.456910749897361, "receipt_sha256": "1ed81a2acf8fc1c1f74a90b325579f1692c73339bcc2581c8f8484c2de7c0887", "recording_sha256": "3d8f81550117b42ee36ada4ccd70dff5808fd4c6c9bf9685eb305daa60e4ffa7" }, "edited": { "seconds": 3.002433125046082, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1", "commands": [ "F 7 xbox", "C 9", "DONE navigate" ], "generated": 23, "thinking": 0, "inference_seconds": 0.6660729580326006, "receipt_sha256": "05fbd793a01633e453f840b53c32ce89d0b97515fcd68aced3b9bdf24de01554", "recording_sha256": "993a915623f7851110f0178c347bd5ae29ad90318f9e9c1c9ace7c0bf53aff2a" } }, { "checkpoint": "search-submit-stop", "task": 190, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 3.2633434005578077, "original": { "seconds": 9.295178166008554, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C28", "", "ANSWER [65.32]" ], "generated": 1104, "thinking": 1060, "inference_seconds": 7.403692627092823, "receipt_sha256": "10e113f940c9e247fd8334c79dbad2180d4601a67a2dc440581a33b0dd942ed8", "recording_sha256": "3cbd6436db955b45efb52b2295633470ed96ded06e28bfaa7a7d9a23acadee81" }, "edited": { "seconds": 2.8483604160137475, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1", "commands": [ "C1", "C28", "ANSWER [65.32]" ], "generated": 25, "thinking": 0, "inference_seconds": 0.6726702089654282, "receipt_sha256": "2944f50cd972d2d335dead10baae03824517bccd51345bb5254e87b397ecbb44", "recording_sha256": "b6aa451ecac294a4142363a3ebec5555bda2325574d81e31a24ab89d071345d0" } }, { "checkpoint": "search-submit-stop", "task": 190, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "RTX 5090", "baseline": "Upstream stock", "comparable": true, "ratio": 2.0244892763429303, "original": { "seconds": 5.566456000087783, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C27", "ANSWER [65.32]" ], "generated": 470, "thinking": 428, "inference_seconds": 3.4088242081925273, "receipt_sha256": "b65e08b040c60338431603e3510aa875d20f73162388d02daa66264fa298102e", "recording_sha256": "07bffc2c0ddb18728b97dca662fe5b7a1b47939140b1fa16384018f5dce26283" }, "edited": { "seconds": 2.74956062505953, "score": 1.0, "grader_status": "success", "device": "RTX 5090", "shard_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1", "commands": [ "C1", "C28", "ANSWER [65.32]" ], "generated": 25, "thinking": 0, "inference_seconds": 0.6097358749248087, "receipt_sha256": "5e9af560c75070b430256e4520921dd3c1989febe05ad211799c8d752aa44650", "recording_sha256": "117012b2166e80a118d3f091da7391a5fbeb5b4a619a2b8a72ac89b4c3df5798" } }, { "checkpoint": "order-history-route", "task": 189, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 4.57448292218901, "original": { "seconds": 24.47346133296378, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C27", "ANSWER [754.99]" ], "generated": 758, "thinking": 701, "inference_seconds": 22.08682024909649, "receipt_sha256": "4bd09806c228658ab91f939594cf2394cbc6a88ae34782ce77d31b3fe1a67e76", "recording_sha256": "2b9b390beb267c67ef41ce4fed6b5896d46cae845ecb7d0410b58873ec2ff49b" }, "edited": { "seconds": 5.3499951249687, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073", "commands": [ "C1", "C1", "C27", "ANSWER [754.99]" ], "generated": 28, "thinking": 0, "inference_seconds": 2.9966340420069173, "receipt_sha256": "2b0995cf755f61451029ee24ada6b0a5674e282ac40d3ba4a86addaa3e575671", "recording_sha256": "bc0f54dce5dc9be7ef20fff1a96e6aec22b02998c7e389d82b4707a07e59e197" } }, { "checkpoint": "order-history-route", "task": 189, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 10.68081008293666, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "DONE navigate" ], "generated": 291, "thinking": 245, "inference_seconds": 9.320337167009711, "receipt_sha256": "82989d6377e02629fa1f2dffe1c95a0d30d76c8c6f6c5f2b66e596ea09ea4724", "recording_sha256": "0715a986758d86297fae6490a51e76d22a8c52d49517c9c12046f4c47e967d80" }, "edited": { "seconds": 5.62365950003732, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073", "commands": [ "C1", "C1", "C27", "ANSWER [754.99]" ], "generated": 28, "thinking": 0, "inference_seconds": 3.3328368751099333, "receipt_sha256": "7d03ccb179350b096228d2aca8f222a62f8dea960520a3457d204a8488717404", "recording_sha256": "662b69db5b7489e98cba75cf23d11ba1bcfca6623a14577a9be52851b69b267d" } }, { "checkpoint": "order-history-route", "task": 190, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 6.463688805082036, "original": { "seconds": 33.8939617080614, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C28", "", "ANSWER [65.32]" ], "generated": 1114, "thinking": 1062, "inference_seconds": 31.489520958042704, "receipt_sha256": "33da861ce0d55a2e50c046dbbc47e5a4b4c84c732ee34703f5b5d4f0c2fa3ccd", "recording_sha256": "93d3cd7fe6d23f2c78f4b3ca84205702795e2b2b4edd9f2cb71a280ff2690080" }, "edited": { "seconds": 5.243749000015669, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073", "commands": [ "C1", "C1", "C27", "ANSWER [65.32]" ], "generated": 28, "thinking": 0, "inference_seconds": 2.925669873948209, "receipt_sha256": "8ec829841ea50b195c871e4cfbaccb0a5f6b776aa256b8a8c4601c67f825bb48", "recording_sha256": "5542b2d9a3e08d20326cc8b0f3aaa648dee8e353bf68825a39747ee9fe79274b" } }, { "checkpoint": "order-history-route", "task": 190, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 8.187736102644966, "original": { "seconds": 47.179218083037995, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C27", "", "ANSWER [65.32]" ], "generated": 1555, "thinking": 1509, "inference_seconds": 43.978050000965595, "receipt_sha256": "99f4d7ef78dd466de0df077ee455cba9f3e8cd3517bc0a94ef33ff6f8aa51cc8", "recording_sha256": "4974c4c2a4452b1874bee80a103ad38b77124bc03ddd05c1cacd41603c8e6258" }, "edited": { "seconds": 5.762181082973257, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073", "commands": [ "C1", "C1", "C27", "ANSWER [65.32]" ], "generated": 28, "thinking": 0, "inference_seconds": 3.2434727910440415, "receipt_sha256": "4ee31a78c56726cb5e325af372acb7cee4ba4d7894ed7ae0d72e06ab645423e1", "recording_sha256": "638a31430f6f8abb2169b4b5b087cdefe93d800f4cde76836b2df959cb35f3cc" } }, { "checkpoint": "search-focus-route", "task": 283, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 87.93015045800712, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 Xbox controller", "K", "K", "C9", "", "D", "C3", "F7 Xbox controller", "K", "C9", "D", "D" ], "generated": 2766, "thinking": 2596, "inference_seconds": 80.11356908199377, "receipt_sha256": "2c450dd02c237f79c8c5162b8916d7a597dd137a771a903101909ff3ff4dba83", "recording_sha256": "e5e09a023862edf3033be65c4a8c22306acd87f4881cf6076507ae184f138943" }, "edited": { "seconds": 18.76859758410137, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "6238b11e7bdac91f63ee8c7ead04730f22fe2fe2d24e9d54a29a1da161109964", "commands": [ "F 7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller" ], "generated": 109, "thinking": 0, "inference_seconds": 9.257374460343271, "receipt_sha256": "5ddc2381679f9010a0cbe1497c08631418a24110efeba721a68207316efb1100", "recording_sha256": "75373c42203df67877ff8e69cb6111aa5b86498715a0a2d75a1c035b6e242d12" } }, { "checkpoint": "search-focus-route", "task": 283, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 91.99072962498758, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 Xbox controller", "K", "K", "C9", "", "D", "C3", "F7 Xbox controller", "K", "C9", "D", "D" ], "generated": 2766, "thinking": 2596, "inference_seconds": 84.98011841706466, "receipt_sha256": "daa117518b5d3a27c71bb6a00dcd118399deb9a811c7e2051561ca3773f2fdb4", "recording_sha256": "2106edb0d09e042d0250aee1f1c1e02db44da8a1ec68a7f3899fab94912eeac8" }, "edited": { "seconds": 19.445797957945615, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "6238b11e7bdac91f63ee8c7ead04730f22fe2fe2d24e9d54a29a1da161109964", "commands": [ "F 7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller" ], "generated": 109, "thinking": 0, "inference_seconds": 9.901397418114357, "receipt_sha256": "647615c6e9794bde47330ac66cffd6b573fc13891a9c8621dcde44bda44a8657", "recording_sha256": "604d6845df54d958fa87ba6b25f494f7b52de11f854103a28130fd5b73ee1db4" } }, { "checkpoint": "search-state-route", "task": 283, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 83.21020058402792, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 Xbox controller", "K", "K", "C9", "", "D", "C3", "F7 Xbox controller", "K", "C9", "D", "D" ], "generated": 2669, "thinking": 2499, "inference_seconds": 76.3192295411136, "receipt_sha256": "ee4856c05d0ff4d0770283c305adda801af5687b4b832386d59b49b5b08d1d47", "recording_sha256": "7e5531f2ff6b397792d722269290804638dd8557eca6c3ff55419a5579ecf7cf" }, "edited": { "seconds": 35.93294025002979, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "fbdc9f126dca8a7a5f12d4e120ab44138ad9b5226b18071811cf1dbab2c2ee2f", "commands": [ "F 7 Xbox controller", "C 9", "C 24", "C 24", "C 24", "F7 Xbox controller", "F7 Xbox controller", "C 9", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller" ], "generated": 619, "thinking": 340, "inference_seconds": 23.04296320991125, "receipt_sha256": "f407596f9683e3096d782dce5c1efd674c319e14bb841f544b1f681f2b5369e7", "recording_sha256": "3be4a09bcb12e77dfbdc95c936122164911447bd4d32d10485dd05cca6023587" } }, { "checkpoint": "search-state-route", "task": 283, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 83.12202070804778, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 Xbox controller", "K", "K", "C9", "", "D", "C3", "F7 Xbox controller", "K", "C9", "D", "D" ], "generated": 2669, "thinking": 2499, "inference_seconds": 76.33653653995134, "receipt_sha256": "14c8b35f16242370748001daf5d2edb9f34bd0a85a767327d17ab406cb484020", "recording_sha256": "fea0ffd4f68222c863169530e7ac9e5a41a42f6fd49a440987ffd9ea84fe3d1e" }, "edited": { "seconds": 36.30528724996839, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "fbdc9f126dca8a7a5f12d4e120ab44138ad9b5226b18071811cf1dbab2c2ee2f", "commands": [ "F 7 Xbox controller", "C 9", "C 24", "C 24", "C 24", "F7 Xbox controller", "F7 Xbox controller", "C 9", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller", "F7 Xbox controller" ], "generated": 619, "thinking": 340, "inference_seconds": 23.61733170680236, "receipt_sha256": "fab4988369b541c5f5fec5f4a693883568a87998f63c44674c9d8dd921cad58d", "recording_sha256": "27a569e67037f1f26cb58c9058144de3d15ad5fb43cf4949fcafe1c96dd8d279" } }, { "checkpoint": "search-product-guard", "task": 283, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 87.58633945893962, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 Xbox controller", "K", "K", "C9", "", "D", "C3", "F7 Xbox controller", "K", "C9", "D", "D" ], "generated": 2669, "thinking": 2499, "inference_seconds": 79.38518158299848, "receipt_sha256": "9b65b62e25ac44058102f06c2d942d6126a77631c3ce56fbbd251db947e7ee9b", "recording_sha256": "34cbd3e8dbd7a2e06aba936385ed06dc098f249606cf3dbc4abc27cebab3d2dc" }, "edited": { "seconds": 7.486511458992027, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "59a63e7b5bf01cf1e30d15bae19160bc87cfbc8817bb76f8d807b67659468d76", "commands": [ "F 7 Xbox controller", "C 9", "D", "DONE navigate" ], "generated": 44, "thinking": 0, "inference_seconds": 3.6446031250525266, "receipt_sha256": "ecb4338098a011c1e62da6714b102055d2cb79f940f14c814134c395fa0410ea", "recording_sha256": "a9a065d9141c9c0330e4508a9b97e3394ffb043b9740630670f2ae6dea8b0ba2" } }, { "checkpoint": "search-product-guard", "task": 283, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 87.25159720901866, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 Xbox controller", "K", "K", "C9", "", "D", "C3", "F7 Xbox controller", "K", "C9", "D", "D" ], "generated": 2669, "thinking": 2499, "inference_seconds": 77.33748512610327, "receipt_sha256": "d66762f6419ae58da4788db933ed3f3b75bfa6e1c51a56cccfa597e6f85d83a4", "recording_sha256": "ef8cf7aad893bab833b8ce5c4ff490a8a3c4bf0bbad712147b988a687f30e1cd" }, "edited": { "seconds": 6.887347457930446, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "59a63e7b5bf01cf1e30d15bae19160bc87cfbc8817bb76f8d807b67659468d76", "commands": [ "F 7 Xbox controller", "C 9", "D", "DONE navigate" ], "generated": 44, "thinking": 0, "inference_seconds": 3.9920278328936547, "receipt_sha256": "ac5ec0fc2c018d744c6ebf943c011d7df703521f975f59fb9c5f34d1ff831899", "recording_sha256": "f38bb011e9cfc41516b02d03ca455f613c7b2e3002a10e896623e6b7d727122f" } }, { "checkpoint": "complete-cross", "task": 188, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 5.139163248802451, "original": { "seconds": 29.21747625002172, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C 1", "C 1", "C 27", "ANSWER [365.42]" ], "generated": 919, "thinking": 871, "inference_seconds": 26.511262458050624, "receipt_sha256": "6bb3d9cee3f2c29dfa6f1f3365a14cf3d6b9fe8889806c3723f34f7b10027e3a", "recording_sha256": "6820017ac3012088d72b67ce71a3033a00c99080ab8c9d7084260614f23acc14" }, "edited": { "seconds": 5.685259415884502, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889", "commands": [ "C1", "C1", "C27", "ANSWER [365.42]" ], "generated": 28, "thinking": 0, "inference_seconds": 3.07812479196582, "receipt_sha256": "28b77c094267bdbd487300f980ab913649643201e4c1fda25e47f1e724f94d5a", "recording_sha256": "8f6003418507a7b980e15c97ccc252482dab2e4b06a055983fcc67aee44bdf2f" } }, { "checkpoint": "complete-cross", "task": 188, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 5.7143168900475185, "original": { "seconds": 28.377125291968696, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C 1", "C 1", "C 27", "ANSWER [365.42]" ], "generated": 919, "thinking": 871, "inference_seconds": 25.887863084091805, "receipt_sha256": "3aaa8a25ffdaa732a1a70c677a37814f170166842ebcc159f1b5c686211fa36b", "recording_sha256": "70a36ad97eea415680f681f942724a0f229e9f29900e0e17a5314dd9e687935a" }, "edited": { "seconds": 4.96596983296331, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889", "commands": [ "C1", "C1", "C27", "ANSWER [365.42]" ], "generated": 28, "thinking": 0, "inference_seconds": 2.887187918066047, "receipt_sha256": "83f983c722b2c509d99b5ad98494f59a2b16ad133e367b1e7b1570b3f2163f90", "recording_sha256": "36b0c3d8a789ab14144424dbeaed9f28ad0339321b3aa2fa79361dd1655825fb" } }, { "checkpoint": "complete-cross", "task": 189, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 6.306722379373041, "original": { "seconds": 33.1291674580425, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C27", "", "ANSWER [754.99]" ], "generated": 1077, "thinking": 1018, "inference_seconds": 30.86204899905715, "receipt_sha256": "32a8f504c05053537034dfbb42a8e60a8001f905b80217e28e9d08bc37ef638c", "recording_sha256": "1ea97126fa12b6406a6cb817782c3a92b22b0e8085a4c1ef4cb276bf0dab9e31" }, "edited": { "seconds": 5.252992832916789, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889", "commands": [ "C1", "C1", "C27", "ANSWER [754.99]" ], "generated": 28, "thinking": 0, "inference_seconds": 2.9513221670640633, "receipt_sha256": "13aaad04bde4944080119ae795b247c8d5da61c67a8946ae6d563b83e58ec72e", "recording_sha256": "b1bf7f929afa1478e03a8d2aa16c2b69d4631cbcd4d8fb469b17e21bb969f8c9" } }, { "checkpoint": "complete-cross", "task": 189, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 6.068339618706268, "original": { "seconds": 34.832229208084755, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C27", "", "ANSWER [754.99]" ], "generated": 1077, "thinking": 1018, "inference_seconds": 32.43203541578259, "receipt_sha256": "783df0e7274659e7aa808deb99b0a8ffed543618fd66a713b181b872c89b8a89", "recording_sha256": "dc4a0ea98c167baa7589d8a10d221e30f8a369ab3d923d212a775c7f71c31a5b" }, "edited": { "seconds": 5.739993374911137, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889", "commands": [ "C1", "C1", "C27", "ANSWER [754.99]" ], "generated": 28, "thinking": 0, "inference_seconds": 3.319720249855891, "receipt_sha256": "7db36f187e055594787561247a0bc3510c9f06c55eb2d0eb5fd971bee803e021", "recording_sha256": "56aea5b1f1787d9126e3cf4f8ca1a33be80f5e67a218b7cdd8b34abe2f520e18" } }, { "checkpoint": "complete-cross", "task": 190, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 8.949829646056076, "original": { "seconds": 45.88679650006816, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C27", "", "ANSWER [65.32]" ], "generated": 1555, "thinking": 1509, "inference_seconds": 43.03133266605437, "receipt_sha256": "db7cc933937f077603d615ef906b8ccd107f4b80f9e4303af200341f55a6e809", "recording_sha256": "001a63b86d63e645ef77f990d03c3f2b17ea4dc17c1d6a34cc91e3b4126bca25" }, "edited": { "seconds": 5.127113958005793, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889", "commands": [ "C1", "C1", "C27", "ANSWER [65.32]" ], "generated": 28, "thinking": 0, "inference_seconds": 2.841448998893611, "receipt_sha256": "de99cd3b9ffdab3dcde0a6ba3b7ab53c4f78c893b5e04a1a42cd729a32ca8816", "recording_sha256": "1c0b7325f4c3119182342a5017acce4187ed18802c46dddd919c151140565054" } }, { "checkpoint": "complete-cross", "task": 190, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 8.004451022465457, "original": { "seconds": 44.66634520795196, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C27", "", "ANSWER [65.32]" ], "generated": 1555, "thinking": 1509, "inference_seconds": 42.26506354298908, "receipt_sha256": "2d22711d2245d3bef88a8fa25184d5bfe72fd77087c705569a64986334ea281a", "recording_sha256": "8e416deffffb3369aa3ac35af4b788820ba18d6235a40210f91cb6c7d10ceb2e" }, "edited": { "seconds": 5.580188457970507, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889", "commands": [ "C1", "C1", "C27", "ANSWER [65.32]" ], "generated": 28, "thinking": 0, "inference_seconds": 3.1810083750169724, "receipt_sha256": "1a87c03f43328859a79397f0de0f6caafee859c476dcf9bf148a50d557a85f69", "recording_sha256": "c0cb3455a04d52f9e57ac56207df5f399bc2dd5211a15b6229b9c0f938f74903" } }, { "checkpoint": "complete-cross", "task": 274, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 3.1075285239080728, "original": { "seconds": 14.461539416923188, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 usb wifi", "C9", "DONE navigate" ], "generated": 377, "thinking": 342, "inference_seconds": 11.723945749923587, "receipt_sha256": "97ab57fe1b7a619938b9644f5c97acec7e95c379d2c04ef9813d6bea10aa0d53", "recording_sha256": "3061e664dfb42af7f7b0965947bed24371ad7444456470d89cf93b625d922e3a" }, "edited": { "seconds": 4.653710917104036, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889", "commands": [ "F7 usb wifi", "C 9", "DONE navigate" ], "generated": 24, "thinking": 0, "inference_seconds": 2.3331015839939937, "receipt_sha256": "11b29066e5072ea0fcdcf58760a9965d22d841c02f3b39d18d4ef73d7fa44fe2", "recording_sha256": "2da7485aae4ee646d4a06db7afef2f21738beeba6051bb2329115b7b95a8f73e" } }, { "checkpoint": "complete-cross", "task": 274, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 1.5838020523350003, "original": { "seconds": 14.471260791062377, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 usb wifi", "C9", "DONE navigate" ], "generated": 377, "thinking": 342, "inference_seconds": 12.084626293042675, "receipt_sha256": "10a1dd4bc841178232816fcdfe3c1e6e6bef1d20cd448a235e821b0bfd9ffd46", "recording_sha256": "2369fd1337c93040c92ee0e75ad7a17561e144d14a5a99977f7f217fe1143664" }, "edited": { "seconds": 9.137038792017847, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889", "commands": [ "F7 usb wifi", "C 9", "DONE navigate" ], "generated": 24, "thinking": 0, "inference_seconds": 2.780198583146557, "receipt_sha256": "b87b3eb4cc9e9cf9a753281994c78c5f7d12128329d70fb66452eaca6fce853c", "recording_sha256": "25a99553aed072252e28c1143088416554f1e77c4a2932b9a02ce50ffb743784" } }, { "checkpoint": "complete-cross", "task": 96, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Published search weights", "comparable": true, "ratio": 6.996141269159831, "original": { "seconds": 48.335796666913666, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "d75b5709ac7a94fde651571ab7b810147f1c9ea43009a62bccfffbe4e3eab44c", "commands": [ "C1", "C1", "C27", "C21", "ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]" ], "generated": 1568, "thinking": 1201, "inference_seconds": 45.24521704099607, "receipt_sha256": "f1c8df045f1cddb4282f5bbff2d96b64feb52187c56d4f156a241cc463597709", "recording_sha256": "b913f6b44e7bc8a909937cb54cb7a111641675eca31c23373fec959ad21525af" }, "edited": { "seconds": 6.908922334085219, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889", "commands": [ "C1", "C1", "C27", "C22", "ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]" ], "generated": 50, "thinking": 0, "inference_seconds": 3.9101676668506116, "receipt_sha256": "61f1d15c0471b7d9c1931eca1246f6b57dcd787f70767ffc8b182e59c288eb7a", "recording_sha256": "b4b9cfb3f5cde9985ba1f8edeaee13168a9d1ba2d38d6cf6da88218cce5c2bcf" } }, { "checkpoint": "complete-cross", "task": 96, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Published search weights", "comparable": true, "ratio": 6.962102422319519, "original": { "seconds": 54.02872458298225, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "d75b5709ac7a94fde651571ab7b810147f1c9ea43009a62bccfffbe4e3eab44c", "commands": [ "C1", "C1", "C28", "C21", "ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]" ], "generated": 1568, "thinking": 1095, "inference_seconds": 50.73290495702531, "receipt_sha256": "b3b9f0824dd2d9bec6ba3a9ee6d30cbe402ed6098254fb4638fdb52eadbd6b65", "recording_sha256": "143cd28185ddaa3e72d9eb8937d5e70b4ff153855f749b8a15e60c5a5033e6dc" }, "edited": { "seconds": 7.760403582942672, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889", "commands": [ "C1", "C1", "C27", "C21", "ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]" ], "generated": 50, "thinking": 0, "inference_seconds": 4.534869041061029, "receipt_sha256": "f38c7f639137934aec85bac03e725a1b7d3a5914bc3b83069ff7b9a553474a8c", "recording_sha256": "2ca24ca049c369c9dc20d628cd48b850b05ed260870cca89b1c5ef0dad1abe06" } }, { "checkpoint": "complete-stop", "task": 188, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 7.1029715640817015, "original": { "seconds": 45.752225124975666, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C 1", "C 1", "C 27", "", "ANSWER [365.42]" ], "generated": 1597, "thinking": 1547, "inference_seconds": 43.50580587692093, "receipt_sha256": "9eab04dc7cff27e3a5092aab8a335560ba1b4f005dd0447e515a22df8198aef8", "recording_sha256": "441458b2d3e51e08b564656b24c2c6e591ce02670f9b444904aa0af865c9e537" }, "edited": { "seconds": 6.441279500024393, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C1", "C1", "C27", "ANSWER [365.42]" ], "generated": 34, "thinking": 0, "inference_seconds": 3.8335188737837598, "receipt_sha256": "6ca9c942603d1cb1047e49b31e961627fa3276c2ee639fa397f45203b2d61380", "recording_sha256": "5b8f672631de970383c6adcfe4e3995a2cb1ddb5422085d72fc3656c5b2073f6" } }, { "checkpoint": "complete-stop", "task": 188, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 6.024516789340297, "original": { "seconds": 29.181214791955426, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C 1", "C 1", "C 27", "ANSWER [365.42]" ], "generated": 919, "thinking": 871, "inference_seconds": 26.31592674984131, "receipt_sha256": "d2258ce41595c58ae9bde3b09186cee94c5f0cc904b87d48e1c6572126130e42", "recording_sha256": "9f1a4ab393b8a5a5b3e0f6240e56002444b322ab789e3bb75449a56beca4fde5" }, "edited": { "seconds": 4.843743624980561, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C1", "C27", "ANSWER [365.42]" ], "generated": 28, "thinking": 0, "inference_seconds": 2.792007416137494, "receipt_sha256": "79f0ca4315771fd1cab3fce07e4fed0b52feeeb417c90f12da2018906d180758", "recording_sha256": "3468a0cea7472e637a3dbc0c6a3b235b6ccdad3ddc63cc29dc8d067cfa7c065e" } }, { "checkpoint": "complete-stop", "task": 189, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 6.254271096464466, "original": { "seconds": 33.172848040936515, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C27", "", "ANSWER [754.99]" ], "generated": 1077, "thinking": 1018, "inference_seconds": 30.75702545803506, "receipt_sha256": "6065551e3f226bb44ba2969600abebd38fd0b913cf16f5f4d94fab45ce14fd96", "recording_sha256": "c022881a4a31ff876167f106dba50a18a847ce86de0f5657f322ea6ccf10a997" }, "edited": { "seconds": 5.304031042032875, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C1", "C27", "ANSWER [754.99]" ], "generated": 28, "thinking": 0, "inference_seconds": 2.961101458990015, "receipt_sha256": "dba900de9b60dde45bb9417e4d62258b90e9d90cecd61dba335258157244dd16", "recording_sha256": "0cd40fcf46fd1d432df3e67061b94ae37d77bd390c03a0c904bfcccb8b5d4ded" } }, { "checkpoint": "complete-stop", "task": 189, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 4.237631562878631, "original": { "seconds": 24.859023457975127, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "C1", "C1", "C27", "ANSWER [754.99]" ], "generated": 758, "thinking": 701, "inference_seconds": 22.223551789997146, "receipt_sha256": "5a25da8336551bb5c4dabb70d8377e0eb1c5a55b8b20acb80832754b22c15c63", "recording_sha256": "5db98471b4e7446b2624eff421d02339cd85ad66b74a5be6358bfaf6749296ab" }, "edited": { "seconds": 5.8662540829973295, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C1", "C27", "ANSWER [754.99]" ], "generated": 28, "thinking": 0, "inference_seconds": 3.574042542022653, "receipt_sha256": "3a3a2de1fa749d05dad8d20fc1e36a9b56e4f9b9f6ccb261c302b8ca13489531", "recording_sha256": "6aada3095f83402fda5df1a73421b674d849c4b0d93fa25985b5553c9f15e78f" } }, { "checkpoint": "complete-stop", "task": 231, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 31.377146000042558, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "1", "1", "1", "1", "1", "1 a", "1 a", "1 a 1", "1 a 1", "1 a 1", "1 a", "1 a 1" ], "generated": 920, "thinking": 748, "inference_seconds": 31.265100041171536, "receipt_sha256": "6ab7b6b4825e98503f76ac54ea4da6d6f19389788486ba6778faef2bcb8fd294", "recording_sha256": "3aea1b0be0d4e8bd1125a74b743474e3f4723109d8f6ecb2b6767bc86ce17ee2" }, "edited": { "seconds": 5.121967457933351, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C1", "C27", "ANSWER [\"000000170\"]" ], "generated": 30, "thinking": 0, "inference_seconds": 2.891214083065279, "receipt_sha256": "c88c14e8d706e1ec5007960ac30dd3c5214714a865525ccea79b6f5463d05cbc", "recording_sha256": "fd230a0bf607e34cda779e74694430fb50e01c81b4fea13b87cecdac971afb6b" } }, { "checkpoint": "complete-stop", "task": 231, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 35.64930824993644, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "1", "1", "1", "1", "1", "1 a", "1 a", "1 a 1", "1 a 1", "1 a 1", "1 a", "1 a 1" ], "generated": 920, "thinking": 748, "inference_seconds": 35.54366799816489, "receipt_sha256": "f7ef11410819ac3656a60c6fd14eea8270b1d837354b42e7b6defab49e06b2a6", "recording_sha256": "04bd342f411014123a9ae223e0b1ca8ab9159658f812476be88e4ad798d638ee" }, "edited": { "seconds": 7.383649916970171, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C1", "C27", "ANSWER [\"000000170\"]" ], "generated": 30, "thinking": 0, "inference_seconds": 4.439961792086251, "receipt_sha256": "5e32187e00f8defc87c829ed6b0de645a2ccd886c444a197e94aee3b06d804aa", "recording_sha256": "c5925326dade396c63fc48991f5e07ac6ae97a02dce1a4d46997f981a6418e3c" } }, { "checkpoint": "complete-stop", "task": 274, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 2.1339759539566816, "original": { "seconds": 15.146393416915089, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 usb wifi", "C9", "DONE navigate" ], "generated": 377, "thinking": 342, "inference_seconds": 11.735453625093214, "receipt_sha256": "152496a6b01bdebe0aeba8ea609086850195d8744822a5fd0b67b5c0f3bc428b", "recording_sha256": "f76d533f90281fe1d95512f225d1169df880f846f91cea5bdc15377f5282c847" }, "edited": { "seconds": 7.097733875038102, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "F7 usb wifi", "C 9", "DONE navigate" ], "generated": 24, "thinking": 0, "inference_seconds": 2.6527612908976153, "receipt_sha256": "9c1b39b97301ecbd9911182d3ce0a13b5ab3f201faa60abbea69cb73b78fcb72", "recording_sha256": "437261111d5df0b41ce83fd2010a4394d5900ee947f322ef60ad5aede114c3ad" } }, { "checkpoint": "complete-stop", "task": 274, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 2.664622233674974, "original": { "seconds": 14.349906249903142, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 usb wifi", "C9", "DONE navigate" ], "generated": 377, "thinking": 342, "inference_seconds": 11.803165707970038, "receipt_sha256": "8e75836126e49acae38b153d83368055a0d49906c9f839a229005e60e8338762", "recording_sha256": "af12cc9808d1e46d8a9c2c52e2e5a547b9e1345a484eec137aa3090d7aa8f212" }, "edited": { "seconds": 5.385343583999202, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "F7 usb wifi", "C 9", "DONE navigate" ], "generated": 24, "thinking": 0, "inference_seconds": 2.6537152079399675, "receipt_sha256": "1bbcfec71ea23fc04d20d9a67f569cf0acfa7a95e356897b655dc885a09a780e", "recording_sha256": "e897298b69c97db8c118e4c2317b0c5f90298da7dc204658cf5e7d67cf008e52" } }, { "checkpoint": "complete-stop", "task": 275, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 2.631130870629422, "original": { "seconds": 18.87460729200393, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F 7 xbox", "K", "K", "C 9", "DONE navigate" ], "generated": 491, "thinking": 448, "inference_seconds": 16.716478960122913, "receipt_sha256": "9c17b994ac387513f3ed37639c64093dd15504f46e73b18facff60417f7804c4", "recording_sha256": "bf459c9f1d0196afdb26555f16f093a9e43d02408762be5fdec0dcb7b7fc64fb" }, "edited": { "seconds": 7.1735722090816125, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "F7 xbox", "C 9", "DONE navigate" ], "generated": 22, "thinking": 0, "inference_seconds": 2.381859084009193, "receipt_sha256": "d3e2c57259f9a441e539598ffa15c70e08b0cbba27b48d0653482cccd62848bb", "recording_sha256": "9b21185ae61f559a9691c5398789890fd030da095d8dd1bd22f4e3f2851906da" } }, { "checkpoint": "complete-stop", "task": 275, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 4.085940566193339, "original": { "seconds": 20.91780145803932, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F 7 xbox", "K", "K", "C 9", "DONE navigate" ], "generated": 491, "thinking": 448, "inference_seconds": 17.97420541709289, "receipt_sha256": "bace17fc01e37ed306f4e64981bf49c67bb9a6b95100b1895f955c2e54fd1c54", "recording_sha256": "00c066740a18093c3dab6ab4a99c066338f6c2e60a0d3675214d560b8a216371" }, "edited": { "seconds": 5.119458082946949, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "F7 xbox", "C 9", "DONE navigate" ], "generated": 22, "thinking": 0, "inference_seconds": 2.7432119579752907, "receipt_sha256": "ecec834e9b1f89a8d4a50df566ab0db1caea8bdaa500e96d765805ac755e99f0", "recording_sha256": "c3836f0939f129721c89b1c4493bc3339533ffd48694876bc341252d06cbca13" } }, { "checkpoint": "complete-stop", "task": 276, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 2.6260388683186013, "original": { "seconds": 14.802143833017908, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 switch accessories", "C9", "DONE navigate" ], "generated": 312, "thinking": 284, "inference_seconds": 10.696286541060545, "receipt_sha256": "3d6755b8986e755acd9efcb9262a3499f3d3eda412384a345d72217a5cbbf27a", "recording_sha256": "621e65298b2636414d62b87c2506ce16ddf0f20a6757db53331a82624360f2f2" }, "edited": { "seconds": 5.636681167059578, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "F7 switch accessories", "C 9", "DONE navigate" ], "generated": 23, "thinking": 0, "inference_seconds": 2.8598002499202266, "receipt_sha256": "2441f22ddf5e9880abbbc20fd4c99c3600bada9eb1f29c08488c582ec64052ba", "recording_sha256": "bfc758912cdfa863e0cf101c55f53bef3a13ca4e3d4ca76f7991763dd25e9758" } }, { "checkpoint": "complete-stop", "task": 276, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 2.6794437974755816, "original": { "seconds": 14.731332250055857, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 switch accessories", "C9", "DONE navigate" ], "generated": 312, "thinking": 284, "inference_seconds": 10.826686957036145, "receipt_sha256": "71905bf063f752c654a13cbfa2f216e892d4c18809658f74f083f4f4e39d1598", "recording_sha256": "18ca2607d91d54a0a6b66b8f3c13e657a3f1eaa1078ec0d2f15e48e8f2a415f1" }, "edited": { "seconds": 5.4979067909298465, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "F7 switch accessories", "C 9", "DONE navigate" ], "generated": 23, "thinking": 0, "inference_seconds": 2.7760118750156835, "receipt_sha256": "1382dbbfddbf5afaa24370bcf99eb3c6605aa18db4b9ba05d74e5a9259b1d686", "recording_sha256": "920d1fdd4076374344a35630e6bd209a8a639523021e90f779d93b62ba109726" } }, { "checkpoint": "complete-stop", "task": 277, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 3.296279570524557, "original": { "seconds": 19.38110354100354, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 batteries for iphone 13", "K", "C9", "DONE navigate" ], "generated": 413, "thinking": 375, "inference_seconds": 14.283008917002007, "receipt_sha256": "cedcfe5b1a31810a6b057846f778a6502ee0c60f84ceb6dc279763555b30696b", "recording_sha256": "9c0d7164e632a65f82c2ab8b3ce320b5521b47404d104ca09247aa8050537c6c" }, "edited": { "seconds": 5.879690458998084, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "F7 batteries for iphone 13", "C 9", "DONE navigate" ], "generated": 26, "thinking": 0, "inference_seconds": 2.8371055430034176, "receipt_sha256": "c20a66a68f4fade474ce0021165497d211331c8281dc3136b93555e663b9c150", "recording_sha256": "372124866542f39ea4e2625481c057e19b69efe4e4ba55b88451f58e191a5c76" } }, { "checkpoint": "complete-stop", "task": 277, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 3.1275596003092097, "original": { "seconds": 20.637632584082894, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 batteries for iphone 13", "K", "C9", "DONE navigate" ], "generated": 413, "thinking": 375, "inference_seconds": 13.629928710055538, "receipt_sha256": "58185eb48ad4e0c400a4294ee6dae8c1de9cbef96e67d64c6dad31f298c56390", "recording_sha256": "d23a8f4908dbeb7069f7c4ac1d11317057ae4b0b7d6ddfd0c68f1e83f6f56d5a" }, "edited": { "seconds": 6.598637666902505, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "F7 batteries for iphone 13", "C 9", "DONE navigate" ], "generated": 26, "thinking": 0, "inference_seconds": 2.9481292499694973, "receipt_sha256": "f297dc8affab7a7d13bbc4d9402b00a4689e1f51b0288fbe88b9f8586e8d9151", "recording_sha256": "b48676bc4cf07cd388c0f16709e983c20eda7aaaccc4f4382901fe7d7d3e3309" } }, { "checkpoint": "complete-stop", "task": 278, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 3.072605270412202, "original": { "seconds": 19.747643290902488, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 green tea bag for weight loss", "K", "K", "C9", "DONE navigate" ], "generated": 498, "thinking": 447, "inference_seconds": 16.53311104211025, "receipt_sha256": "dc2bb501a144c4091b54d3464426557c7283a1da397150d916db8ef1efed3f96", "recording_sha256": "8acec465aac0e97baa9057990322b7e45548060d5cd4d2f8b60c46d0944e6774" }, "edited": { "seconds": 6.427003000047989, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "F7 green tea bag for weight loss", "C 9", "DONE navigate" ], "generated": 26, "thinking": 0, "inference_seconds": 2.7816810000222176, "receipt_sha256": "93002218d76d699268c6b36d6efe7aa6c231eed8151fc50906943ff94ced8b06", "recording_sha256": "4401d49f06fcc1423c49b97f088eb5d2085a7f95ee36b4537acaae7593b261bf" } }, { "checkpoint": "complete-stop", "task": 278, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": 3.971020153923018, "original": { "seconds": 24.29026104195509, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 green tea bag for weight loss", "K", "K", "C9", "DONE navigate" ], "generated": 498, "thinking": 447, "inference_seconds": 16.81286837288644, "receipt_sha256": "f54ed2ed164381f7fce725a451b1243c220bd58f2cf76e6864a5b4ce14c7363d", "recording_sha256": "051a34b4ceeff354add69fd699507064e20260cddc69d406284ec3e69a89005f" }, "edited": { "seconds": 6.11688183399383, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "F7 green tea bag for weight loss", "C 9", "DONE navigate" ], "generated": 26, "thinking": 0, "inference_seconds": 2.8285570829175413, "receipt_sha256": "5ad1b4a1fa25da2ebe1e5ed4f13976b9d52457987b254a55e09912f3d1ded67a", "recording_sha256": "1fc55b1c968704e5e58ae61587eaedc473f3bf7e4e69926c42efa56eca019525" } }, { "checkpoint": "complete-stop", "task": 96, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Published search weights", "comparable": true, "ratio": 6.765838304025578, "original": { "seconds": 47.030071915942244, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "d75b5709ac7a94fde651571ab7b810147f1c9ea43009a62bccfffbe4e3eab44c", "commands": [ "C1", "C1", "C27", "C21", "ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]" ], "generated": 1568, "thinking": 1201, "inference_seconds": 43.0982726670336, "receipt_sha256": "f4d2295f22d03a79aeb5746e8659c03e0bf8edc43cc902c7f1c5a0fc2367b1ac", "recording_sha256": "ffbad15481de75a1a2a4d755a0d133dc6398fd87001b8bbabac083b9708da591" }, "edited": { "seconds": 6.951107874978334, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C1", "C27", "C21", "ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]" ], "generated": 50, "thinking": 0, "inference_seconds": 3.9192599579691887, "receipt_sha256": "ed5ab8f7e7590abcc136d9d008e090952896195358a71986d6dac5919d35043d", "recording_sha256": "752e6ce4c20402ef8c25902b95b5fae1e2699abf46df5148aa351c64e057e2f8" } }, { "checkpoint": "complete-stop", "task": 96, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Published search weights", "comparable": true, "ratio": 5.8896550787316775, "original": { "seconds": 44.6329190840479, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "d75b5709ac7a94fde651571ab7b810147f1c9ea43009a62bccfffbe4e3eab44c", "commands": [ "C1", "C1", "C27", "C21", "ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]" ], "generated": 1568, "thinking": 1201, "inference_seconds": 41.65323895891197, "receipt_sha256": "8c80ef2d7d79ed00bcc7d01e5682f0ac7e97164e2c70c821794c315875599ec3", "recording_sha256": "af1d76372398c05719b345a5da3e648a22a3419cb600a6187bc29ee07f828a68" }, "edited": { "seconds": 7.578188957995735, "score": 1.0, "grader_status": "success", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C1", "C27", "C21", "ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]" ], "generated": 50, "thinking": 0, "inference_seconds": 4.394748790888116, "receipt_sha256": "902a18fe5537a3b856fc1602cb1e4591dd66e669c2e349152859db0701c81e23", "recording_sha256": "a2f9b51758f28278a313efa357575ed3d4e772ea15d173d9721a39b9340ae865" } }, { "checkpoint": "complete-stop", "task": 191, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 72.89141737495083, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "1", "1", "C1", "C27", "D", "D", "C17", "D", "C18", "D", "D", "D" ], "generated": 2323, "thinking": 1889, "inference_seconds": 67.62824633193668, "receipt_sha256": "1b0f760548d1a346c64962ec9d28e96599619d967e6a45ed79596b7c9ecaf44b", "recording_sha256": "68c336263c3ddb7e33bb2c2397001df3c1f9d3ac0e8a732c9dd444252a24dbdd" }, "edited": { "seconds": 19.83579604199622, "score": 0.0, "grader_status": "error", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C1", "C28", "ANSWER [\"$365.42\"]" ], "generated": 537, "thinking": 308, "inference_seconds": 16.693413792178035, "receipt_sha256": "65a1fffe7d9a22163195d72cc9badbacc70adc6fea9b93c0d1db398a66491975", "recording_sha256": "6d96505a8709636d495217bf4faea1f0a9324809b7892cd05e5a601e0d947d82" } }, { "checkpoint": "complete-stop", "task": 191, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 82.10596250009257, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "1", "1", "C1", "C27", "", "D", "D", "C17", "D", "C18", "D", "D" ], "generated": 2767, "thinking": 2332, "inference_seconds": 76.9749869160587, "receipt_sha256": "1dad1b17cc2dedd8e12f18a1854674c8f6ca4e59383cff29baf168b77dfb743d", "recording_sha256": "cd7cbdda2840468bbf5e0ebf02c56df2b417499a567ea88df81087495b59ede4" }, "edited": { "seconds": 21.124708750052378, "score": 0.0, "grader_status": "error", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C1", "C1", "C27", "ANSWER [\"$365.42\"]" ], "generated": 537, "thinking": 308, "inference_seconds": 18.80111487407703, "receipt_sha256": "d6cba48d27dc727f599c7ec22367f0868530e4c30c911e5fdfb446b272cb19f2", "recording_sha256": "8d84d0f6e6a12c5c9008894c7fa6186af9980c19361d56e83e76b5fc422e7256" } }, { "checkpoint": "complete-stop", "task": 283, "repeat": 1, "order": [ "edited", "original" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 81.33533662499394, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 Xbox controller", "K", "K", "C9", "", "D", "C3", "F7 Xbox controller", "K", "C9", "D", "D" ], "generated": 2668, "thinking": 2498, "inference_seconds": 74.3948387921555, "receipt_sha256": "c0668e2011b93eb9c2e80746d17db3de4c0a07df6aee1b2dc76e65a918dbe99d", "recording_sha256": "fe0be6adb38961edcb5c4e5e48798336e93ebd4d962ea6db10fb51ecac152e67" }, "edited": { "seconds": 12.874797667027451, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C7", "C7", "C7", "C7", "C7", "C7", "C7", "C7", "C7", "C7", "C7", "C7" ], "generated": 82, "thinking": 0, "inference_seconds": 8.267056584125385, "receipt_sha256": "522a89222aa4484cb9a5801275aa38ea28c6451b336fe6327eb8ec1714590ff2", "recording_sha256": "a2c70c7239ec95f2be1d56b722cf1d51d7fddca0c772d48f9e9dd0146b9c01e9" } }, { "checkpoint": "complete-stop", "task": 283, "repeat": 2, "order": [ "original", "edited" ], "site": "shopping", "device": "M5 Max MacBook Pro", "baseline": "Upstream stock", "comparable": true, "ratio": null, "original": { "seconds": 81.81984350003768, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1", "commands": [ "F7 Xbox controller", "K", "K", "C9", "", "D", "C3", "F7 Xbox controller", "K", "C9", "D", "D" ], "generated": 2669, "thinking": 2499, "inference_seconds": 74.8769646261353, "receipt_sha256": "1584689f458fb5263fc5558a005934bd8c20cb22c561c0afa8aa930b1de0dd1d", "recording_sha256": "b52d58cfa69a4287740e230d7ba541ad499688189adc92f5daa73c6893e5fcc6" }, "edited": { "seconds": 12.775214791996405, "score": 0.0, "grader_status": "failure", "device": "M5 Max MacBook Pro", "shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a", "commands": [ "C7", "C7", "C7", "C7", "C7", "C7", "C7", "C7", "C7", "C7", "C7", "C7" ], "generated": 82, "thinking": 0, "inference_seconds": 8.176048293127678, "receipt_sha256": "6271021877e564aafe7042846c9838554a213ab14db6bc2e19456ba010e3d322", "recording_sha256": "00afb4eef4002d38d985e4966142b1f8f8412181575a9a17f60767458c78373d" } } ] }
{
"pairs": [
{
"checkpoint": "order-boundary",
"task": 188,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": false,
"ratio": null,
"original": {
"seconds": 5.450880583026446,
"score": 0.0,
"grader_status": "failure",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"FAIL RETRIEVE UNKNOWN_ERROR The site metis.lti.cs.cmu.edu is unreachable (ERR_FAILED), so no order data can be retrieved."
],
"generated": 639,
"thinking": 546,
"inference_seconds": 4.341966916923411,
"receipt_sha256": "957b7bbfe248a0d71c07dfe9f6143283ca95163ce64e0c4d27d3d58a78c7e0f8",
"recording_sha256": "9e49c54ad4cf8133a89475b938369b28d94fe9e23950def74d4329db26adb377"
},
"edited": {
"seconds": 3.161069875000976,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 0.9746392919914797,
"receipt_sha256": "0f6c92e5cec25fca09c73407032489ee4f57b9fdb24eec2c419388557ab3ddf0",
"recording_sha256": "c576fd15d4c275fa3a4268d21074c49f47bf719c5e9853432f816bf289247064"
}
},
{
"checkpoint": "order-boundary",
"task": 188,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.591420520284619,
"original": {
"seconds": 6.857609499944374,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 636,
"thinking": 601,
"inference_seconds": 4.802369457902387,
"receipt_sha256": "c7cec829eb022d0eedbe230b27a36d6ee5346ccd33ac284c0703eed777f3f584",
"recording_sha256": "fb5ff40e56daa8f3b1bb3ec7cf75c4737be808a7451d370246bb6cb44779c593"
},
"edited": {
"seconds": 2.6462742909789085,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 0.6069882510928437,
"receipt_sha256": "f28317df706ae824db182c96082d262e272b6c3c545cf23251c189b8db5fe92a",
"recording_sha256": "dfd9521b812a2461962c05936ce0bf25ea5c2f998355315dc07d7ca122247807"
}
},
{
"checkpoint": "order-boundary",