Image-Text-to-Text
Transformers
Safetensors
inkling2_loop
weight-edit
browser-automation
conversational
8-bit precision
modelopt
Instructions to use husky8/Inkling-2-Mini-Fast-Browser-Use with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use husky8/Inkling-2-Mini-Fast-Browser-Use with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="husky8/Inkling-2-Mini-Fast-Browser-Use") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("husky8/Inkling-2-Mini-Fast-Browser-Use") model = AutoModelForMultimodalLM.from_pretrained("husky8/Inkling-2-Mini-Fast-Browser-Use", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use husky8/Inkling-2-Mini-Fast-Browser-Use with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "husky8/Inkling-2-Mini-Fast-Browser-Use" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "husky8/Inkling-2-Mini-Fast-Browser-Use", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/husky8/Inkling-2-Mini-Fast-Browser-Use
- SGLang
How to use husky8/Inkling-2-Mini-Fast-Browser-Use with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "husky8/Inkling-2-Mini-Fast-Browser-Use" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "husky8/Inkling-2-Mini-Fast-Browser-Use", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "husky8/Inkling-2-Mini-Fast-Browser-Use" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "husky8/Inkling-2-Mini-Fast-Browser-Use", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use husky8/Inkling-2-Mini-Fast-Browser-Use with Docker Model Runner:
docker model run hf.co/husky8/Inkling-2-Mini-Fast-Browser-Use
Download benchmark/pairs.json from husky8/Inkling-2-Mini-Fast-Browser-Use: direct link, hf CLI and curl.
- Browser
- Download file 157 kB
-
https://huggingface.co/husky8/Inkling-2-Mini-Fast-Browser-Use/resolve/main/benchmark/pairs.json
- Command line
-
hf download hf://husky8/Inkling-2-Mini-Fast-Browser-Use/benchmark/pairs.json
-
curl -L -o pairs.json https://huggingface.co/husky8/Inkling-2-Mini-Fast-Browser-Use/resolve/main/benchmark/pairs.json
157 kB
{
"pairs": [
{
"checkpoint": "order-boundary",
"task": 188,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": false,
"ratio": null,
"original": {
"seconds": 5.450880583026446,
"score": 0.0,
"grader_status": "failure",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"FAIL RETRIEVE UNKNOWN_ERROR The site metis.lti.cs.cmu.edu is unreachable (ERR_FAILED), so no order data can be retrieved."
],
"generated": 639,
"thinking": 546,
"inference_seconds": 4.341966916923411,
"receipt_sha256": "957b7bbfe248a0d71c07dfe9f6143283ca95163ce64e0c4d27d3d58a78c7e0f8",
"recording_sha256": "9e49c54ad4cf8133a89475b938369b28d94fe9e23950def74d4329db26adb377"
},
"edited": {
"seconds": 3.161069875000976,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 0.9746392919914797,
"receipt_sha256": "0f6c92e5cec25fca09c73407032489ee4f57b9fdb24eec2c419388557ab3ddf0",
"recording_sha256": "c576fd15d4c275fa3a4268d21074c49f47bf719c5e9853432f816bf289247064"
}
},
{
"checkpoint": "order-boundary",
"task": 188,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.591420520284619,
"original": {
"seconds": 6.857609499944374,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 636,
"thinking": 601,
"inference_seconds": 4.802369457902387,
"receipt_sha256": "c7cec829eb022d0eedbe230b27a36d6ee5346ccd33ac284c0703eed777f3f584",
"recording_sha256": "fb5ff40e56daa8f3b1bb3ec7cf75c4737be808a7451d370246bb6cb44779c593"
},
"edited": {
"seconds": 2.6462742909789085,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 0.6069882510928437,
"receipt_sha256": "f28317df706ae824db182c96082d262e272b6c3c545cf23251c189b8db5fe92a",
"recording_sha256": "dfd9521b812a2461962c05936ce0bf25ea5c2f998355315dc07d7ca122247807"
}
},
{
"checkpoint": "order-boundary",
"task": 188,
"repeat": 3,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.072767228160558,
"original": {
"seconds": 6.469065667013638,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 636,
"thinking": 601,
"inference_seconds": 4.4553279160754755,
"receipt_sha256": "5ef518cc46d164ffeb4d3a3a80a4dc5563fb2f7f96daae5999c4534847fc483a",
"recording_sha256": "958c9ea2f0a1b32fc75186e207337bc871abf869e87d165836072698ac9dba36"
},
"edited": {
"seconds": 3.120980291045271,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 0.9501105819363147,
"receipt_sha256": "0357e8ffe6af7f1701f92db3a70b9078d706683ea6677b571462b4e0eabbc5e9",
"recording_sha256": "93072ab561becd3e4a598612f93a1332211a98caa87d7bd86ebe1d32301ccde3"
}
},
{
"checkpoint": "order-boundary",
"task": 188,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 5.359326661640119,
"original": {
"seconds": 27.987604541005567,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C 1",
"C 1",
"C 27",
"ANSWER [365.42]"
],
"generated": 919,
"thinking": 871,
"inference_seconds": 25.7202133750543,
"receipt_sha256": "128fc030b4598015a85db7f0dd018c2b1bc84b5ae6843d0f058cf3f6c0367569",
"recording_sha256": "4bd9540d4d612fb6907b69fadec3f04118f14317e5798648d8e8b5585da28496"
},
"edited": {
"seconds": 5.222224041935988,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [365.42]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 2.9366160839563236,
"receipt_sha256": "3ec9fe5c258b22955ee880afed5c775409b04fc431e3c8fbb8ea288ea60b93e4",
"recording_sha256": "8c8aabd21f93162e25a87524faf4ded911d24b5e672e7da3fb182fb0477d12eb"
}
},
{
"checkpoint": "order-boundary",
"task": 188,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 3.819419314247951,
"original": {
"seconds": 29.266462499974295,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C 1",
"C 1",
"C 27",
"ANSWER [365.42]"
],
"generated": 919,
"thinking": 871,
"inference_seconds": 26.097251708968543,
"receipt_sha256": "cb77da3b789910b351884448ed9b081ae0865494650b55c0f08b16c2cdfa1ce8",
"recording_sha256": "451bc4eb12375e54d238548134731d7ed6703e51064ec823413f158f330c1c18"
},
"edited": {
"seconds": 7.66254241601564,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [365.42]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 3.8887990009970963,
"receipt_sha256": "357383bc371d424f36a29926a4f863957d26184ff921180f62ceada031526d24",
"recording_sha256": "6e894e4f5b3790fc3bf50bf1dad31c972e2aeb573d78dce400acb289720b8121"
}
},
{
"checkpoint": "order-boundary",
"task": 274,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.2733463979361197,
"original": {
"seconds": 15.23012145794928,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 usb wifi",
"C9",
"DONE navigate"
],
"generated": 377,
"thinking": 342,
"inference_seconds": 12.311935833073221,
"receipt_sha256": "dbbbe5106ffa51e7a6820ca560359732ece5934ae28e52a06dc56445c11aba4e",
"recording_sha256": "ab4548bbbc6c0d5b41c9c0875ee51573688e82031acc0bf40ccca070b6dc94b2"
},
"edited": {
"seconds": 6.699428416090086,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d",
"commands": [
"F7 usb wifi",
"C 9",
"DONE navigate"
],
"generated": 24,
"thinking": 0,
"inference_seconds": 2.4915796670829877,
"receipt_sha256": "6631149152033fe004eb356f17393e8d0a14314c3344c57b561f1ee9edd8d456",
"recording_sha256": "fa7bd6a481e34444be8687e5991964fbe2486d077c277594b4f5acd833df530d"
}
},
{
"checkpoint": "order-boundary",
"task": 274,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 1.6974869808068649,
"original": {
"seconds": 14.144440833013505,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 usb wifi",
"C9",
"DONE navigate"
],
"generated": 377,
"thinking": 342,
"inference_seconds": 11.703376956982538,
"receipt_sha256": "d49cbe46168654cc31315a957c0b496c597b90502e31be1de3dc6882c9ef3b01",
"recording_sha256": "f78c57a3d107f7f2711d06a6d4ba6ef8f498970baa21809cbc7a828149a3fec1"
},
"edited": {
"seconds": 8.332576916902326,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d",
"commands": [
"F7 usb wifi",
"C 9",
"DONE navigate"
],
"generated": 24,
"thinking": 0,
"inference_seconds": 3.6981667910004035,
"receipt_sha256": "161a1f02a25a507280279a2e2f36a7472cf4d86ef135eb5535e77d63605e8809",
"recording_sha256": "7d4c872f1668ad34517d3431ca292f8f79327c514043209de8f7a7d11714e7f8"
}
},
{
"checkpoint": "pending-native-stop",
"task": 189,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 4.690249713838864,
"original": {
"seconds": 24.013557916972786,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [754.99]"
],
"generated": 758,
"thinking": 701,
"inference_seconds": 21.782493917038664,
"receipt_sha256": "92cded3eaeb1c6ac345ee7153dc235b48713bb1848157613b2a7a43cc61a2cca",
"recording_sha256": "1337f1b87fd8979bef3d2cdd755bf87c1334d23427d0f358a9728d715b836cc8"
},
"edited": {
"seconds": 5.119888999965042,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [754.99]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 2.8962276668753475,
"receipt_sha256": "20f5c7b9017be392ef4924dab5e91f762102f6d254f80148817a8d3b10ddba76",
"recording_sha256": "8244b9f143231b5b92e7ba0d60f39146d8e831f4d4ed68f5f7bc8e57c8c6d399"
}
},
{
"checkpoint": "pending-native-stop",
"task": 189,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 5.741061375550865,
"original": {
"seconds": 23.85580195800867,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [754.99]"
],
"generated": 758,
"thinking": 701,
"inference_seconds": 21.622225041966885,
"receipt_sha256": "b1634b0f8243545d4097ffe4ca933354bde835338404e687caf3aae5c2018efc",
"recording_sha256": "1ee1ac955ae3bda5d4734587196fc9c44130579aa394ed8db92f8f008b490ab0"
},
"edited": {
"seconds": 4.1552947090240195,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"C1",
"C27",
"ANSWER [754.99]"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 2.4584740839200094,
"receipt_sha256": "e6c644893422e06b9a286c7b2a9ddf26a1cde522b86c712bbf9ca2de49158152",
"recording_sha256": "3cfe1ad7af674a69fbeca895f5c882f54b60d2cf811bcb95d1bb15552a817703"
}
},
{
"checkpoint": "pending-native-stop",
"task": 189,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 3.4908572236245425,
"original": {
"seconds": 12.520012957975268,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C27",
"",
"",
"ANSWER [754.99]"
],
"generated": 1562,
"thinking": 1516,
"inference_seconds": 10.342188334092498,
"receipt_sha256": "637ddc2bcc9efed7959e36faf0409d31339d847d40b7cdc22e4a90a383178681",
"recording_sha256": "d6dc8ea68b740d7ee8708a1a761b95a3f56f6a5ea020f8bdecde4fb3ba82bf4b"
},
"edited": {
"seconds": 3.5865153330378234,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"C1",
"C28",
"ANSWER [754.99]"
],
"generated": 27,
"thinking": 0,
"inference_seconds": 1.3370232919696718,
"receipt_sha256": "ad87f98adb0c4f50778203c7ef93c0da8ce1340f16a480be81ab0a965fd84639",
"recording_sha256": "7dac24884bef50b5afebd54a69aeeedd5fdd67abbad3e484f7ddc8e978b055ef"
}
},
{
"checkpoint": "pending-native-stop",
"task": 189,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.3868772001649847,
"original": {
"seconds": 6.852668249979615,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"ANSWER [754.99]"
],
"generated": 620,
"thinking": 578,
"inference_seconds": 4.467606668011285,
"receipt_sha256": "5ca42820d967775d2c77ca87c7a07b09ff805ad2ac827cc99774ce01c169be7e",
"recording_sha256": "462fde75ead0791de0fbf04f15c0283f3052fd9bc88a6bdce55eb671940dc451"
},
"edited": {
"seconds": 2.8709764580707997,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"C1",
"C28",
"ANSWER [754.99]"
],
"generated": 27,
"thinking": 0,
"inference_seconds": 0.8831789589021355,
"receipt_sha256": "5ea4e294f8dcf1b0dbdc9462cd11262af9c7803bd16fda7540e5455c723c4c46",
"recording_sha256": "bf4d4b52fe1fcf00e99fccfef11a2cc7844367cba01340f7ddd991101877c2ba"
}
},
{
"checkpoint": "pending-native-stop",
"task": 188,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 5.4233442163780445,
"original": {
"seconds": 27.98192616691813,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C 1",
"C 1",
"C 27",
"ANSWER [365.42]"
],
"generated": 919,
"thinking": 871,
"inference_seconds": 25.61580516619142,
"receipt_sha256": "c4e77e913a2700d41fc3c2c0f92210fa9984f150e34db5108682efb9aa5d9d69",
"recording_sha256": "091e6e77a2fa5a8ee9236e8596ba70a97c218ae4adbf2002289786c8c1b615aa"
},
"edited": {
"seconds": 5.15953350008931,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [365.42]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 2.9151421249844134,
"receipt_sha256": "1a5fd921f02fcda29dda6d58913f0b3b9928f292b380cac33fbc33499bf1d2db",
"recording_sha256": "a4a1cb13c081ca25ec98305179606451fd1c39bd2d855bf5672a99cecc63d367"
}
},
{
"checkpoint": "pending-native-stop",
"task": 188,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 5.317121801071281,
"original": {
"seconds": 28.35290774994064,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C 1",
"C 1",
"C 27",
"ANSWER [365.42]"
],
"generated": 919,
"thinking": 871,
"inference_seconds": 26.04253954195883,
"receipt_sha256": "f041e43abedf42b3a82505b3103d2ab475a775b192763acf9a0ca7950e5ec37a",
"recording_sha256": "f583df51c82b5316d8bec41dee8f2381a9521b08bb8694502721793998870939"
},
"edited": {
"seconds": 5.332378834020346,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [365.42]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 2.835695124929771,
"receipt_sha256": "a2cd8e1e72203e060e25b4fec5e3cf3ace430637e579fc2b81298b43ebcac803",
"recording_sha256": "03b6a501240822c7bbae639704f26618abac051a77a66ccbf60bb6f23901dfd9"
}
},
{
"checkpoint": "pending-native-stop",
"task": 274,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.576897133969056,
"original": {
"seconds": 16.203930207993835,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 usb wifi",
"C9",
"DONE navigate"
],
"generated": 377,
"thinking": 342,
"inference_seconds": 12.098144791903906,
"receipt_sha256": "2b71849b36d362f701517b915550525910f4afc5b426a88c137842bd1f0382a2",
"recording_sha256": "409331b716e8b290701a23c1ae30d86dc28155b45374d723101e2b11ca268daf"
},
"edited": {
"seconds": 6.288155624992214,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"F7 usb wifi",
"C 9",
"DONE navigate"
],
"generated": 24,
"thinking": 0,
"inference_seconds": 2.6552863750839606,
"receipt_sha256": "e66a2f773bb3f091e5c4fe01c5e5a2043d91bb62c87a94468df370c5d522539f",
"recording_sha256": "185965f2d023ea1c681268a9fa6fee53ceef4f6fb489641f19153b4612366180"
}
},
{
"checkpoint": "pending-native-stop",
"task": 274,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.1599602468069996,
"original": {
"seconds": 15.693350290996023,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 usb wifi",
"C9",
"DONE navigate"
],
"generated": 377,
"thinking": 342,
"inference_seconds": 12.281953208963387,
"receipt_sha256": "571feff368482ac1bdd966a55a7768822bb8c0e57ba34e82459620d22edf9918",
"recording_sha256": "d9228f92519c624ffbff61dd48a540c7b9d3925d8e4606f7ce5aec4db2e73d53"
},
"edited": {
"seconds": 7.265573667013086,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"F7 usb wifi",
"C 9",
"DONE navigate"
],
"generated": 24,
"thinking": 0,
"inference_seconds": 3.17822329106275,
"receipt_sha256": "395b7ef79d73fc39daa7011fffd20d8d2deaa21d28620b8fe81a402625776e0c",
"recording_sha256": "390bae81b54061e1ea797bcfd61659b866b9aeb246b78d3a9a4a887769dddb6e"
}
},
{
"checkpoint": "pending-native-stop",
"task": 275,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 3.869762236782253,
"original": {
"seconds": 18.401632540975697,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F 7 xbox",
"K",
"K",
"C 9",
"DONE navigate"
],
"generated": 491,
"thinking": 448,
"inference_seconds": 15.962464833981358,
"receipt_sha256": "08e061c6ed6db2fd62fe4d7fe05eb57385c78e21d90b3bc5c760c317c6a7cf91",
"recording_sha256": "d40c572948a3ba59457b824f18a11a7081018f8096fec4a2815686aaa5c4bd92"
},
"edited": {
"seconds": 4.755235958960839,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"F7 xbox",
"C 9",
"DONE navigate"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 2.4998124580597505,
"receipt_sha256": "0b66fecbb366ec5510b95a03b1d8e4181e0c17a123653d7212f2f28f78349965",
"recording_sha256": "39c761e1df8fd1eb594145cbddff0874c2f2ed93b03096a1dd4556edcbb2d808"
}
},
{
"checkpoint": "pending-native-stop",
"task": 275,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 3.435848931056179,
"original": {
"seconds": 18.26574841607362,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F 7 xbox",
"K",
"K",
"C 9",
"DONE navigate"
],
"generated": 491,
"thinking": 448,
"inference_seconds": 16.133834291016683,
"receipt_sha256": "d32ba301e36399f71bb0cc6f49e6856ec4285c774feb1bae99df4c4b56b5f142",
"recording_sha256": "853754b6e2dc851c85ed8ccde96ff32d47cf398721d8787ebfd379632baa0e55"
},
"edited": {
"seconds": 5.316225707996637,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"F7 xbox",
"C 9",
"DONE navigate"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 2.954169916920364,
"receipt_sha256": "9b1fe463d73c3cc4610ed0338e7c18b4f1809e9e2136e24ed609a91d908698cc",
"recording_sha256": "9429d85b909bc2fab4f79b702330b717dd7aa7ede2867576b45451df54019cd4"
}
},
{
"checkpoint": "pending-native-stop",
"task": 276,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.343943583427812,
"original": {
"seconds": 14.041020957985893,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 switch accessories",
"C9",
"DONE navigate"
],
"generated": 312,
"thinking": 284,
"inference_seconds": 10.495542207965627,
"receipt_sha256": "541e5d568123893c38017af7ab728bfb236491636ecd9f40a02ef83e99e7242e",
"recording_sha256": "dea2bff742159d7ae154651cc1aa822c225bdb2f020b04ede0565ef74b5c5aeb"
},
"edited": {
"seconds": 5.990340832970105,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"F7 switch accessories",
"C 9",
"DONE navigate"
],
"generated": 23,
"thinking": 0,
"inference_seconds": 2.522448667092249,
"receipt_sha256": "3f4a1ffd9aa8a109bb9c40e9f630b4f32c8671a6242fdb01acc1dfa761b25949",
"recording_sha256": "4454ed999d13e56ca986b6b419f4e858f8a15a8471a2cb62e5b331df542911b4"
}
},
{
"checkpoint": "pending-native-stop",
"task": 276,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 1.7859771140842748,
"original": {
"seconds": 14.029689416987821,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 switch accessories",
"C9",
"DONE navigate"
],
"generated": 312,
"thinking": 284,
"inference_seconds": 10.5524590840796,
"receipt_sha256": "d4ff5a29f8e72c1f4a3a41943448262c3d5e1856aab0a683d048b28be4c47a55",
"recording_sha256": "77aa1ee806f8f440553efe8a2251e417034ac21a69f297115e31ca8f0a26c835"
},
"edited": {
"seconds": 7.855469874921255,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"F7 switch accessories",
"C 9",
"DONE navigate"
],
"generated": 23,
"thinking": 0,
"inference_seconds": 5.144786624936387,
"receipt_sha256": "aad2e34d010b71a07c4d5c43bf72601d30f2d7b6ab780834f8e08c1ddfe3b800",
"recording_sha256": "6b947bf405c7d7325d07d1ebfed1a3f9684d648f7d3eb03e1e0467e173084ff3"
}
},
{
"checkpoint": "pending-native-stop",
"task": 96,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Published search weights",
"comparable": true,
"ratio": 6.102700053998085,
"original": {
"seconds": 46.78624800010584,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "d75b5709ac7a94fde651571ab7b810147f1c9ea43009a62bccfffbe4e3eab44c",
"commands": [
"C1",
"C1",
"C27",
"C21",
"ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]"
],
"generated": 1568,
"thinking": 1201,
"inference_seconds": 43.03139620705042,
"receipt_sha256": "eab3fd9452d84e7f88f60e7af2a9b4a3c8eba1c0fd06069a0e6c357ad2d15719",
"recording_sha256": "a1e0deaee168cf97f2aa2bcab75637c68ec0e1c6e73142dd696f6b3b4d4baef7"
},
"edited": {
"seconds": 7.6664832920068875,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"C1",
"C1",
"C27",
"C21",
"ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]"
],
"generated": 50,
"thinking": 0,
"inference_seconds": 4.144598416867666,
"receipt_sha256": "a318d072c734c8025be9ca65e1504e4874a5917b414f237521c2ce94cc3ab6ce",
"recording_sha256": "f6a10adb1c0b5bcd988acf0727a23515287d7d145b8940d932ef4cd6cf57ede0"
}
},
{
"checkpoint": "pending-native-stop",
"task": 96,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Published search weights",
"comparable": true,
"ratio": 6.567726605205213,
"original": {
"seconds": 48.26774325000588,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "d75b5709ac7a94fde651571ab7b810147f1c9ea43009a62bccfffbe4e3eab44c",
"commands": [
"C1",
"C1",
"C27",
"C21",
"ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]"
],
"generated": 1568,
"thinking": 1201,
"inference_seconds": 45.08624020800926,
"receipt_sha256": "aeca3c77f135fa9e7b9c3726bb5835d903ab635ed4fd8875179be1909a5725ae",
"recording_sha256": "3aa13a46f9003e40ed8bd3da88f9e84967b1574b051eb9fb7861742539c5f7e8"
},
"edited": {
"seconds": 7.349231499945745,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"C1",
"C1",
"C27",
"C21",
"ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]"
],
"generated": 50,
"thinking": 0,
"inference_seconds": 4.145376333151944,
"receipt_sha256": "492b2c7a22b0db8499facb0841cfe1c2f39a7f2116635dfe99f35fdf0091d837",
"recording_sha256": "43269aea0aeac812951a512bc814013ef28e34c1808f3d8f450f608e724b9d20"
}
},
{
"checkpoint": "pending-native-stop",
"task": 188,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 1.943401366683929,
"original": {
"seconds": 5.755649958970025,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C27",
"ANSWER [365.42]"
],
"generated": 494,
"thinking": 459,
"inference_seconds": 3.584349957993254,
"receipt_sha256": "f94f9adc964d4670e727d8c593afef0545590296ffb1b86fa6e9c0e9d4b44811",
"recording_sha256": "efc55d108ab168208d79ba827228dfa840759a17189ebeb0a72bc3f0bfffbf87"
},
"edited": {
"seconds": 2.961637291009538,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 0.8650916670449078,
"receipt_sha256": "0303bcaf5546b57453f9b40851dc3537df8534a7ab5bf9fa361e4dd2dc3dd9f8",
"recording_sha256": "026734ff64cf3e9dba913d682f960ce050139fc4a4690fda5fe809aca755dced"
}
},
{
"checkpoint": "pending-native-stop",
"task": 188,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.428293647989783,
"original": {
"seconds": 6.62423158402089,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 636,
"thinking": 601,
"inference_seconds": 4.467334917979315,
"receipt_sha256": "32c46688488dfe923d9402a00d6402b3451d010dded23453b6c4db0408364eaa",
"recording_sha256": "6d5e63865be0dd998908296402ca9c213474022bdb02655f9e802dba4f37df35"
},
"edited": {
"seconds": 2.727936791954562,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 0.6054706659633666,
"receipt_sha256": "e86f58d22dccb23b78a4eef217a99ce6d4854034e87b1d20e34f2d650045efa0",
"recording_sha256": "af6b0db45fb22c880537000deba98bb7b8cedff4edafd6dd081ce0d6d8f78803"
}
},
{
"checkpoint": "pending-native-stop",
"task": 190,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.482351834068574,
"original": {
"seconds": 45.501502291997895,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C27",
"",
"ANSWER [65.32]"
],
"generated": 1555,
"thinking": 1509,
"inference_seconds": 42.78597154200543,
"receipt_sha256": "0c5039a54ece418d72927e94ef46bf347afc2f7c4328924b409d0bd8029f30b9",
"recording_sha256": "aef068d2419ff13564764c7d28509ff34185b09fa4e7c68e4365eba05171aa2f"
},
"edited": {
"seconds": 18.32999724999536,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [65.32]"
],
"generated": 534,
"thinking": 343,
"inference_seconds": 16.06127941503655,
"receipt_sha256": "c7266a5d572d38b90d3949ed3958f7a547c3e495cd2d2d13089c111af9e022d9",
"recording_sha256": "e2ecd40026fc920fedc2bd53366aed4636b1e97fc769c771694a3f1516581c5a"
}
},
{
"checkpoint": "pending-native-stop",
"task": 190,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.4815721427158386,
"original": {
"seconds": 49.00226112501696,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C27",
"",
"",
"ANSWER [65.32]"
],
"generated": 1720,
"thinking": 1676,
"inference_seconds": 47.02861291600857,
"receipt_sha256": "9d2dc23993320ee07ee8a433c42f2d2f8213ea8325fda157996c97121c1d1745",
"recording_sha256": "4c8f95c65e3c46768337005a263e32b21eee32912970bdb400ae3c851b9885b5"
},
"edited": {
"seconds": 19.746458417037502,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [65.32]"
],
"generated": 534,
"thinking": 343,
"inference_seconds": 17.46305316616781,
"receipt_sha256": "5d95a3825cd56c0f25059384003386af36d72f7ddc18939485fd981846662cd3",
"recording_sha256": "92bcb1b32fc35ffaeac839747167f0b96cf755dfcb02b2a2c55928629f71d88a"
}
},
{
"checkpoint": "pending-native-stop",
"task": 62,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping_admin",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 75.4982749170158,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"5",
"C5",
"C5",
"C5",
"C5",
"C5",
"C5",
"C5",
"C6",
"C8",
"C24",
"F17 2020-01-01"
],
"generated": 2379,
"thinking": 2161,
"inference_seconds": 68.99048875004519,
"receipt_sha256": "3e081086af99c49c181ece856f66992d59331e4332aadf1d33fa4d54a51b7f73",
"recording_sha256": "8b121887d4ef926468b58cc5873d2a2a737180e5e237be6aef8d813dc6353387"
},
"edited": {
"seconds": 0.7192069169832394,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"DONE navigate"
],
"generated": 7,
"thinking": 0,
"inference_seconds": 0.7151282919803634,
"receipt_sha256": "b1e4632555bd70819b5b05162b66bb945dcc83480914ac92f79ed9945dd0df90",
"recording_sha256": "916a14f93614ba687b3cb27bd38f4b31af8cb6ecdc6ef1762e6774af7e68cc51"
}
},
{
"checkpoint": "pending-native-stop",
"task": 62,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping_admin",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 76.96602091705427,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"5",
"C5",
"C5",
"C5",
"C5",
"C5",
"C5",
"C5",
"C6",
"C8",
"C24",
"F17 2020-01-01"
],
"generated": 2379,
"thinking": 2161,
"inference_seconds": 70.3279061667854,
"receipt_sha256": "0a3e878d66f833d6be4e41d7b163ee16a444ec91c9b088d6b90250fc8c0cabad",
"recording_sha256": "1f998482e8b079e22bafd05f3fdfabdbdac79cad7e6c65e9dd23b40c23d7d2ed"
},
"edited": {
"seconds": 1.008508124970831,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "09a222296da27d31acd5d25a543063c395fb74560f8db61dda5ed4946110cc2c",
"commands": [
"DONE navigate"
],
"generated": 7,
"thinking": 0,
"inference_seconds": 1.005628832965158,
"receipt_sha256": "1bea02762e10cecab560e7f1b1de4df9a9d7e92338cf00ef1c1a07e52e2e887c",
"recording_sha256": "069d872e8fa980b43a8410fdcef1ed6a48b6f6d224b5e141659fee79719f7eb8"
}
},
{
"checkpoint": "complete-stop",
"task": 190,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 8.734988222647253,
"original": {
"seconds": 45.33590604097117,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C27",
"",
"ANSWER [65.32]"
],
"generated": 1555,
"thinking": 1509,
"inference_seconds": 42.99088237492833,
"receipt_sha256": "b944a6bd099412a0bad414e64b4990c7b99d8d9c25c191cd040399511f710f3e",
"recording_sha256": "942526c0551c329e1ac489749aad074e1ef028e893eb28801167a6512d218e66"
},
"edited": {
"seconds": 5.190150791895576,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [65.32]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 2.92674979113508,
"receipt_sha256": "84cf20808be895d2e05f3534cac15b07c016dc676049a987a70ae08f0f108be2",
"recording_sha256": "7832813623fc7a3cf56fe92630aa5d7346ee07478b94d24bafffad56ced8b3fe"
}
},
{
"checkpoint": "complete-stop",
"task": 190,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 7.7460947913453335,
"original": {
"seconds": 46.58707808400504,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C27",
"",
"ANSWER [65.32]"
],
"generated": 1555,
"thinking": 1509,
"inference_seconds": 44.305321957101114,
"receipt_sha256": "9b47dcf6c674a568ceb0f8fbc4119dd16df576a41f3396b34dd249eb9b202307",
"recording_sha256": "cec595452188d8350b71db44e21484f22217c4e84e955a3c27844b664b879dec"
},
"edited": {
"seconds": 6.014266457990743,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [65.32]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 3.309025958995335,
"receipt_sha256": "8b729c4bcaea345f9744a11f42c1257d38ec3be17b1236f694e4de2bae22cc38",
"recording_sha256": "ad265854077e737e48522c06297e43afc05be25661d9a37e4a29a0fd0eba844a"
}
},
{
"checkpoint": "complete-stop",
"task": 190,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 0.7768753233595105,
"original": {
"seconds": 7.21626258303877,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C28",
"ANSWER [65.32]"
],
"generated": 662,
"thinking": 612,
"inference_seconds": 4.76776066608727,
"receipt_sha256": "8726d5aa918653cd9a022445555ba3f978836fefa0d160feac9b580ed0cd5372",
"recording_sha256": "fbb66bed1ce5064b2bf276daf9e310be8e8cf82473c4397f03cd3e66eac6f37a"
},
"edited": {
"seconds": 9.288829708006233,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C28",
"ANSWER [65.32]"
],
"generated": 1034,
"thinking": 767,
"inference_seconds": 7.138767582830042,
"receipt_sha256": "8837ce095ebbf98d48c0c2cf246993515e50042e85d13ced316f91a1c1cd3662",
"recording_sha256": "b0e9e487c12576311698b8e8efb727745b3c5abb98b3dcb6004591524c4f4699"
}
},
{
"checkpoint": "complete-stop",
"task": 190,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 1.3608271132567153,
"original": {
"seconds": 12.153708999976516,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C28",
"ANSWER [65.32]"
],
"generated": 662,
"thinking": 612,
"inference_seconds": 4.9256642068503425,
"receipt_sha256": "2c7a4e2807c73cd87a088b3d20e2d958db61fed9c14b8317980503920b3be701",
"recording_sha256": "24915a2e17337a0b0d87f7cfcdc83f58cd02e4c8d9b541a6811426a06e940a17"
},
"edited": {
"seconds": 8.931119083077647,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C28",
"ANSWER [65.32]"
],
"generated": 1034,
"thinking": 767,
"inference_seconds": 6.83465154201258,
"receipt_sha256": "72fc0400d00b8206a2d5ed97e3056e362a4f7c0d05a0e932d414b6ae97611a00",
"recording_sha256": "bc1d171a325af34b00b81ba751c1d2f23b8a0a8b8ac5d6496264456aedbe319c"
}
},
{
"checkpoint": "complete-stop",
"task": 191,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 11.728927165968344,
"score": 0.0,
"grader_status": "failure",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"1",
"1",
"1",
"1 a",
"1 a",
"1 a 1",
"1 a 1",
"1 a 1",
"1 a",
"C 1",
"C 28",
"D"
],
"generated": 1240,
"thinking": 1080,
"inference_seconds": 9.507527501089498,
"receipt_sha256": "a50856d15bb6364982e3dee40947266669790d0829db39fb9c3e0d07533d3603",
"recording_sha256": "cb08030636da33fcc3620fc94101b555aa61fc8f832e2382cacf13b30e318041"
},
"edited": {
"seconds": 3.0059442919446155,
"score": 0.0,
"grader_status": "error",
"device": "RTX 5090",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C28",
"ANSWER [\"$365.42\"]"
],
"generated": 30,
"thinking": 0,
"inference_seconds": 0.9276975840330124,
"receipt_sha256": "742867ddf6f8b75053dfe21966d372805bb4310d938dfd5828f9d8f41216d479",
"recording_sha256": "cbd08257ce72a112f677b7486daefa30e143b6a8c91b166a5e0f127f6ec55549"
}
},
{
"checkpoint": "complete-stop",
"task": 191,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 9.138379249954596,
"score": 0.0,
"grader_status": "failure",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"1",
"1",
"1",
"1<click>",
"1<click>",
"1<click>",
"1<click> 1",
"1<click>",
"1<click>",
"1<click>",
"1<click>",
"1<click> 1"
],
"generated": 1056,
"thinking": 858,
"inference_seconds": 8.987814538180828,
"receipt_sha256": "37a6b8b838075a84424405af6f22d02d5f23e13b8f30b8afbf9b65e145da7029",
"recording_sha256": "c6c80ac2530dd3a6f5556c5c1eda08687d602a388c1c4b179c295fe3a7ad5992"
},
"edited": {
"seconds": 2.7042575840605423,
"score": 0.0,
"grader_status": "error",
"device": "RTX 5090",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C28",
"ANSWER [\"$365.42\"]"
],
"generated": 30,
"thinking": 0,
"inference_seconds": 0.6422569160349667,
"receipt_sha256": "ed3a93933827ae679c54f7d586de41e62783e48599d4673cac1ef35254778c42",
"recording_sha256": "4f7e01a2274723d5fd21a9642fffe945beff254d9fdf3a980b29b33ed3873561"
}
},
{
"checkpoint": "complete-stop",
"task": 275,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 0.7906818639505235,
"original": {
"seconds": 4.807198666036129,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 xbox",
"C9",
"DONE navigate"
],
"generated": 315,
"thinking": 288,
"inference_seconds": 2.4517232079524547,
"receipt_sha256": "bda878fa25ef3d346c5c56d56854475ec23c5176097057290769c47a615edca2",
"recording_sha256": "1d2e68f8a990e8209ded2fc6c8b965a3b472fc3cdf32455e8d4620f0b8e5871a"
},
"edited": {
"seconds": 6.079814000055194,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"F 7 xbox",
"C 9",
"DONE navigate"
],
"generated": 528,
"thinking": 358,
"inference_seconds": 3.751093417056836,
"receipt_sha256": "b36a2386bd99b5bcce06fd963cd5e114add5e373323cb4948fff505f4311ddf2",
"recording_sha256": "0c9e5d7de9282572e6d465d9cd9cf88f3ad32d5029665dd3e5ad3c250773a53c"
}
},
{
"checkpoint": "complete-stop",
"task": 275,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 0.8236512005633566,
"original": {
"seconds": 5.00892183301039,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 xbox",
"C9",
"DONE navigate"
],
"generated": 315,
"thinking": 288,
"inference_seconds": 2.400264916010201,
"receipt_sha256": "e2ddcc46e4c061c79c9d2669ecbff0f6ccd069a3fa895273916cdefa9a5dd614",
"recording_sha256": "90909532e5e7a03ef11e431f670b650c64adc4b87ed6173b91b4474a451e325d"
},
"edited": {
"seconds": 6.081362874945626,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"F 7 xbox",
"C 9",
"DONE navigate"
],
"generated": 528,
"thinking": 358,
"inference_seconds": 3.70308424893301,
"receipt_sha256": "3de8aefd1e2fcce1fec46d94a4e3498444550b0c77062e94d219e6db5f1902d4",
"recording_sha256": "d126217723fea7e12c378ec0e931a570d40178157550c328b307bbb8b58a935e"
}
},
{
"checkpoint": "complete-cross",
"task": 190,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 9.333173334016465,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"",
"ANSWER [65.32]"
],
"generated": 1104,
"thinking": 1060,
"inference_seconds": 7.393800664925948,
"receipt_sha256": "583e40d83a2dae419c463e1a58a2f47de180396ad80763f32dbbbf3a55d52fe7",
"recording_sha256": "ca72633366aff0727d18d68f0fdb00af50b53630d994725e56f2055f9006fbb9"
},
"edited": {
"seconds": 3.2936403329949826,
"score": 0.0,
"grader_status": "failure",
"device": "RTX 5090",
"shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889",
"commands": [
"C1",
"C28",
"DONE navigate"
],
"generated": 19,
"thinking": 0,
"inference_seconds": 1.0277085829293355,
"receipt_sha256": "c9c7fe2cf441982418f578852a90317ed712a0ff675a75864085f40825ccd304",
"recording_sha256": "8d5ee987796266a46304fc15dffd7e24e748c345cc6dae4bf3904712d795bba6"
}
},
{
"checkpoint": "complete-cross",
"task": 190,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 12.414950583013706,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"",
"",
"ANSWER [65.32]"
],
"generated": 1545,
"thinking": 1499,
"inference_seconds": 10.28308187611401,
"receipt_sha256": "28dfb5fe4aa6758b8ce48d99869af6bececc54431280a23781aa6bdbfbf42a78",
"recording_sha256": "6cc9161e3fe28c6f4770eae3dd5bf7adf923cff8c8aa09e8912a1e377b51e79f"
},
"edited": {
"seconds": 3.1889395409962162,
"score": 0.0,
"grader_status": "failure",
"device": "RTX 5090",
"shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889",
"commands": [
"C1",
"C1",
"C28",
"ANSWER [762.18]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 0.8131322080735117,
"receipt_sha256": "82e5d2e9d7890ec8db155b8d402d9ea526073957c4b18cebd21a38c8211f48ac",
"recording_sha256": "aefcbb784438eec64ab97f93fcb26c2fdc527ca9176d01a8a1c36aece794a043"
}
},
{
"checkpoint": "order-short-margin",
"task": 190,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 0.8425275440700961,
"original": {
"seconds": 5.717545500025153,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C27",
"ANSWER [65.32]"
],
"generated": 470,
"thinking": 428,
"inference_seconds": 3.4925888748839498,
"receipt_sha256": "3d27fc260c6451f5bf4d262375982a9e3405da0f0e8f5101bd0d2e76e6e51450",
"recording_sha256": "153ddba796ec0cea3348bdf7bd04db3f859fd799f9da48a1cc94fc3fc1efc293"
},
"edited": {
"seconds": 6.7861822919221595,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "cf2d48bcddb8952fbb415293b1155b17f2941562d664d290b2d7a0c75944422f",
"commands": [
"C1",
"C28",
"",
"DONE",
"ANSWER [65.32]"
],
"generated": 540,
"thinking": 510,
"inference_seconds": 4.444257041905075,
"receipt_sha256": "0e182901fc852d33d93ece0a897f195fc1a606c6355dff4cf78cdd7ff0543324",
"recording_sha256": "0ea83acd758f6e9fc7994e04609925d6a5d10954abfe3104107b66f6590eac2f"
}
},
{
"checkpoint": "order-short-margin",
"task": 190,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 3.0895748438194923,
"original": {
"seconds": 10.326355000026524,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"",
"ANSWER [65.32]"
],
"generated": 1181,
"thinking": 1137,
"inference_seconds": 7.871952000074089,
"receipt_sha256": "1c087c5814f1b002e1ea9377eda5ae25767e0362adbf324db50698ae35aa14bd",
"recording_sha256": "75627feca13b116dd5c8711c21396b0ad8ba92904f6d6985cfdcdff2e39128dd"
},
"edited": {
"seconds": 3.342322333017364,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "cf2d48bcddb8952fbb415293b1155b17f2941562d664d290b2d7a0c75944422f",
"commands": [
"C1",
"C28",
"ANSWER [65.32]"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 1.1054121260531247,
"receipt_sha256": "57474389b1de11eecd972d4151cf8e8887e673c33bb669705efa6b8966067709",
"recording_sha256": "05bf4f59c5b86c85d42dd5b4919c9a63d0d52aea1c6bfbfb93ca0e322f79aee9"
}
},
{
"checkpoint": "order-history-route",
"task": 190,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.0347174710697638,
"original": {
"seconds": 9.880964207928628,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"",
"ANSWER [65.32]"
],
"generated": 1181,
"thinking": 1137,
"inference_seconds": 7.881787376129068,
"receipt_sha256": "372d90d6141a4dda6107d86ae775038e2e0a64fd8ca2284a27a703c3a4916722",
"recording_sha256": "f79074bb7f4cb544f66fa3f312794508f1c5f990eea5afa581dcbc188ea95735"
},
"edited": {
"seconds": 4.856184875010513,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073",
"commands": [
"C1",
"C28",
"ANSWER [65.32]"
],
"generated": 335,
"thinking": 252,
"inference_seconds": 2.784148833132349,
"receipt_sha256": "18dc25332d1bccbbd06e9769cebcbc6f3a9d93369113dacfc94b3d63af488656",
"recording_sha256": "ad50979b258cd8f5d3980dbe720beaa013787474a9b4c4968bcee86d6b1f9bd5"
}
},
{
"checkpoint": "order-history-route",
"task": 190,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.1755407560053266,
"original": {
"seconds": 10.092377167078666,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"",
"ANSWER [65.32]"
],
"generated": 1181,
"thinking": 1137,
"inference_seconds": 7.956198376021348,
"receipt_sha256": "d80a54f157029a2fdf18c152cf4c9be428055a5574ecf70100a031d63422eb85",
"recording_sha256": "10264730f938c44c0831997391e4afcfea057cba7c97f31043653d8e58856865"
},
"edited": {
"seconds": 4.63902004098054,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073",
"commands": [
"C1",
"C28",
"ANSWER [65.32]"
],
"generated": 335,
"thinking": 252,
"inference_seconds": 2.5504851238802075,
"receipt_sha256": "0c0427e0d66d742d46459dde8b9b9cbeeb7744057b831a49ef28fe71d636c16a",
"recording_sha256": "34d4f7f84ec4a464b2cf9ad45dc74950ecf4177b43691f46aa0d71bdc3512397"
}
},
{
"checkpoint": "order-history-route",
"task": 189,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.1573877246999222,
"original": {
"seconds": 6.301418249960989,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"ANSWER [754.99]"
],
"generated": 620,
"thinking": 578,
"inference_seconds": 4.256730875116773,
"receipt_sha256": "a931070b8d954ea26af76e8a472877b8810e7d4d165c4c9af63e8e16115f5cb1",
"recording_sha256": "2038aa9c1016a75aef04d502a4f790f76259f27ebbffb86ddb532c755dc4b5cf"
},
"edited": {
"seconds": 2.9208557079546154,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073",
"commands": [
"C1",
"C28",
"ANSWER [754.99]"
],
"generated": 30,
"thinking": 0,
"inference_seconds": 0.7570810420438647,
"receipt_sha256": "27ac1dc9318bc5e1026f478030af5934d3c8ebd9f3d274f6e895e847c254a409",
"recording_sha256": "710d46d01844b0323c393730ca915aff375b00f8b7fd642fed6c303398e2b058"
}
},
{
"checkpoint": "order-history-route",
"task": 189,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.3613654577613685,
"original": {
"seconds": 6.77029658399988,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"ANSWER [754.99]"
],
"generated": 620,
"thinking": 578,
"inference_seconds": 4.633256333065219,
"receipt_sha256": "0cea03e6cc7571e55dc7bf0fcfdae72c74474be28a708ed8c8a8ef89d405058b",
"recording_sha256": "4a5813e8ccab420942171b54d76c3c8c6979f2238075a2f1615da63647d27d5f"
},
"edited": {
"seconds": 2.8671108750859275,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073",
"commands": [
"C1",
"C28",
"ANSWER [754.99]"
],
"generated": 30,
"thinking": 0,
"inference_seconds": 0.6372076660627499,
"receipt_sha256": "38994d8c0ccc902ff2a20f85752d8996348439c8f2de2fed16ac66e0e130ccb8",
"recording_sha256": "77e0dd320302f1be3b54c6052cb774f2f3cf67981508c2f3bb041138002d4d0c"
}
},
{
"checkpoint": "order-history-route",
"task": 188,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.2340988253506344,
"original": {
"seconds": 6.307632959098555,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 636,
"thinking": 601,
"inference_seconds": 4.462099457974546,
"receipt_sha256": "ece2e1ff126dfe2e79cc898f9f0abc9900ed6316e2361251acc06394b4c94712",
"recording_sha256": "cbe8a8253a5e729703f3435b4472ae9bf8505dd6ba2c1d1a0440a70790115ef3"
},
"edited": {
"seconds": 2.823345542070456,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 0.9109670828329399,
"receipt_sha256": "cfd52c7f0d070ccf4bcdb41e61353929243173510fd17ef2816172a36587c3bc",
"recording_sha256": "04ff5295e15c53889a00aee652cf12e12368d270a7419b1c03f86c7f2dfe4dcc"
}
},
{
"checkpoint": "order-history-route",
"task": 188,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 3.526484137272057,
"original": {
"seconds": 9.687184332986362,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 1077,
"thinking": 1026,
"inference_seconds": 7.271809124969877,
"receipt_sha256": "cdc35d8bbdee31f92e5940f640e5d65584e4fa5f1d7dfa23d48c722cc3aa7506",
"recording_sha256": "408df306a04a38c47902fd374b9fea8cea2a379b66a0fae370f18059428a7d3f"
},
"edited": {
"seconds": 2.7469808330060914,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 0.6333339589182287,
"receipt_sha256": "b8fa575755735d911468f7fe35395a5c56e0954cb79df2458b9f415b252433fe",
"recording_sha256": "da565e7526d5613ac7b4a6405214591cd8b2b706fd9bc474830627469f8fce1d"
}
},
{
"checkpoint": "order-history-route",
"task": 191,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 8.418479042011313,
"score": 0.0,
"grader_status": "failure",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"1",
"1",
"1",
"1<click>",
"1<click>",
"1<click>",
"1<click> 1",
"1<click>",
"1<click>",
"1<click>",
"1<click>",
"1<click> 1"
],
"generated": 1056,
"thinking": 858,
"inference_seconds": 8.246465002186596,
"receipt_sha256": "7e8cbc36e5095c7df6a6b54ff0165fb76a83d8909bc1074d4beb222f537788c9",
"recording_sha256": "b5e52b0d037f58be3f3f5d3e33fa4c61577a57ccf71cb8486209c076e4e95a69"
},
"edited": {
"seconds": 4.19265154202003,
"score": 0.0,
"grader_status": "error",
"device": "RTX 5090",
"shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073",
"commands": [
"C1",
"C1",
"C28",
"The page shows orders with statuses Canceled,",
"ANSWER [\"$754.99\"]"
],
"generated": 70,
"thinking": 15,
"inference_seconds": 1.5621499170083553,
"receipt_sha256": "f515e9de11c72ea33e3bbd84e2f2acc402998968fee8eaf8d35462f8924a6ad7",
"recording_sha256": "e947f45bcea752e6b8ac792fe0b8e9b1a188311839fcd822088f4524f5a4810f"
}
},
{
"checkpoint": "order-history-route",
"task": 191,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 12.566988083999604,
"score": 0.0,
"grader_status": "failure",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"1",
"1",
"1",
"1 a",
"1 a",
"1 a 1",
"1 a 1",
"1 a 1",
"1 a",
"C 1",
"C 28",
"D"
],
"generated": 1294,
"thinking": 1134,
"inference_seconds": 10.013543709064834,
"receipt_sha256": "5610b4c1e255ec71dba6390a839e52aa53632ca3e5eb94e0a80acb4e2e749b57",
"recording_sha256": "850cad11095323b921fe44fd8a50487cbbc3b1a51a312664f688d0dc073e0ab4"
},
"edited": {
"seconds": 11.337048665969633,
"score": 0.0,
"grader_status": "error",
"device": "RTX 5090",
"shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073",
"commands": [
"C1",
"C28",
"C1",
"C28",
"DONE",
"DONE",
"ANSWER [\"$365.42\"]"
],
"generated": 1076,
"thinking": 569,
"inference_seconds": 7.667066333931871,
"receipt_sha256": "01307ed3f7cee0eed3d9d915eb039beba143a5f55f3b9a566210894b2ee37574",
"recording_sha256": "bee98dbd6fa6fbfcf54d871edf97d374285878291a7c49d1420bada18ed0d17e"
}
},
{
"checkpoint": "order-history-route",
"task": 275,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 0.7902392212209556,
"original": {
"seconds": 5.020188458962366,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 xbox",
"C9",
"DONE navigate"
],
"generated": 315,
"thinking": 288,
"inference_seconds": 2.458598250988871,
"receipt_sha256": "61fb1b50bc29906ad350fd24a1b1e15ebed4a36b679e61d1f06a5f89ed9c963b",
"recording_sha256": "14429436b695064bb95a4db5ad46cd2f1c92e6a2283c2aa9175d0cdb19be67fd"
},
"edited": {
"seconds": 6.352745249983855,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073",
"commands": [
"F 7 xbox",
"C 9",
"DONE navigate"
],
"generated": 528,
"thinking": 388,
"inference_seconds": 3.990014291019179,
"receipt_sha256": "8ef5a9dda6679658b3549adf62a04655d89ca128503e539deab907b65d214294",
"recording_sha256": "524f90fa5fa0edb6001e43e3e61ed5b73c053ba1139d7b494a0ff7eb185c340e"
}
},
{
"checkpoint": "order-history-route",
"task": 275,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 0.7715830968978724,
"original": {
"seconds": 4.637832708074711,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 xbox",
"C9",
"DONE navigate"
],
"generated": 315,
"thinking": 288,
"inference_seconds": 2.3957099590916187,
"receipt_sha256": "f131cf000fc72ff2f1759f348c05098a0ff22b4623c2cb559f9419f71a8c2aff",
"recording_sha256": "3f1dae74ea90990353d351262ee52ad0bcb41d6554169763dc5d9f6490edcbeb"
},
"edited": {
"seconds": 6.010801333934069,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073",
"commands": [
"F 7 xbox",
"C 9",
"DONE navigate"
],
"generated": 528,
"thinking": 388,
"inference_seconds": 3.70716108311899,
"receipt_sha256": "7915ce1de6a4f4d67a5e2d30d84fdf7ce9a03380f8c3d76cea2333f9222268fc",
"recording_sha256": "805f68103cbd9dc63cc61d40fb3c97b2847d91174103f8585ca826bf630d5978"
}
},
{
"checkpoint": "search-submit-stop",
"task": 275,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 1.4533256825965108,
"original": {
"seconds": 4.781691708951257,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 xbox",
"C9",
"DONE navigate"
],
"generated": 315,
"thinking": 288,
"inference_seconds": 2.4454436249798164,
"receipt_sha256": "2cf58128d38a9db061d9749db7e3bd1811c0832b99b45036c81103a99ac695b1",
"recording_sha256": "da5153ef7157b25aaccf66b9579b58c5a49882b70e08d00eb8c216b7543d09fa"
},
"edited": {
"seconds": 3.290172165958211,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1",
"commands": [
"F 7 xbox",
"C 9",
"DONE navigate"
],
"generated": 23,
"thinking": 0,
"inference_seconds": 0.8812786679482087,
"receipt_sha256": "d2eeef9872f3686a34d77f90bc36122ff3c5c16d69817ff200bb894d0dad016f",
"recording_sha256": "3c5b0de9c51ba3e024bbe3c6dd21227860b6feddfbb03d4e8de50d17e7e245ea"
}
},
{
"checkpoint": "search-submit-stop",
"task": 275,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 1.582510684555791,
"original": {
"seconds": 4.751382500049658,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 xbox",
"C9",
"DONE navigate"
],
"generated": 315,
"thinking": 288,
"inference_seconds": 2.456910749897361,
"receipt_sha256": "1ed81a2acf8fc1c1f74a90b325579f1692c73339bcc2581c8f8484c2de7c0887",
"recording_sha256": "3d8f81550117b42ee36ada4ccd70dff5808fd4c6c9bf9685eb305daa60e4ffa7"
},
"edited": {
"seconds": 3.002433125046082,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1",
"commands": [
"F 7 xbox",
"C 9",
"DONE navigate"
],
"generated": 23,
"thinking": 0,
"inference_seconds": 0.6660729580326006,
"receipt_sha256": "05fbd793a01633e453f840b53c32ce89d0b97515fcd68aced3b9bdf24de01554",
"recording_sha256": "993a915623f7851110f0178c347bd5ae29ad90318f9e9c1c9ace7c0bf53aff2a"
}
},
{
"checkpoint": "search-submit-stop",
"task": 190,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 3.2633434005578077,
"original": {
"seconds": 9.295178166008554,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"",
"ANSWER [65.32]"
],
"generated": 1104,
"thinking": 1060,
"inference_seconds": 7.403692627092823,
"receipt_sha256": "10e113f940c9e247fd8334c79dbad2180d4601a67a2dc440581a33b0dd942ed8",
"recording_sha256": "3cbd6436db955b45efb52b2295633470ed96ded06e28bfaa7a7d9a23acadee81"
},
"edited": {
"seconds": 2.8483604160137475,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1",
"commands": [
"C1",
"C28",
"ANSWER [65.32]"
],
"generated": 25,
"thinking": 0,
"inference_seconds": 0.6726702089654282,
"receipt_sha256": "2944f50cd972d2d335dead10baae03824517bccd51345bb5254e87b397ecbb44",
"recording_sha256": "b6aa451ecac294a4142363a3ebec5555bda2325574d81e31a24ab89d071345d0"
}
},
{
"checkpoint": "search-submit-stop",
"task": 190,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.0244892763429303,
"original": {
"seconds": 5.566456000087783,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C27",
"ANSWER [65.32]"
],
"generated": 470,
"thinking": 428,
"inference_seconds": 3.4088242081925273,
"receipt_sha256": "b65e08b040c60338431603e3510aa875d20f73162388d02daa66264fa298102e",
"recording_sha256": "07bffc2c0ddb18728b97dca662fe5b7a1b47939140b1fa16384018f5dce26283"
},
"edited": {
"seconds": 2.74956062505953,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "bbc639c4b54b0b748b011b9289512ccdce4c95298a4c19056d1b73a2a654e1d1",
"commands": [
"C1",
"C28",
"ANSWER [65.32]"
],
"generated": 25,
"thinking": 0,
"inference_seconds": 0.6097358749248087,
"receipt_sha256": "5e9af560c75070b430256e4520921dd3c1989febe05ad211799c8d752aa44650",
"recording_sha256": "117012b2166e80a118d3f091da7391a5fbeb5b4a619a2b8a72ac89b4c3df5798"
}
},
{
"checkpoint": "order-history-route",
"task": 189,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 4.57448292218901,
"original": {
"seconds": 24.47346133296378,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [754.99]"
],
"generated": 758,
"thinking": 701,
"inference_seconds": 22.08682024909649,
"receipt_sha256": "4bd09806c228658ab91f939594cf2394cbc6a88ae34782ce77d31b3fe1a67e76",
"recording_sha256": "2b9b390beb267c67ef41ce4fed6b5896d46cae845ecb7d0410b58873ec2ff49b"
},
"edited": {
"seconds": 5.3499951249687,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [754.99]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 2.9966340420069173,
"receipt_sha256": "2b0995cf755f61451029ee24ada6b0a5674e282ac40d3ba4a86addaa3e575671",
"recording_sha256": "bc0f54dce5dc9be7ef20fff1a96e6aec22b02998c7e389d82b4707a07e59e197"
}
},
{
"checkpoint": "order-history-route",
"task": 189,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 10.68081008293666,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"DONE navigate"
],
"generated": 291,
"thinking": 245,
"inference_seconds": 9.320337167009711,
"receipt_sha256": "82989d6377e02629fa1f2dffe1c95a0d30d76c8c6f6c5f2b66e596ea09ea4724",
"recording_sha256": "0715a986758d86297fae6490a51e76d22a8c52d49517c9c12046f4c47e967d80"
},
"edited": {
"seconds": 5.62365950003732,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [754.99]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 3.3328368751099333,
"receipt_sha256": "7d03ccb179350b096228d2aca8f222a62f8dea960520a3457d204a8488717404",
"recording_sha256": "662b69db5b7489e98cba75cf23d11ba1bcfca6623a14577a9be52851b69b267d"
}
},
{
"checkpoint": "order-history-route",
"task": 190,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 6.463688805082036,
"original": {
"seconds": 33.8939617080614,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C28",
"",
"ANSWER [65.32]"
],
"generated": 1114,
"thinking": 1062,
"inference_seconds": 31.489520958042704,
"receipt_sha256": "33da861ce0d55a2e50c046dbbc47e5a4b4c84c732ee34703f5b5d4f0c2fa3ccd",
"recording_sha256": "93d3cd7fe6d23f2c78f4b3ca84205702795e2b2b4edd9f2cb71a280ff2690080"
},
"edited": {
"seconds": 5.243749000015669,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [65.32]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 2.925669873948209,
"receipt_sha256": "8ec829841ea50b195c871e4cfbaccb0a5f6b776aa256b8a8c4601c67f825bb48",
"recording_sha256": "5542b2d9a3e08d20326cc8b0f3aaa648dee8e353bf68825a39747ee9fe79274b"
}
},
{
"checkpoint": "order-history-route",
"task": 190,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 8.187736102644966,
"original": {
"seconds": 47.179218083037995,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C27",
"",
"ANSWER [65.32]"
],
"generated": 1555,
"thinking": 1509,
"inference_seconds": 43.978050000965595,
"receipt_sha256": "99f4d7ef78dd466de0df077ee455cba9f3e8cd3517bc0a94ef33ff6f8aa51cc8",
"recording_sha256": "4974c4c2a4452b1874bee80a103ad38b77124bc03ddd05c1cacd41603c8e6258"
},
"edited": {
"seconds": 5.762181082973257,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "a393cbbe28eb0b830b7f9cbf00f4b7df4b85da74b62b7aa5aa20df78f9d4a073",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [65.32]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 3.2434727910440415,
"receipt_sha256": "4ee31a78c56726cb5e325af372acb7cee4ba4d7894ed7ae0d72e06ab645423e1",
"recording_sha256": "638a31430f6f8abb2169b4b5b087cdefe93d800f4cde76836b2df959cb35f3cc"
}
},
{
"checkpoint": "search-focus-route",
"task": 283,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 87.93015045800712,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 Xbox controller",
"K",
"K",
"C9",
"",
"D",
"C3",
"F7 Xbox controller",
"K",
"C9",
"D",
"D"
],
"generated": 2766,
"thinking": 2596,
"inference_seconds": 80.11356908199377,
"receipt_sha256": "2c450dd02c237f79c8c5162b8916d7a597dd137a771a903101909ff3ff4dba83",
"recording_sha256": "e5e09a023862edf3033be65c4a8c22306acd87f4881cf6076507ae184f138943"
},
"edited": {
"seconds": 18.76859758410137,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "6238b11e7bdac91f63ee8c7ead04730f22fe2fe2d24e9d54a29a1da161109964",
"commands": [
"F 7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller"
],
"generated": 109,
"thinking": 0,
"inference_seconds": 9.257374460343271,
"receipt_sha256": "5ddc2381679f9010a0cbe1497c08631418a24110efeba721a68207316efb1100",
"recording_sha256": "75373c42203df67877ff8e69cb6111aa5b86498715a0a2d75a1c035b6e242d12"
}
},
{
"checkpoint": "search-focus-route",
"task": 283,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 91.99072962498758,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 Xbox controller",
"K",
"K",
"C9",
"",
"D",
"C3",
"F7 Xbox controller",
"K",
"C9",
"D",
"D"
],
"generated": 2766,
"thinking": 2596,
"inference_seconds": 84.98011841706466,
"receipt_sha256": "daa117518b5d3a27c71bb6a00dcd118399deb9a811c7e2051561ca3773f2fdb4",
"recording_sha256": "2106edb0d09e042d0250aee1f1c1e02db44da8a1ec68a7f3899fab94912eeac8"
},
"edited": {
"seconds": 19.445797957945615,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "6238b11e7bdac91f63ee8c7ead04730f22fe2fe2d24e9d54a29a1da161109964",
"commands": [
"F 7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller"
],
"generated": 109,
"thinking": 0,
"inference_seconds": 9.901397418114357,
"receipt_sha256": "647615c6e9794bde47330ac66cffd6b573fc13891a9c8621dcde44bda44a8657",
"recording_sha256": "604d6845df54d958fa87ba6b25f494f7b52de11f854103a28130fd5b73ee1db4"
}
},
{
"checkpoint": "search-state-route",
"task": 283,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 83.21020058402792,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 Xbox controller",
"K",
"K",
"C9",
"",
"D",
"C3",
"F7 Xbox controller",
"K",
"C9",
"D",
"D"
],
"generated": 2669,
"thinking": 2499,
"inference_seconds": 76.3192295411136,
"receipt_sha256": "ee4856c05d0ff4d0770283c305adda801af5687b4b832386d59b49b5b08d1d47",
"recording_sha256": "7e5531f2ff6b397792d722269290804638dd8557eca6c3ff55419a5579ecf7cf"
},
"edited": {
"seconds": 35.93294025002979,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "fbdc9f126dca8a7a5f12d4e120ab44138ad9b5226b18071811cf1dbab2c2ee2f",
"commands": [
"F 7 Xbox controller",
"C 9",
"C 24",
"C 24",
"C 24",
"F7 Xbox controller",
"F7 Xbox controller",
"C 9",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller"
],
"generated": 619,
"thinking": 340,
"inference_seconds": 23.04296320991125,
"receipt_sha256": "f407596f9683e3096d782dce5c1efd674c319e14bb841f544b1f681f2b5369e7",
"recording_sha256": "3be4a09bcb12e77dfbdc95c936122164911447bd4d32d10485dd05cca6023587"
}
},
{
"checkpoint": "search-state-route",
"task": 283,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 83.12202070804778,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 Xbox controller",
"K",
"K",
"C9",
"",
"D",
"C3",
"F7 Xbox controller",
"K",
"C9",
"D",
"D"
],
"generated": 2669,
"thinking": 2499,
"inference_seconds": 76.33653653995134,
"receipt_sha256": "14c8b35f16242370748001daf5d2edb9f34bd0a85a767327d17ab406cb484020",
"recording_sha256": "fea0ffd4f68222c863169530e7ac9e5a41a42f6fd49a440987ffd9ea84fe3d1e"
},
"edited": {
"seconds": 36.30528724996839,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "fbdc9f126dca8a7a5f12d4e120ab44138ad9b5226b18071811cf1dbab2c2ee2f",
"commands": [
"F 7 Xbox controller",
"C 9",
"C 24",
"C 24",
"C 24",
"F7 Xbox controller",
"F7 Xbox controller",
"C 9",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller",
"F7 Xbox controller"
],
"generated": 619,
"thinking": 340,
"inference_seconds": 23.61733170680236,
"receipt_sha256": "fab4988369b541c5f5fec5f4a693883568a87998f63c44674c9d8dd921cad58d",
"recording_sha256": "27a569e67037f1f26cb58c9058144de3d15ad5fb43cf4949fcafe1c96dd8d279"
}
},
{
"checkpoint": "search-product-guard",
"task": 283,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 87.58633945893962,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 Xbox controller",
"K",
"K",
"C9",
"",
"D",
"C3",
"F7 Xbox controller",
"K",
"C9",
"D",
"D"
],
"generated": 2669,
"thinking": 2499,
"inference_seconds": 79.38518158299848,
"receipt_sha256": "9b65b62e25ac44058102f06c2d942d6126a77631c3ce56fbbd251db947e7ee9b",
"recording_sha256": "34cbd3e8dbd7a2e06aba936385ed06dc098f249606cf3dbc4abc27cebab3d2dc"
},
"edited": {
"seconds": 7.486511458992027,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "59a63e7b5bf01cf1e30d15bae19160bc87cfbc8817bb76f8d807b67659468d76",
"commands": [
"F 7 Xbox controller",
"C 9",
"D",
"DONE navigate"
],
"generated": 44,
"thinking": 0,
"inference_seconds": 3.6446031250525266,
"receipt_sha256": "ecb4338098a011c1e62da6714b102055d2cb79f940f14c814134c395fa0410ea",
"recording_sha256": "a9a065d9141c9c0330e4508a9b97e3394ffb043b9740630670f2ae6dea8b0ba2"
}
},
{
"checkpoint": "search-product-guard",
"task": 283,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 87.25159720901866,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 Xbox controller",
"K",
"K",
"C9",
"",
"D",
"C3",
"F7 Xbox controller",
"K",
"C9",
"D",
"D"
],
"generated": 2669,
"thinking": 2499,
"inference_seconds": 77.33748512610327,
"receipt_sha256": "d66762f6419ae58da4788db933ed3f3b75bfa6e1c51a56cccfa597e6f85d83a4",
"recording_sha256": "ef8cf7aad893bab833b8ce5c4ff490a8a3c4bf0bbad712147b988a687f30e1cd"
},
"edited": {
"seconds": 6.887347457930446,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "59a63e7b5bf01cf1e30d15bae19160bc87cfbc8817bb76f8d807b67659468d76",
"commands": [
"F 7 Xbox controller",
"C 9",
"D",
"DONE navigate"
],
"generated": 44,
"thinking": 0,
"inference_seconds": 3.9920278328936547,
"receipt_sha256": "ac5ec0fc2c018d744c6ebf943c011d7df703521f975f59fb9c5f34d1ff831899",
"recording_sha256": "f38bb011e9cfc41516b02d03ca455f613c7b2e3002a10e896623e6b7d727122f"
}
},
{
"checkpoint": "complete-cross",
"task": 188,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 5.139163248802451,
"original": {
"seconds": 29.21747625002172,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C 1",
"C 1",
"C 27",
"ANSWER [365.42]"
],
"generated": 919,
"thinking": 871,
"inference_seconds": 26.511262458050624,
"receipt_sha256": "6bb3d9cee3f2c29dfa6f1f3365a14cf3d6b9fe8889806c3723f34f7b10027e3a",
"recording_sha256": "6820017ac3012088d72b67ce71a3033a00c99080ab8c9d7084260614f23acc14"
},
"edited": {
"seconds": 5.685259415884502,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [365.42]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 3.07812479196582,
"receipt_sha256": "28b77c094267bdbd487300f980ab913649643201e4c1fda25e47f1e724f94d5a",
"recording_sha256": "8f6003418507a7b980e15c97ccc252482dab2e4b06a055983fcc67aee44bdf2f"
}
},
{
"checkpoint": "complete-cross",
"task": 188,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 5.7143168900475185,
"original": {
"seconds": 28.377125291968696,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C 1",
"C 1",
"C 27",
"ANSWER [365.42]"
],
"generated": 919,
"thinking": 871,
"inference_seconds": 25.887863084091805,
"receipt_sha256": "3aaa8a25ffdaa732a1a70c677a37814f170166842ebcc159f1b5c686211fa36b",
"recording_sha256": "70a36ad97eea415680f681f942724a0f229e9f29900e0e17a5314dd9e687935a"
},
"edited": {
"seconds": 4.96596983296331,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [365.42]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 2.887187918066047,
"receipt_sha256": "83f983c722b2c509d99b5ad98494f59a2b16ad133e367b1e7b1570b3f2163f90",
"recording_sha256": "36b0c3d8a789ab14144424dbeaed9f28ad0339321b3aa2fa79361dd1655825fb"
}
},
{
"checkpoint": "complete-cross",
"task": 189,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 6.306722379373041,
"original": {
"seconds": 33.1291674580425,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C27",
"",
"ANSWER [754.99]"
],
"generated": 1077,
"thinking": 1018,
"inference_seconds": 30.86204899905715,
"receipt_sha256": "32a8f504c05053537034dfbb42a8e60a8001f905b80217e28e9d08bc37ef638c",
"recording_sha256": "1ea97126fa12b6406a6cb817782c3a92b22b0e8085a4c1ef4cb276bf0dab9e31"
},
"edited": {
"seconds": 5.252992832916789,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [754.99]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 2.9513221670640633,
"receipt_sha256": "13aaad04bde4944080119ae795b247c8d5da61c67a8946ae6d563b83e58ec72e",
"recording_sha256": "b1bf7f929afa1478e03a8d2aa16c2b69d4631cbcd4d8fb469b17e21bb969f8c9"
}
},
{
"checkpoint": "complete-cross",
"task": 189,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 6.068339618706268,
"original": {
"seconds": 34.832229208084755,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C27",
"",
"ANSWER [754.99]"
],
"generated": 1077,
"thinking": 1018,
"inference_seconds": 32.43203541578259,
"receipt_sha256": "783df0e7274659e7aa808deb99b0a8ffed543618fd66a713b181b872c89b8a89",
"recording_sha256": "dc4a0ea98c167baa7589d8a10d221e30f8a369ab3d923d212a775c7f71c31a5b"
},
"edited": {
"seconds": 5.739993374911137,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [754.99]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 3.319720249855891,
"receipt_sha256": "7db36f187e055594787561247a0bc3510c9f06c55eb2d0eb5fd971bee803e021",
"recording_sha256": "56aea5b1f1787d9126e3cf4f8ca1a33be80f5e67a218b7cdd8b34abe2f520e18"
}
},
{
"checkpoint": "complete-cross",
"task": 190,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 8.949829646056076,
"original": {
"seconds": 45.88679650006816,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C27",
"",
"ANSWER [65.32]"
],
"generated": 1555,
"thinking": 1509,
"inference_seconds": 43.03133266605437,
"receipt_sha256": "db7cc933937f077603d615ef906b8ccd107f4b80f9e4303af200341f55a6e809",
"recording_sha256": "001a63b86d63e645ef77f990d03c3f2b17ea4dc17c1d6a34cc91e3b4126bca25"
},
"edited": {
"seconds": 5.127113958005793,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [65.32]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 2.841448998893611,
"receipt_sha256": "de99cd3b9ffdab3dcde0a6ba3b7ab53c4f78c893b5e04a1a42cd729a32ca8816",
"recording_sha256": "1c0b7325f4c3119182342a5017acce4187ed18802c46dddd919c151140565054"
}
},
{
"checkpoint": "complete-cross",
"task": 190,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 8.004451022465457,
"original": {
"seconds": 44.66634520795196,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C27",
"",
"ANSWER [65.32]"
],
"generated": 1555,
"thinking": 1509,
"inference_seconds": 42.26506354298908,
"receipt_sha256": "2d22711d2245d3bef88a8fa25184d5bfe72fd77087c705569a64986334ea281a",
"recording_sha256": "8e416deffffb3369aa3ac35af4b788820ba18d6235a40210f91cb6c7d10ceb2e"
},
"edited": {
"seconds": 5.580188457970507,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [65.32]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 3.1810083750169724,
"receipt_sha256": "1a87c03f43328859a79397f0de0f6caafee859c476dcf9bf148a50d557a85f69",
"recording_sha256": "c0cb3455a04d52f9e57ac56207df5f399bc2dd5211a15b6229b9c0f938f74903"
}
},
{
"checkpoint": "complete-cross",
"task": 274,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 3.1075285239080728,
"original": {
"seconds": 14.461539416923188,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 usb wifi",
"C9",
"DONE navigate"
],
"generated": 377,
"thinking": 342,
"inference_seconds": 11.723945749923587,
"receipt_sha256": "97ab57fe1b7a619938b9644f5c97acec7e95c379d2c04ef9813d6bea10aa0d53",
"recording_sha256": "3061e664dfb42af7f7b0965947bed24371ad7444456470d89cf93b625d922e3a"
},
"edited": {
"seconds": 4.653710917104036,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889",
"commands": [
"F7 usb wifi",
"C 9",
"DONE navigate"
],
"generated": 24,
"thinking": 0,
"inference_seconds": 2.3331015839939937,
"receipt_sha256": "11b29066e5072ea0fcdcf58760a9965d22d841c02f3b39d18d4ef73d7fa44fe2",
"recording_sha256": "2da7485aae4ee646d4a06db7afef2f21738beeba6051bb2329115b7b95a8f73e"
}
},
{
"checkpoint": "complete-cross",
"task": 274,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 1.5838020523350003,
"original": {
"seconds": 14.471260791062377,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 usb wifi",
"C9",
"DONE navigate"
],
"generated": 377,
"thinking": 342,
"inference_seconds": 12.084626293042675,
"receipt_sha256": "10a1dd4bc841178232816fcdfe3c1e6e6bef1d20cd448a235e821b0bfd9ffd46",
"recording_sha256": "2369fd1337c93040c92ee0e75ad7a17561e144d14a5a99977f7f217fe1143664"
},
"edited": {
"seconds": 9.137038792017847,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889",
"commands": [
"F7 usb wifi",
"C 9",
"DONE navigate"
],
"generated": 24,
"thinking": 0,
"inference_seconds": 2.780198583146557,
"receipt_sha256": "b87b3eb4cc9e9cf9a753281994c78c5f7d12128329d70fb66452eaca6fce853c",
"recording_sha256": "25a99553aed072252e28c1143088416554f1e77c4a2932b9a02ce50ffb743784"
}
},
{
"checkpoint": "complete-cross",
"task": 96,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Published search weights",
"comparable": true,
"ratio": 6.996141269159831,
"original": {
"seconds": 48.335796666913666,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "d75b5709ac7a94fde651571ab7b810147f1c9ea43009a62bccfffbe4e3eab44c",
"commands": [
"C1",
"C1",
"C27",
"C21",
"ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]"
],
"generated": 1568,
"thinking": 1201,
"inference_seconds": 45.24521704099607,
"receipt_sha256": "f1c8df045f1cddb4282f5bbff2d96b64feb52187c56d4f156a241cc463597709",
"recording_sha256": "b913f6b44e7bc8a909937cb54cb7a111641675eca31c23373fec959ad21525af"
},
"edited": {
"seconds": 6.908922334085219,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889",
"commands": [
"C1",
"C1",
"C27",
"C22",
"ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]"
],
"generated": 50,
"thinking": 0,
"inference_seconds": 3.9101676668506116,
"receipt_sha256": "61f1d15c0471b7d9c1931eca1246f6b57dcd787f70767ffc8b182e59c288eb7a",
"recording_sha256": "b4b9cfb3f5cde9985ba1f8edeaee13168a9d1ba2d38d6cf6da88218cce5c2bcf"
}
},
{
"checkpoint": "complete-cross",
"task": 96,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Published search weights",
"comparable": true,
"ratio": 6.962102422319519,
"original": {
"seconds": 54.02872458298225,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "d75b5709ac7a94fde651571ab7b810147f1c9ea43009a62bccfffbe4e3eab44c",
"commands": [
"C1",
"C1",
"C28",
"C21",
"ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]"
],
"generated": 1568,
"thinking": 1095,
"inference_seconds": 50.73290495702531,
"receipt_sha256": "b3b9f0824dd2d9bec6ba3a9ee6d30cbe402ed6098254fb4638fdb52eadbd6b65",
"recording_sha256": "143cd28185ddaa3e72d9eb8937d5e70b4ff153855f749b8a15e60c5a5033e6dc"
},
"edited": {
"seconds": 7.760403582942672,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "3a62b5a986ea503b31809ef10717bd5c73ff0d377f87e77fc811323a60414889",
"commands": [
"C1",
"C1",
"C27",
"C21",
"ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]"
],
"generated": 50,
"thinking": 0,
"inference_seconds": 4.534869041061029,
"receipt_sha256": "f38c7f639137934aec85bac03e725a1b7d3a5914bc3b83069ff7b9a553474a8c",
"recording_sha256": "2ca24ca049c369c9dc20d628cd48b850b05ed260870cca89b1c5ef0dad1abe06"
}
},
{
"checkpoint": "complete-stop",
"task": 188,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 7.1029715640817015,
"original": {
"seconds": 45.752225124975666,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C 1",
"C 1",
"C 27",
"",
"ANSWER [365.42]"
],
"generated": 1597,
"thinking": 1547,
"inference_seconds": 43.50580587692093,
"receipt_sha256": "9eab04dc7cff27e3a5092aab8a335560ba1b4f005dd0447e515a22df8198aef8",
"recording_sha256": "441458b2d3e51e08b564656b24c2c6e591ce02670f9b444904aa0af865c9e537"
},
"edited": {
"seconds": 6.441279500024393,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C1",
"C1",
"C27",
"ANSWER [365.42]"
],
"generated": 34,
"thinking": 0,
"inference_seconds": 3.8335188737837598,
"receipt_sha256": "6ca9c942603d1cb1047e49b31e961627fa3276c2ee639fa397f45203b2d61380",
"recording_sha256": "5b8f672631de970383c6adcfe4e3995a2cb1ddb5422085d72fc3656c5b2073f6"
}
},
{
"checkpoint": "complete-stop",
"task": 188,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 6.024516789340297,
"original": {
"seconds": 29.181214791955426,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C 1",
"C 1",
"C 27",
"ANSWER [365.42]"
],
"generated": 919,
"thinking": 871,
"inference_seconds": 26.31592674984131,
"receipt_sha256": "d2258ce41595c58ae9bde3b09186cee94c5f0cc904b87d48e1c6572126130e42",
"recording_sha256": "9f1a4ab393b8a5a5b3e0f6240e56002444b322ab789e3bb75449a56beca4fde5"
},
"edited": {
"seconds": 4.843743624980561,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [365.42]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 2.792007416137494,
"receipt_sha256": "79f0ca4315771fd1cab3fce07e4fed0b52feeeb417c90f12da2018906d180758",
"recording_sha256": "3468a0cea7472e637a3dbc0c6a3b235b6ccdad3ddc63cc29dc8d067cfa7c065e"
}
},
{
"checkpoint": "complete-stop",
"task": 189,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 6.254271096464466,
"original": {
"seconds": 33.172848040936515,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C27",
"",
"ANSWER [754.99]"
],
"generated": 1077,
"thinking": 1018,
"inference_seconds": 30.75702545803506,
"receipt_sha256": "6065551e3f226bb44ba2969600abebd38fd0b913cf16f5f4d94fab45ce14fd96",
"recording_sha256": "c022881a4a31ff876167f106dba50a18a847ce86de0f5657f322ea6ccf10a997"
},
"edited": {
"seconds": 5.304031042032875,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [754.99]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 2.961101458990015,
"receipt_sha256": "dba900de9b60dde45bb9417e4d62258b90e9d90cecd61dba335258157244dd16",
"recording_sha256": "0cd40fcf46fd1d432df3e67061b94ae37d77bd390c03a0c904bfcccb8b5d4ded"
}
},
{
"checkpoint": "complete-stop",
"task": 189,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 4.237631562878631,
"original": {
"seconds": 24.859023457975127,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [754.99]"
],
"generated": 758,
"thinking": 701,
"inference_seconds": 22.223551789997146,
"receipt_sha256": "5a25da8336551bb5c4dabb70d8377e0eb1c5a55b8b20acb80832754b22c15c63",
"recording_sha256": "5db98471b4e7446b2624eff421d02339cd85ad66b74a5be6358bfaf6749296ab"
},
"edited": {
"seconds": 5.8662540829973295,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [754.99]"
],
"generated": 28,
"thinking": 0,
"inference_seconds": 3.574042542022653,
"receipt_sha256": "3a3a2de1fa749d05dad8d20fc1e36a9b56e4f9b9f6ccb261c302b8ca13489531",
"recording_sha256": "6aada3095f83402fda5df1a73421b674d849c4b0d93fa25985b5553c9f15e78f"
}
},
{
"checkpoint": "complete-stop",
"task": 231,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 31.377146000042558,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"1",
"1",
"1",
"1",
"1",
"1 a",
"1 a",
"1 a 1",
"1 a 1",
"1 a 1",
"1 a",
"1 a 1"
],
"generated": 920,
"thinking": 748,
"inference_seconds": 31.265100041171536,
"receipt_sha256": "6ab7b6b4825e98503f76ac54ea4da6d6f19389788486ba6778faef2bcb8fd294",
"recording_sha256": "3aea1b0be0d4e8bd1125a74b743474e3f4723109d8f6ecb2b6767bc86ce17ee2"
},
"edited": {
"seconds": 5.121967457933351,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [\"000000170\"]"
],
"generated": 30,
"thinking": 0,
"inference_seconds": 2.891214083065279,
"receipt_sha256": "c88c14e8d706e1ec5007960ac30dd3c5214714a865525ccea79b6f5463d05cbc",
"recording_sha256": "fd230a0bf607e34cda779e74694430fb50e01c81b4fea13b87cecdac971afb6b"
}
},
{
"checkpoint": "complete-stop",
"task": 231,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 35.64930824993644,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"1",
"1",
"1",
"1",
"1",
"1 a",
"1 a",
"1 a 1",
"1 a 1",
"1 a 1",
"1 a",
"1 a 1"
],
"generated": 920,
"thinking": 748,
"inference_seconds": 35.54366799816489,
"receipt_sha256": "f7ef11410819ac3656a60c6fd14eea8270b1d837354b42e7b6defab49e06b2a6",
"recording_sha256": "04bd342f411014123a9ae223e0b1ca8ab9159658f812476be88e4ad798d638ee"
},
"edited": {
"seconds": 7.383649916970171,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [\"000000170\"]"
],
"generated": 30,
"thinking": 0,
"inference_seconds": 4.439961792086251,
"receipt_sha256": "5e32187e00f8defc87c829ed6b0de645a2ccd886c444a197e94aee3b06d804aa",
"recording_sha256": "c5925326dade396c63fc48991f5e07ac6ae97a02dce1a4d46997f981a6418e3c"
}
},
{
"checkpoint": "complete-stop",
"task": 274,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.1339759539566816,
"original": {
"seconds": 15.146393416915089,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 usb wifi",
"C9",
"DONE navigate"
],
"generated": 377,
"thinking": 342,
"inference_seconds": 11.735453625093214,
"receipt_sha256": "152496a6b01bdebe0aeba8ea609086850195d8744822a5fd0b67b5c0f3bc428b",
"recording_sha256": "f76d533f90281fe1d95512f225d1169df880f846f91cea5bdc15377f5282c847"
},
"edited": {
"seconds": 7.097733875038102,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"F7 usb wifi",
"C 9",
"DONE navigate"
],
"generated": 24,
"thinking": 0,
"inference_seconds": 2.6527612908976153,
"receipt_sha256": "9c1b39b97301ecbd9911182d3ce0a13b5ab3f201faa60abbea69cb73b78fcb72",
"recording_sha256": "437261111d5df0b41ce83fd2010a4394d5900ee947f322ef60ad5aede114c3ad"
}
},
{
"checkpoint": "complete-stop",
"task": 274,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.664622233674974,
"original": {
"seconds": 14.349906249903142,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 usb wifi",
"C9",
"DONE navigate"
],
"generated": 377,
"thinking": 342,
"inference_seconds": 11.803165707970038,
"receipt_sha256": "8e75836126e49acae38b153d83368055a0d49906c9f839a229005e60e8338762",
"recording_sha256": "af12cc9808d1e46d8a9c2c52e2e5a547b9e1345a484eec137aa3090d7aa8f212"
},
"edited": {
"seconds": 5.385343583999202,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"F7 usb wifi",
"C 9",
"DONE navigate"
],
"generated": 24,
"thinking": 0,
"inference_seconds": 2.6537152079399675,
"receipt_sha256": "1bbcfec71ea23fc04d20d9a67f569cf0acfa7a95e356897b655dc885a09a780e",
"recording_sha256": "e897298b69c97db8c118e4c2317b0c5f90298da7dc204658cf5e7d67cf008e52"
}
},
{
"checkpoint": "complete-stop",
"task": 275,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.631130870629422,
"original": {
"seconds": 18.87460729200393,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F 7 xbox",
"K",
"K",
"C 9",
"DONE navigate"
],
"generated": 491,
"thinking": 448,
"inference_seconds": 16.716478960122913,
"receipt_sha256": "9c17b994ac387513f3ed37639c64093dd15504f46e73b18facff60417f7804c4",
"recording_sha256": "bf459c9f1d0196afdb26555f16f093a9e43d02408762be5fdec0dcb7b7fc64fb"
},
"edited": {
"seconds": 7.1735722090816125,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"F7 xbox",
"C 9",
"DONE navigate"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 2.381859084009193,
"receipt_sha256": "d3e2c57259f9a441e539598ffa15c70e08b0cbba27b48d0653482cccd62848bb",
"recording_sha256": "9b21185ae61f559a9691c5398789890fd030da095d8dd1bd22f4e3f2851906da"
}
},
{
"checkpoint": "complete-stop",
"task": 275,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 4.085940566193339,
"original": {
"seconds": 20.91780145803932,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F 7 xbox",
"K",
"K",
"C 9",
"DONE navigate"
],
"generated": 491,
"thinking": 448,
"inference_seconds": 17.97420541709289,
"receipt_sha256": "bace17fc01e37ed306f4e64981bf49c67bb9a6b95100b1895f955c2e54fd1c54",
"recording_sha256": "00c066740a18093c3dab6ab4a99c066338f6c2e60a0d3675214d560b8a216371"
},
"edited": {
"seconds": 5.119458082946949,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"F7 xbox",
"C 9",
"DONE navigate"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 2.7432119579752907,
"receipt_sha256": "ecec834e9b1f89a8d4a50df566ab0db1caea8bdaa500e96d765805ac755e99f0",
"recording_sha256": "c3836f0939f129721c89b1c4493bc3339533ffd48694876bc341252d06cbca13"
}
},
{
"checkpoint": "complete-stop",
"task": 276,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.6260388683186013,
"original": {
"seconds": 14.802143833017908,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 switch accessories",
"C9",
"DONE navigate"
],
"generated": 312,
"thinking": 284,
"inference_seconds": 10.696286541060545,
"receipt_sha256": "3d6755b8986e755acd9efcb9262a3499f3d3eda412384a345d72217a5cbbf27a",
"recording_sha256": "621e65298b2636414d62b87c2506ce16ddf0f20a6757db53331a82624360f2f2"
},
"edited": {
"seconds": 5.636681167059578,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"F7 switch accessories",
"C 9",
"DONE navigate"
],
"generated": 23,
"thinking": 0,
"inference_seconds": 2.8598002499202266,
"receipt_sha256": "2441f22ddf5e9880abbbc20fd4c99c3600bada9eb1f29c08488c582ec64052ba",
"recording_sha256": "bfc758912cdfa863e0cf101c55f53bef3a13ca4e3d4ca76f7991763dd25e9758"
}
},
{
"checkpoint": "complete-stop",
"task": 276,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.6794437974755816,
"original": {
"seconds": 14.731332250055857,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 switch accessories",
"C9",
"DONE navigate"
],
"generated": 312,
"thinking": 284,
"inference_seconds": 10.826686957036145,
"receipt_sha256": "71905bf063f752c654a13cbfa2f216e892d4c18809658f74f083f4f4e39d1598",
"recording_sha256": "18ca2607d91d54a0a6b66b8f3c13e657a3f1eaa1078ec0d2f15e48e8f2a415f1"
},
"edited": {
"seconds": 5.4979067909298465,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"F7 switch accessories",
"C 9",
"DONE navigate"
],
"generated": 23,
"thinking": 0,
"inference_seconds": 2.7760118750156835,
"receipt_sha256": "1382dbbfddbf5afaa24370bcf99eb3c6605aa18db4b9ba05d74e5a9259b1d686",
"recording_sha256": "920d1fdd4076374344a35630e6bd209a8a639523021e90f779d93b62ba109726"
}
},
{
"checkpoint": "complete-stop",
"task": 277,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 3.296279570524557,
"original": {
"seconds": 19.38110354100354,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 batteries for iphone 13",
"K",
"C9",
"DONE navigate"
],
"generated": 413,
"thinking": 375,
"inference_seconds": 14.283008917002007,
"receipt_sha256": "cedcfe5b1a31810a6b057846f778a6502ee0c60f84ceb6dc279763555b30696b",
"recording_sha256": "9c0d7164e632a65f82c2ab8b3ce320b5521b47404d104ca09247aa8050537c6c"
},
"edited": {
"seconds": 5.879690458998084,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"F7 batteries for iphone 13",
"C 9",
"DONE navigate"
],
"generated": 26,
"thinking": 0,
"inference_seconds": 2.8371055430034176,
"receipt_sha256": "c20a66a68f4fade474ce0021165497d211331c8281dc3136b93555e663b9c150",
"recording_sha256": "372124866542f39ea4e2625481c057e19b69efe4e4ba55b88451f58e191a5c76"
}
},
{
"checkpoint": "complete-stop",
"task": 277,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 3.1275596003092097,
"original": {
"seconds": 20.637632584082894,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 batteries for iphone 13",
"K",
"C9",
"DONE navigate"
],
"generated": 413,
"thinking": 375,
"inference_seconds": 13.629928710055538,
"receipt_sha256": "58185eb48ad4e0c400a4294ee6dae8c1de9cbef96e67d64c6dad31f298c56390",
"recording_sha256": "d23a8f4908dbeb7069f7c4ac1d11317057ae4b0b7d6ddfd0c68f1e83f6f56d5a"
},
"edited": {
"seconds": 6.598637666902505,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"F7 batteries for iphone 13",
"C 9",
"DONE navigate"
],
"generated": 26,
"thinking": 0,
"inference_seconds": 2.9481292499694973,
"receipt_sha256": "f297dc8affab7a7d13bbc4d9402b00a4689e1f51b0288fbe88b9f8586e8d9151",
"recording_sha256": "b48676bc4cf07cd388c0f16709e983c20eda7aaaccc4f4382901fe7d7d3e3309"
}
},
{
"checkpoint": "complete-stop",
"task": 278,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 3.072605270412202,
"original": {
"seconds": 19.747643290902488,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 green tea bag for weight loss",
"K",
"K",
"C9",
"DONE navigate"
],
"generated": 498,
"thinking": 447,
"inference_seconds": 16.53311104211025,
"receipt_sha256": "dc2bb501a144c4091b54d3464426557c7283a1da397150d916db8ef1efed3f96",
"recording_sha256": "8acec465aac0e97baa9057990322b7e45548060d5cd4d2f8b60c46d0944e6774"
},
"edited": {
"seconds": 6.427003000047989,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"F7 green tea bag for weight loss",
"C 9",
"DONE navigate"
],
"generated": 26,
"thinking": 0,
"inference_seconds": 2.7816810000222176,
"receipt_sha256": "93002218d76d699268c6b36d6efe7aa6c231eed8151fc50906943ff94ced8b06",
"recording_sha256": "4401d49f06fcc1423c49b97f088eb5d2085a7f95ee36b4537acaae7593b261bf"
}
},
{
"checkpoint": "complete-stop",
"task": 278,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 3.971020153923018,
"original": {
"seconds": 24.29026104195509,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 green tea bag for weight loss",
"K",
"K",
"C9",
"DONE navigate"
],
"generated": 498,
"thinking": 447,
"inference_seconds": 16.81286837288644,
"receipt_sha256": "f54ed2ed164381f7fce725a451b1243c220bd58f2cf76e6864a5b4ce14c7363d",
"recording_sha256": "051a34b4ceeff354add69fd699507064e20260cddc69d406284ec3e69a89005f"
},
"edited": {
"seconds": 6.11688183399383,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"F7 green tea bag for weight loss",
"C 9",
"DONE navigate"
],
"generated": 26,
"thinking": 0,
"inference_seconds": 2.8285570829175413,
"receipt_sha256": "5ad1b4a1fa25da2ebe1e5ed4f13976b9d52457987b254a55e09912f3d1ded67a",
"recording_sha256": "1fc55b1c968704e5e58ae61587eaedc473f3bf7e4e69926c42efa56eca019525"
}
},
{
"checkpoint": "complete-stop",
"task": 96,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Published search weights",
"comparable": true,
"ratio": 6.765838304025578,
"original": {
"seconds": 47.030071915942244,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "d75b5709ac7a94fde651571ab7b810147f1c9ea43009a62bccfffbe4e3eab44c",
"commands": [
"C1",
"C1",
"C27",
"C21",
"ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]"
],
"generated": 1568,
"thinking": 1201,
"inference_seconds": 43.0982726670336,
"receipt_sha256": "f4d2295f22d03a79aeb5746e8659c03e0bf8edc43cc902c7f1c5a0fc2367b1ac",
"recording_sha256": "ffbad15481de75a1a2a4d755a0d133dc6398fd87001b8bbabac083b9708da591"
},
"edited": {
"seconds": 6.951107874978334,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C1",
"C27",
"C21",
"ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]"
],
"generated": 50,
"thinking": 0,
"inference_seconds": 3.9192599579691887,
"receipt_sha256": "ed5ab8f7e7590abcc136d9d008e090952896195358a71986d6dac5919d35043d",
"recording_sha256": "752e6ce4c20402ef8c25902b95b5fae1e2699abf46df5148aa351c64e057e2f8"
}
},
{
"checkpoint": "complete-stop",
"task": 96,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Published search weights",
"comparable": true,
"ratio": 5.8896550787316775,
"original": {
"seconds": 44.6329190840479,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "d75b5709ac7a94fde651571ab7b810147f1c9ea43009a62bccfffbe4e3eab44c",
"commands": [
"C1",
"C1",
"C27",
"C21",
"ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]"
],
"generated": 1568,
"thinking": 1201,
"inference_seconds": 41.65323895891197,
"receipt_sha256": "8c80ef2d7d79ed00bcc7d01e5682f0ac7e97164e2c70c821794c315875599ec3",
"recording_sha256": "af1d76372398c05719b345a5da3e648a22a3419cb600a6187bc29ee07f828a68"
},
"edited": {
"seconds": 7.578188957995735,
"score": 1.0,
"grader_status": "success",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C1",
"C27",
"C21",
"ANSWER [{\"status\":\"Canceled\",\"arrival_date\":null}]"
],
"generated": 50,
"thinking": 0,
"inference_seconds": 4.394748790888116,
"receipt_sha256": "902a18fe5537a3b856fc1602cb1e4591dd66e669c2e349152859db0701c81e23",
"recording_sha256": "a2f9b51758f28278a313efa357575ed3d4e772ea15d173d9721a39b9340ae865"
}
},
{
"checkpoint": "complete-stop",
"task": 191,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 72.89141737495083,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"1",
"1",
"C1",
"C27",
"D",
"D",
"C17",
"D",
"C18",
"D",
"D",
"D"
],
"generated": 2323,
"thinking": 1889,
"inference_seconds": 67.62824633193668,
"receipt_sha256": "1b0f760548d1a346c64962ec9d28e96599619d967e6a45ed79596b7c9ecaf44b",
"recording_sha256": "68c336263c3ddb7e33bb2c2397001df3c1f9d3ac0e8a732c9dd444252a24dbdd"
},
"edited": {
"seconds": 19.83579604199622,
"score": 0.0,
"grader_status": "error",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C1",
"C28",
"ANSWER [\"$365.42\"]"
],
"generated": 537,
"thinking": 308,
"inference_seconds": 16.693413792178035,
"receipt_sha256": "65a1fffe7d9a22163195d72cc9badbacc70adc6fea9b93c0d1db398a66491975",
"recording_sha256": "6d96505a8709636d495217bf4faea1f0a9324809b7892cd05e5a601e0d947d82"
}
},
{
"checkpoint": "complete-stop",
"task": 191,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 82.10596250009257,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"1",
"1",
"C1",
"C27",
"",
"D",
"D",
"C17",
"D",
"C18",
"D",
"D"
],
"generated": 2767,
"thinking": 2332,
"inference_seconds": 76.9749869160587,
"receipt_sha256": "1dad1b17cc2dedd8e12f18a1854674c8f6ca4e59383cff29baf168b77dfb743d",
"recording_sha256": "cd7cbdda2840468bbf5e0ebf02c56df2b417499a567ea88df81087495b59ede4"
},
"edited": {
"seconds": 21.124708750052378,
"score": 0.0,
"grader_status": "error",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C1",
"C1",
"C27",
"ANSWER [\"$365.42\"]"
],
"generated": 537,
"thinking": 308,
"inference_seconds": 18.80111487407703,
"receipt_sha256": "d6cba48d27dc727f599c7ec22367f0868530e4c30c911e5fdfb446b272cb19f2",
"recording_sha256": "8d84d0f6e6a12c5c9008894c7fa6186af9980c19361d56e83e76b5fc422e7256"
}
},
{
"checkpoint": "complete-stop",
"task": 283,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 81.33533662499394,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 Xbox controller",
"K",
"K",
"C9",
"",
"D",
"C3",
"F7 Xbox controller",
"K",
"C9",
"D",
"D"
],
"generated": 2668,
"thinking": 2498,
"inference_seconds": 74.3948387921555,
"receipt_sha256": "c0668e2011b93eb9c2e80746d17db3de4c0a07df6aee1b2dc76e65a918dbe99d",
"recording_sha256": "fe0be6adb38961edcb5c4e5e48798336e93ebd4d962ea6db10fb51ecac152e67"
},
"edited": {
"seconds": 12.874797667027451,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C7",
"C7",
"C7",
"C7",
"C7",
"C7",
"C7",
"C7",
"C7",
"C7",
"C7",
"C7"
],
"generated": 82,
"thinking": 0,
"inference_seconds": 8.267056584125385,
"receipt_sha256": "522a89222aa4484cb9a5801275aa38ea28c6451b336fe6327eb8ec1714590ff2",
"recording_sha256": "a2c70c7239ec95f2be1d56b722cf1d51d7fddca0c772d48f9e9dd0146b9c01e9"
}
},
{
"checkpoint": "complete-stop",
"task": 283,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "M5 Max MacBook Pro",
"baseline": "Upstream stock",
"comparable": true,
"ratio": null,
"original": {
"seconds": 81.81984350003768,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"F7 Xbox controller",
"K",
"K",
"C9",
"",
"D",
"C3",
"F7 Xbox controller",
"K",
"C9",
"D",
"D"
],
"generated": 2669,
"thinking": 2499,
"inference_seconds": 74.8769646261353,
"receipt_sha256": "1584689f458fb5263fc5558a005934bd8c20cb22c561c0afa8aa930b1de0dd1d",
"recording_sha256": "b52d58cfa69a4287740e230d7ba541ad499688189adc92f5daa73c6893e5fcc6"
},
"edited": {
"seconds": 12.775214791996405,
"score": 0.0,
"grader_status": "failure",
"device": "M5 Max MacBook Pro",
"shard_sha256": "4123a4c2fe5f770cdabbe08ff0bedccfee63e9beee73a64290ea5fd20f19754a",
"commands": [
"C7",
"C7",
"C7",
"C7",
"C7",
"C7",
"C7",
"C7",
"C7",
"C7",
"C7",
"C7"
],
"generated": 82,
"thinking": 0,
"inference_seconds": 8.176048293127678,
"receipt_sha256": "6271021877e564aafe7042846c9838554a213ab14db6bc2e19456ba010e3d322",
"recording_sha256": "00afb4eef4002d38d985e4966142b1f8f8412181575a9a17f60767458c78373d"
}
}
]
}
{
"pairs": [
{
"checkpoint": "order-boundary",
"task": 188,
"repeat": 1,
"order": [
"edited",
"original"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": false,
"ratio": null,
"original": {
"seconds": 5.450880583026446,
"score": 0.0,
"grader_status": "failure",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C1",
"FAIL RETRIEVE UNKNOWN_ERROR The site metis.lti.cs.cmu.edu is unreachable (ERR_FAILED), so no order data can be retrieved."
],
"generated": 639,
"thinking": 546,
"inference_seconds": 4.341966916923411,
"receipt_sha256": "957b7bbfe248a0d71c07dfe9f6143283ca95163ce64e0c4d27d3d58a78c7e0f8",
"recording_sha256": "9e49c54ad4cf8133a89475b938369b28d94fe9e23950def74d4329db26adb377"
},
"edited": {
"seconds": 3.161069875000976,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 0.9746392919914797,
"receipt_sha256": "0f6c92e5cec25fca09c73407032489ee4f57b9fdb24eec2c419388557ab3ddf0",
"recording_sha256": "c576fd15d4c275fa3a4268d21074c49f47bf719c5e9853432f816bf289247064"
}
},
{
"checkpoint": "order-boundary",
"task": 188,
"repeat": 2,
"order": [
"original",
"edited"
],
"site": "shopping",
"device": "RTX 5090",
"baseline": "Upstream stock",
"comparable": true,
"ratio": 2.591420520284619,
"original": {
"seconds": 6.857609499944374,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "65e60f51aa1284d4fb022584908c315270334d991828bb774af60584523539d1",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 636,
"thinking": 601,
"inference_seconds": 4.802369457902387,
"receipt_sha256": "c7cec829eb022d0eedbe230b27a36d6ee5346ccd33ac284c0703eed777f3f584",
"recording_sha256": "fb5ff40e56daa8f3b1bb3ec7cf75c4737be808a7451d370246bb6cb44779c593"
},
"edited": {
"seconds": 2.6462742909789085,
"score": 1.0,
"grader_status": "success",
"device": "RTX 5090",
"shard_sha256": "3b6a96c60bbe2cfbea609f004386ad6ed4fed325184f0ec6658fcaa89efe331d",
"commands": [
"C1",
"C28",
"ANSWER [365.42]"
],
"generated": 22,
"thinking": 0,
"inference_seconds": 0.6069882510928437,
"receipt_sha256": "f28317df706ae824db182c96082d262e272b6c3c545cf23251c189b8db5fe92a",
"recording_sha256": "dfd9521b812a2461962c05936ce0bf25ea5c2f998355315dc07d7ca122247807"
}
},
{
"checkpoint": "order-boundary",