VisualInference's picture
Inkling 2 Mini for MLX, 2bit: weights, card, validation results
031489f verified
Raw History Blame Contribute Delete
85.6 kB
{ "held_out_loss": { "native": { "tokens": 40806, "nll": 1.4480010462071342, "acc": 0.6699995098759987, "seconds": 115, "peak_gb": 22.03, "weights_gib": null, "activations": "bf16", "actquant": null, "chat": { "tokens": 6597, "nll": 0.4662371408908642 }, "code": { "tokens": 8877, "nll": 0.8892847396130361 }, "wikitext": { "tokens": 25332, "nll": 1.8994624837962406 } }, "2bit": { "tokens": 40806, "nll": 1.5432901726466433, "acc": 0.6456893594079302, "seconds": 112, "peak_gb": 11.49, "weights_gib": null, "activations": "bf16", "actquant": null, "chat": { "tokens": 6597, "nll": 0.6182796457454582 }, "code": { "tokens": 8877, "nll": 0.967472937161608 }, "wikitext": { "tokens": 25332, "nll": 1.9859644204505187 } }, "3bit_round_to_nearest": { "tokens": 40806, "nll": 1.4701619848213856, "acc": 0.6639709846591187, "seconds": 113, "peak_gb": 19.63, "weights_gib": 11.910704612731934, "activations": "bf16", "actquant": null, "chat": { "tokens": 6597, "nll": 0.49830741327206834 }, "code": { "tokens": 8877, "nll": 0.9093072548956838 }, "wikitext": { "tokens": 25332, "nll": 1.919792177702378 } } }, "numerics_experiments": { "native_bf16_actquant": { "tokens": 40806, "nll": 1.4507249015946835, "acc": 0.6688967308729108, "seconds": 151, "peak_gb": 22.22, "weights_gib": null, "activations": "bf16", "actquant": [ "input", "hidden" ], "chat": { "tokens": 6597, "nll": 0.470449288324712 }, "code": { "tokens": 8877, "nll": 0.8916940795444644 }, "wikitext": { "tokens": 25332, "nll": 1.9019089702857381 } }, "native_fp32": { "tokens": 40806, "nll": 1.4489235311713002, "acc": 0.6690682742733912, "seconds": 134, "peak_gb": 24.1, "weights_gib": null, "activations": "fp32", "actquant": null, "chat": { "tokens": 6597, "nll": 0.4647530796709649 }, "code": { "tokens": 8877, "nll": 0.8885994847056913 }, "wikitext": { "tokens": 25332, "nll": 1.9015750797668678 } }, "native_fp32_actquant": { "tokens": 40806, "nll": 1.4478448928936214, "acc": 0.6682105572709895, "seconds": 163, "peak_gb": 24.24, "weights_gib": null, "activations": "fp32", "actquant": [ "input", "hidden" ], "chat": { "tokens": 6597, "nll": 0.4712143190091029 }, "code": { "tokens": 8877, "nll": 0.8948558546528027 }, "wikitext": { "tokens": 25332, "nll": 1.8959625144150138 } }, "native_bf16_actquant_input": { "tokens": 40806, "nll": 1.4477849032594567, "acc": 0.6688477184727736, "seconds": 120, "peak_gb": 21.99, "weights_gib": null, "activations": "bf16", "actquant": [ "input" ], "chat": { "tokens": 6597, "nll": 0.46967875793049946 }, "code": { "tokens": 8877, "nll": 0.8883638166019585 }, "wikitext": { "tokens": 25332, "nll": 1.8985407546329665 } } }, "tasks": { "tools": { "passed": 19, "total": 19, "leaked": 0, "seconds": 96, "rows": [ { "kind": "single", "i": 0, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 2.5, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 1, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 2.8, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 2, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 2.2, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 3, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 2.4, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 4, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 2.7, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 5, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 5.4, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 6, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 3.9, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 7, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 2.7, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 8, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 3.3, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 9, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 2.2, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 10, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 6.3, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 11, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 3.8, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "no_tool", "i": 0, "ok": true, "n_calls": 0, "finish": "stop", "leaked": false, "seconds": 2.0 }, { "kind": "no_tool", "i": 1, "ok": true, "n_calls": 0, "finish": "stop", "leaked": false, "seconds": 2.8 }, { "kind": "multi", "i": 0, "ok": true, "rounds": 2, "n_calls": 2, "finish": "stop", "leaked": false, "seconds": 7.2, "final": "Lisbon is warmer right now: 19\u00b0C vs Oslo at 3\u00b0C.", "error": null }, { "kind": "multi", "i": 1, "ok": true, "rounds": 2, "n_calls": 1, "finish": "stop", "leaked": false, "seconds": 5.5, "final": "RETRIES is 3.", "error": null }, { "kind": "multi", "i": 2, "ok": true, "rounds": 3, "n_calls": 2, "finish": "stop", "leaked": false, "seconds": 11.3, "final": "docs/ARCHITECTURE.md does not exist. There is no docs/ directory in the repo; the only files are Makefile, README.md, app/config.py, src/queue.py, and tests/test_queue.py. No architecture documentation is present.", "error": null }, { "kind": "multi", "i": 3, "ok": true, "rounds": 3, "n_calls": 3, "finish": "stop", "leaked": false, "seconds": 19.4, "final": "(1234 \u00d7 57) \u2212 999 = 69,339. \n69,339 \u00f7 7 \u2248 9,905.5714.", "error": null }, { "kind": "multi", "i": 4, "ok": true, "rounds": 2, "n_calls": 1, "finish": "stop", "leaked": false, "seconds": 7.5, "final": "Customer C-88 has 2 orders in total. The most expensive is A-1042, totaling 129.50.", "error": null } ] }, "json": { "passed": 9, "total": 10, "leaked": 0, "seconds": 44, "rows": [ { "i": 0, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 7.8, "content": null }, { "i": 1, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 2.4, "content": null }, { "i": 2, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 4.1, "content": null }, { "i": 3, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 4.1, "content": null }, { "i": 4, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 5.2, "content": null }, { "i": 5, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 3.3, "content": null }, { "i": 6, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 3.8, "content": null }, { "i": 7, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 2.9, "content": null }, { "i": 8, "ok": false, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 4.1, "content": "{\"quote\":\"She said \\\"stop\\\" and left.\"}" }, { "i": 9, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 6.2, "content": null } ] }, "humaneval": { "passed": 32, "total": 40, "leaked": 0, "seconds": 644, "rows": [ { "task": "HumanEval/0", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 530, "seconds": 17.1 }, { "task": "HumanEval/4", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 288, "seconds": 9.5 }, { "task": "HumanEval/8", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 327, "seconds": 10.7 }, { "task": "HumanEval/12", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 453, "seconds": 14.7 }, { "task": "HumanEval/16", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 192, "seconds": 6.3 }, { "task": "HumanEval/20", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 602, "seconds": 19.6 }, { "task": "HumanEval/24", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 475, "seconds": 15.1 }, { "task": "HumanEval/28", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 142, "seconds": 4.8 }, { "task": "HumanEval/32", "ok": false, "err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmpkef8hadu/prog.py\", line 26, in <module>\n check(find_zero)\n ^^^^^^^^^\nNameError: name 'find_zero' is not defined\n", "finish": "length", "leaked": false, "tokens": 900, "seconds": 30.0 }, { "task": "HumanEval/36", "ok": false, "err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp99q5n2iz/prog.py\", line 20, in <module>\n check(fizz_buzz)\n ^^^^^^^^^\nNameError: name 'fizz_buzz' is not defined\n", "finish": "length", "leaked": false, "tokens": 900, "seconds": 28.9 }, { "task": "HumanEval/40", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 631, "seconds": 20.6 }, { "task": "HumanEval/44", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 546, "seconds": 17.5 }, { "task": "HumanEval/48", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 171, "seconds": 5.7 }, { "task": "HumanEval/52", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 181, "seconds": 6.0 }, { "task": "HumanEval/56", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 456, "seconds": 14.8 }, { "task": "HumanEval/60", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 277, "seconds": 9.1 }, { "task": "HumanEval/64", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 417, "seconds": 13.6 }, { "task": "HumanEval/68", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 297, "seconds": 10.4 }, { "task": "HumanEval/72", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 178, "seconds": 6.4 }, { "task": "HumanEval/76", "ok": false, "err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmpif1io117/prog.py\", line 21, in <module>\n check(is_simple_power)\n ^^^^^^^^^^^^^^^\nNameError: name 'is_simple_power' is not defined\n", "finish": "length", "leaked": false, "tokens": 900, "seconds": 29.3 }, { "task": "HumanEval/80", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 376, "seconds": 12.2 }, { "task": "HumanEval/84", "ok": false, "err": " File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmpkyh0_05r/prog.py\", line 3\n \"\"\"Given a positive integer N, return the total sum of its digits in binary.\n ^\nSyntaxError: unterminated triple-quoted string literal (detected at line 25)\n", "finish": "length", "leaked": false, "tokens": 900, "seconds": 29.2 }, { "task": "HumanEval/88", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 487, "seconds": 16.2 }, { "task": "HumanEval/92", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 500, "seconds": 16.2 }, { "task": "HumanEval/96", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 603, "seconds": 19.5 }, { "task": "HumanEval/100", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 379, "seconds": 12.3 }, { "task": "HumanEval/104", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 347, "seconds": 11.3 }, { "task": "HumanEval/108", "ok": false, "err": "/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp3s50ci9p/prog.py\", line 31, in <module>\n check(count_nums)\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp3s50ci9p/prog.py\", line 18, in check\n assert candidate([-1, -2, 0]) == 0\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError\n", "finish": "stop", "leaked": false, "tokens": 589, "seconds": 19.3 }, { "task": "HumanEval/112", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 203, "seconds": 6.9 }, { "task": "HumanEval/116", "ok": false, "err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmpwj8frp_n/prog.py\", line 21, in <module>\n check(sort_array)\n ^^^^^^^^^^\nNameError: name 'sort_array' is not defined\n", "finish": "length", "leaked": false, "tokens": 900, "seconds": 29.8 }, { "task": "HumanEval/120", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 404, "seconds": 13.5 }, { "task": "HumanEval/124", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 813, "seconds": 27.1 }, { "task": "HumanEval/128", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 441, "seconds": 14.3 }, { "task": "HumanEval/132", "ok": false, "err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmpdd4l577r/prog.py\", line 25, in <module>\n check(is_nested)\n ^^^^^^^^^\nNameError: name 'is_nested' is not defined\n", "finish": "length", "leaked": false, "tokens": 900, "seconds": 29.4 }, { "task": "HumanEval/136", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 462, "seconds": 15.0 }, { "task": "HumanEval/140", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 473, "seconds": 15.3 }, { "task": "HumanEval/144", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 328, "seconds": 10.8 }, { "task": "HumanEval/148", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 360, "seconds": 12.0 }, { "task": "HumanEval/152", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 381, "seconds": 12.7 }, { "task": "HumanEval/156", "ok": false, "err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp_694mygi/prog.py\", line 26, in <module>\n check(int_to_mini_roman)\n ^^^^^^^^^^^^^^^^^\nNameError: name 'int_to_mini_roman' is not defined\n", "finish": "length", "leaked": false, "tokens": 900, "seconds": 29.1 } ] }, "gsm8k": { "passed": 47, "total": 50, "leaked": 0, "seconds": 372, "rows": [ { "i": 0, "ok": true, "pred": 18.0, "gold": 18.0, "finish": "stop", "leaked": false, "tokens": 156, "seconds": 5.2 }, { "i": 26, "ok": true, "pred": 243.0, "gold": 243.0, "finish": "stop", "leaked": false, "tokens": 192, "seconds": 6.3 }, { "i": 52, "ok": true, "pred": 15.0, "gold": 15.0, "finish": "stop", "leaked": false, "tokens": 157, "seconds": 5.2 }, { "i": 78, "ok": true, "pred": 6.0, "gold": 6.0, "finish": "stop", "leaked": false, "tokens": 244, "seconds": 7.9 }, { "i": 104, "ok": true, "pred": 500.0, "gold": 500.0, "finish": "stop", "leaked": false, "tokens": 134, "seconds": 4.5 }, { "i": 130, "ok": true, "pred": 30.0, "gold": 30.0, "finish": "stop", "leaked": false, "tokens": 137, "seconds": 4.6 }, { "i": 156, "ok": true, "pred": 1400.0, "gold": 1400.0, "finish": "stop", "leaked": false, "tokens": 152, "seconds": 5.1 }, { "i": 182, "ok": true, "pred": 23.0, "gold": 23.0, "finish": "stop", "leaked": false, "tokens": 399, "seconds": 12.6 }, { "i": 208, "ok": true, "pred": 76.0, "gold": 76.0, "finish": "stop", "leaked": false, "tokens": 172, "seconds": 5.7 }, { "i": 234, "ok": false, "pred": 14.0, "gold": 21.0, "finish": "stop", "leaked": false, "tokens": 230, "seconds": 7.4 }, { "i": 260, "ok": true, "pred": 11.0, "gold": 11.0, "finish": "stop", "leaked": false, "tokens": 225, "seconds": 7.3 }, { "i": 286, "ok": true, "pred": 90.0, "gold": 90.0, "finish": "stop", "leaked": false, "tokens": 160, "seconds": 5.2 }, { "i": 312, "ok": true, "pred": 32.0, "gold": 32.0, "finish": "stop", "leaked": false, "tokens": 146, "seconds": 4.8 }, { "i": 338, "ok": true, "pred": 60.0, "gold": 60.0, "finish": "stop", "leaked": false, "tokens": 184, "seconds": 6.0 }, { "i": 364, "ok": true, "pred": 49.0, "gold": 49.0, "finish": "stop", "leaked": false, "tokens": 133, "seconds": 4.4 }, { "i": 390, "ok": true, "pred": 12.0, "gold": 12.0, "finish": "stop", "leaked": false, "tokens": 122, "seconds": 4.2 }, { "i": 416, "ok": true, "pred": 56.0, "gold": 56.0, "finish": "stop", "leaked": false, "tokens": 175, "seconds": 5.7 }, { "i": 442, "ok": true, "pred": 49.0, "gold": 49.0, "finish": "stop", "leaked": false, "tokens": 215, "seconds": 7.0 }, { "i": 468, "ok": true, "pred": 120.0, "gold": 120.0, "finish": "stop", "leaked": false, "tokens": 213, "seconds": 6.9 }, { "i": 494, "ok": false, "pred": 15.6666666667, "gold": 21.0, "finish": "stop", "leaked": false, "tokens": 861, "seconds": 27.6 }, { "i": 520, "ok": true, "pred": 63.0, "gold": 63.0, "finish": "stop", "leaked": false, "tokens": 135, "seconds": 4.7 }, { "i": 546, "ok": true, "pred": 130.0, "gold": 130.0, "finish": "stop", "leaked": false, "tokens": 270, "seconds": 8.7 }, { "i": 572, "ok": true, "pred": 100.0, "gold": 100.0, "finish": "stop", "leaked": false, "tokens": 261, "seconds": 8.5 }, { "i": 598, "ok": true, "pred": 69.0, "gold": 69.0, "finish": "stop", "leaked": false, "tokens": 211, "seconds": 6.9 }, { "i": 624, "ok": true, "pred": 85.0, "gold": 85.0, "finish": "stop", "leaked": false, "tokens": 177, "seconds": 5.8 }, { "i": 650, "ok": true, "pred": 142.0, "gold": 142.0, "finish": "stop", "leaked": false, "tokens": 230, "seconds": 7.5 }, { "i": 676, "ok": true, "pred": 240.0, "gold": 240.0, "finish": "stop", "leaked": false, "tokens": 173, "seconds": 5.7 }, { "i": 702, "ok": true, "pred": 2800.0, "gold": 2800.0, "finish": "stop", "leaked": false, "tokens": 150, "seconds": 5.1 }, { "i": 728, "ok": true, "pred": 40.0, "gold": 40.0, "finish": "stop", "leaked": false, "tokens": 222, "seconds": 7.2 }, { "i": 754, "ok": true, "pred": 89.0, "gold": 89.0, "finish": "stop", "leaked": false, "tokens": 594, "seconds": 19.1 }, { "i": 780, "ok": true, "pred": 25.0, "gold": 25.0, "finish": "stop", "leaked": false, "tokens": 378, "seconds": 12.2 }, { "i": 806, "ok": true, "pred": 70.0, "gold": 70.0, "finish": "stop", "leaked": false, "tokens": 364, "seconds": 11.7 }, { "i": 832, "ok": true, "pred": 42.0, "gold": 42.0, "finish": "stop", "leaked": false, "tokens": 207, "seconds": 6.7 }, { "i": 858, "ok": true, "pred": 3430.0, "gold": 3430.0, "finish": "stop", "leaked": false, "tokens": 350, "seconds": 11.1 }, { "i": 884, "ok": true, "pred": 50.0, "gold": 50.0, "finish": "stop", "leaked": false, "tokens": 138, "seconds": 4.6 }, { "i": 910, "ok": true, "pred": 700.0, "gold": 700.0, "finish": "stop", "leaked": false, "tokens": 127, "seconds": 4.3 }, { "i": 936, "ok": true, "pred": 250.0, "gold": 250.0, "finish": "stop", "leaked": false, "tokens": 337, "seconds": 10.9 }, { "i": 962, "ok": false, "pred": 600.0, "gold": 1000.0, "finish": "stop", "leaked": false, "tokens": 186, "seconds": 6.1 }, { "i": 988, "ok": true, "pred": 6.0, "gold": 6.0, "finish": "stop", "leaked": false, "tokens": 149, "seconds": 5.0 }, { "i": 1014, "ok": true, "pred": 3.0, "gold": 3.0, "finish": "stop", "leaked": false, "tokens": 378, "seconds": 12.2 }, { "i": 1040, "ok": true, "pred": 736.0, "gold": 736.0, "finish": "stop", "leaked": false, "tokens": 214, "seconds": 7.0 }, { "i": 1066, "ok": true, "pred": 14.0, "gold": 14.0, "finish": "stop", "leaked": false, "tokens": 177, "seconds": 5.8 }, { "i": 1092, "ok": true, "pred": 1080.0, "gold": 1080.0, "finish": "stop", "leaked": false, "tokens": 322, "seconds": 10.4 }, { "i": 1118, "ok": true, "pred": 5.0, "gold": 5.0, "finish": "stop", "leaked": false, "tokens": 253, "seconds": 8.3 }, { "i": 1144, "ok": true, "pred": 34.0, "gold": 34.0, "finish": "stop", "leaked": false, "tokens": 172, "seconds": 5.7 }, { "i": 1170, "ok": true, "pred": 2.0, "gold": 2.0, "finish": "stop", "leaked": false, "tokens": 143, "seconds": 4.8 }, { "i": 1196, "ok": true, "pred": 75.0, "gold": 75.0, "finish": "stop", "leaked": false, "tokens": 160, "seconds": 5.3 }, { "i": 1222, "ok": true, "pred": 350.0, "gold": 350.0, "finish": "stop", "leaked": false, "tokens": 145, "seconds": 4.8 }, { "i": 1248, "ok": true, "pred": 750.0, "gold": 750.0, "finish": "stop", "leaked": false, "tokens": 156, "seconds": 5.3 }, { "i": 1274, "ok": true, "pred": 38.0, "gold": 38.0, "finish": "stop", "leaked": false, "tokens": 203, "seconds": 6.6 } ] } }, "tasks_original": { "tools": { "passed": 18, "total": 19, "leaked": 0, "seconds": 104, "rows": [ { "kind": "single", "i": 0, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 3.3, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 1, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 4.0, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 2, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 2.9, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 3, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 2.9, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 4, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 3.2, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 5, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 5.3, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 6, "n_calls": 0, "finish": "stop", "leaked": false, "seconds": 4.5, "valid_json": false, "right_tool": false, "ok": false, "got": "" }, { "kind": "single", "i": 7, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 3.3, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 8, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 3.9, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 9, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 2.2, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 10, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 7.6, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "single", "i": 11, "n_calls": 1, "finish": "tool_calls", "leaked": false, "seconds": 4.7, "valid_json": true, "right_tool": true, "ok": true }, { "kind": "no_tool", "i": 0, "ok": true, "n_calls": 0, "finish": "stop", "leaked": false, "seconds": 2.2 }, { "kind": "no_tool", "i": 1, "ok": true, "n_calls": 0, "finish": "stop", "leaked": false, "seconds": 3.7 }, { "kind": "multi", "i": 0, "ok": true, "rounds": 2, "n_calls": 2, "finish": "stop", "leaked": false, "seconds": 9.7, "final": "Lisbon is warmer right now: 19\u00b0C vs Oslo\u2019s 3\u00b0C \u2014 about 16\u00b0C warmer.", "error": null }, { "kind": "multi", "i": 1, "ok": true, "rounds": 2, "n_calls": 1, "finish": "stop", "leaked": false, "seconds": 6.2, "final": "RETRIES is 3.", "error": null }, { "kind": "multi", "i": 2, "ok": true, "rounds": 2, "n_calls": 1, "finish": "stop", "leaked": false, "seconds": 9.1, "final": "docs/ARCHITECTURE.md does not exist \u2014 there is no `docs/` directory in this repository. The repo contains only `Makefile`, `README.md`, `app/`, `src/`, and `tests/`. I can summarise the README instead if you'd like.", "error": null }, { "kind": "multi", "i": 3, "ok": true, "rounds": 2, "n_calls": 2, "finish": "stop", "leaked": false, "seconds": 15.3, "final": "(1234 \u00d7 57) \u2212 999 = **69,339**\n\n69,339 \u00f7 7 = **9,905.57142857143** (about **9,905.57**)", "error": null }, { "kind": "multi", "i": 4, "ok": true, "rounds": 2, "n_calls": 1, "finish": "stop", "leaked": false, "seconds": 9.6, "final": "Customer C-88 has 2 orders in total. The most expensive is A-1042, totaling $129.50.", "error": null } ] }, "json": { "passed": 9, "total": 10, "leaked": 0, "seconds": 86, "rows": [ { "i": 0, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 17.1, "content": null }, { "i": 1, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 3.0, "content": null }, { "i": 2, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 5.2, "content": null }, { "i": 3, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 4.6, "content": null }, { "i": 4, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 8.4, "content": null }, { "i": 5, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 3.9, "content": null }, { "i": 6, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 4.8, "content": null }, { "i": 7, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 3.3, "content": null }, { "i": 8, "ok": false, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 27.3, "content": "{\"quote\":\"She said \\\"stop\\\" and left.\\\\\"}" }, { "i": 9, "ok": true, "valid_json": true, "finish": "stop", "leaked": false, "seconds": 8.0, "content": null } ] }, "humaneval": { "passed": 35, "total": 40, "leaked": 0, "seconds": 759, "rows": [ { "task": "HumanEval/0", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 427, "seconds": 17.7 }, { "task": "HumanEval/4", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 285, "seconds": 11.9 }, { "task": "HumanEval/8", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 493, "seconds": 20.3 }, { "task": "HumanEval/12", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 573, "seconds": 23.6 }, { "task": "HumanEval/16", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 227, "seconds": 9.4 }, { "task": "HumanEval/20", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 625, "seconds": 26.1 }, { "task": "HumanEval/24", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 249, "seconds": 10.3 }, { "task": "HumanEval/28", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 165, "seconds": 7.0 }, { "task": "HumanEval/32", "ok": false, "err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp38s2jvzb/prog.py\", line 26, in <module>\n check(find_zero)\n ^^^^^^^^^\nNameError: name 'find_zero' is not defined\n", "finish": "length", "leaked": false, "tokens": 900, "seconds": 38.6 }, { "task": "HumanEval/36", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 640, "seconds": 26.4 }, { "task": "HumanEval/40", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 641, "seconds": 27.0 }, { "task": "HumanEval/44", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 365, "seconds": 15.1 }, { "task": "HumanEval/48", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 210, "seconds": 8.8 }, { "task": "HumanEval/52", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 239, "seconds": 10.0 }, { "task": "HumanEval/56", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 270, "seconds": 11.3 }, { "task": "HumanEval/60", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 269, "seconds": 11.2 }, { "task": "HumanEval/64", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 497, "seconds": 20.6 }, { "task": "HumanEval/68", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 335, "seconds": 14.7 }, { "task": "HumanEval/72", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 154, "seconds": 7.0 }, { "task": "HumanEval/76", "ok": false, "err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp0mwlmoao/prog.py\", line 21, in <module>\n check(is_simple_power)\n ^^^^^^^^^^^^^^^\nNameError: name 'is_simple_power' is not defined\n", "finish": "length", "leaked": false, "tokens": 900, "seconds": 37.7 }, { "task": "HumanEval/80", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 243, "seconds": 10.2 }, { "task": "HumanEval/84", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 756, "seconds": 31.5 }, { "task": "HumanEval/88", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 266, "seconds": 11.6 }, { "task": "HumanEval/92", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 455, "seconds": 18.9 }, { "task": "HumanEval/96", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 555, "seconds": 23.2 }, { "task": "HumanEval/100", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 353, "seconds": 14.7 }, { "task": "HumanEval/104", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 412, "seconds": 17.1 }, { "task": "HumanEval/108", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 557, "seconds": 23.1 }, { "task": "HumanEval/112", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 288, "seconds": 12.2 }, { "task": "HumanEval/116", "ok": false, "err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp4huqxua3/prog.py\", line 21, in <module>\n check(sort_array)\n ^^^^^^^^^^\nNameError: name 'sort_array' is not defined\n", "finish": "length", "leaked": false, "tokens": 900, "seconds": 37.9 }, { "task": "HumanEval/120", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 331, "seconds": 14.2 }, { "task": "HumanEval/124", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 464, "seconds": 19.8 }, { "task": "HumanEval/128", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 398, "seconds": 16.5 }, { "task": "HumanEval/132", "ok": false, "err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp_1cc4urv/prog.py\", line 25, in <module>\n check(is_nested)\n ^^^^^^^^^\nNameError: name 'is_nested' is not defined\n", "finish": "length", "leaked": false, "tokens": 900, "seconds": 37.8 }, { "task": "HumanEval/136", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 328, "seconds": 13.8 }, { "task": "HumanEval/140", "ok": false, "err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp7bobb2r_/prog.py\", line 16, in <module>\n check(fix_spaces)\n ^^^^^^^^^^\nNameError: name 'fix_spaces' is not defined\n", "finish": "length", "leaked": false, "tokens": 900, "seconds": 37.6 }, { "task": "HumanEval/144", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 238, "seconds": 10.2 }, { "task": "HumanEval/148", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 582, "seconds": 24.6 }, { "task": "HumanEval/152", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 137, "seconds": 6.3 }, { "task": "HumanEval/156", "ok": true, "err": null, "finish": "stop", "leaked": false, "tokens": 507, "seconds": 21.0 } ] }, "gsm8k": { "passed": 48, "total": 50, "leaked": 0, "seconds": 543, "rows": [ { "i": 0, "ok": true, "pred": 18.0, "gold": 18.0, "finish": "stop", "leaked": false, "tokens": 150, "seconds": 6.4 }, { "i": 26, "ok": true, "pred": 243.0, "gold": 243.0, "finish": "stop", "leaked": false, "tokens": 192, "seconds": 8.0 }, { "i": 52, "ok": true, "pred": 15.0, "gold": 15.0, "finish": "stop", "leaked": false, "tokens": 204, "seconds": 8.5 }, { "i": 78, "ok": true, "pred": 6.0, "gold": 6.0, "finish": "stop", "leaked": false, "tokens": 234, "seconds": 9.8 }, { "i": 104, "ok": true, "pred": 500.0, "gold": 500.0, "finish": "stop", "leaked": false, "tokens": 180, "seconds": 7.5 }, { "i": 130, "ok": true, "pred": 30.0, "gold": 30.0, "finish": "stop", "leaked": false, "tokens": 124, "seconds": 5.4 }, { "i": 156, "ok": true, "pred": 1400.0, "gold": 1400.0, "finish": "stop", "leaked": false, "tokens": 164, "seconds": 7.0 }, { "i": 182, "ok": false, "pred": 24.0, "gold": 23.0, "finish": "stop", "leaked": false, "tokens": 851, "seconds": 34.9 }, { "i": 208, "ok": true, "pred": 76.0, "gold": 76.0, "finish": "stop", "leaked": false, "tokens": 139, "seconds": 5.9 }, { "i": 234, "ok": true, "pred": 21.0, "gold": 21.0, "finish": "stop", "leaked": false, "tokens": 613, "seconds": 25.1 }, { "i": 260, "ok": true, "pred": 11.0, "gold": 11.0, "finish": "stop", "leaked": false, "tokens": 210, "seconds": 8.7 }, { "i": 286, "ok": true, "pred": 90.0, "gold": 90.0, "finish": "stop", "leaked": false, "tokens": 148, "seconds": 6.2 }, { "i": 312, "ok": true, "pred": 32.0, "gold": 32.0, "finish": "stop", "leaked": false, "tokens": 113, "seconds": 4.8 }, { "i": 338, "ok": true, "pred": 60.0, "gold": 60.0, "finish": "stop", "leaked": false, "tokens": 186, "seconds": 7.8 }, { "i": 364, "ok": true, "pred": 49.0, "gold": 49.0, "finish": "stop", "leaked": false, "tokens": 115, "seconds": 5.0 }, { "i": 390, "ok": true, "pred": 12.0, "gold": 12.0, "finish": "stop", "leaked": false, "tokens": 295, "seconds": 12.2 }, { "i": 416, "ok": true, "pred": 56.0, "gold": 56.0, "finish": "stop", "leaked": false, "tokens": 185, "seconds": 7.7 }, { "i": 442, "ok": true, "pred": 49.0, "gold": 49.0, "finish": "stop", "leaked": false, "tokens": 229, "seconds": 9.6 }, { "i": 468, "ok": true, "pred": 120.0, "gold": 120.0, "finish": "stop", "leaked": false, "tokens": 225, "seconds": 9.3 }, { "i": 494, "ok": true, "pred": 21.0, "gold": 21.0, "finish": "stop", "leaked": false, "tokens": 702, "seconds": 28.9 }, { "i": 520, "ok": true, "pred": 63.0, "gold": 63.0, "finish": "stop", "leaked": false, "tokens": 182, "seconds": 7.8 }, { "i": 546, "ok": true, "pred": 130.0, "gold": 130.0, "finish": "stop", "leaked": false, "tokens": 382, "seconds": 15.7 }, { "i": 572, "ok": true, "pred": 100.0, "gold": 100.0, "finish": "stop", "leaked": false, "tokens": 217, "seconds": 9.1 }, { "i": 598, "ok": true, "pred": 69.0, "gold": 69.0, "finish": "stop", "leaked": false, "tokens": 189, "seconds": 7.8 }, { "i": 624, "ok": true, "pred": 85.0, "gold": 85.0, "finish": "stop", "leaked": false, "tokens": 192, "seconds": 8.0 }, { "i": 650, "ok": true, "pred": 142.0, "gold": 142.0, "finish": "stop", "leaked": false, "tokens": 259, "seconds": 10.8 }, { "i": 676, "ok": true, "pred": 240.0, "gold": 240.0, "finish": "stop", "leaked": false, "tokens": 260, "seconds": 10.8 }, { "i": 702, "ok": true, "pred": 2800.0, "gold": 2800.0, "finish": "stop", "leaked": false, "tokens": 179, "seconds": 7.6 }, { "i": 728, "ok": true, "pred": 40.0, "gold": 40.0, "finish": "stop", "leaked": false, "tokens": 232, "seconds": 9.6 }, { "i": 754, "ok": true, "pred": 89.0, "gold": 89.0, "finish": "stop", "leaked": false, "tokens": 612, "seconds": 25.3 }, { "i": 780, "ok": true, "pred": 25.0, "gold": 25.0, "finish": "stop", "leaked": false, "tokens": 390, "seconds": 16.2 }, { "i": 806, "ok": true, "pred": 70.0, "gold": 70.0, "finish": "stop", "leaked": false, "tokens": 321, "seconds": 13.3 }, { "i": 832, "ok": true, "pred": 42.0, "gold": 42.0, "finish": "stop", "leaked": false, "tokens": 321, "seconds": 13.1 }, { "i": 858, "ok": true, "pred": 3430.0, "gold": 3430.0, "finish": "stop", "leaked": false, "tokens": 429, "seconds": 17.5 }, { "i": 884, "ok": true, "pred": 50.0, "gold": 50.0, "finish": "stop", "leaked": false, "tokens": 102, "seconds": 4.5 }, { "i": 910, "ok": true, "pred": 700.0, "gold": 700.0, "finish": "stop", "leaked": false, "tokens": 137, "seconds": 5.9 }, { "i": 936, "ok": true, "pred": 250.0, "gold": 250.0, "finish": "stop", "leaked": false, "tokens": 284, "seconds": 11.9 }, { "i": 962, "ok": false, "pred": 600.0, "gold": 1000.0, "finish": "stop", "leaked": false, "tokens": 135, "seconds": 5.8 }, { "i": 988, "ok": true, "pred": 6.0, "gold": 6.0, "finish": "stop", "leaked": false, "tokens": 581, "seconds": 23.8 }, { "i": 1014, "ok": true, "pred": 3.0, "gold": 3.0, "finish": "stop", "leaked": false, "tokens": 381, "seconds": 15.8 }, { "i": 1040, "ok": true, "pred": 736.0, "gold": 736.0, "finish": "stop", "leaked": false, "tokens": 199, "seconds": 8.3 }, { "i": 1066, "ok": true, "pred": 14.0, "gold": 14.0, "finish": "stop", "leaked": false, "tokens": 196, "seconds": 8.2 }, { "i": 1092, "ok": true, "pred": 1080.0, "gold": 1080.0, "finish": "stop", "leaked": false, "tokens": 200, "seconds": 8.4 }, { "i": 1118, "ok": true, "pred": 5.0, "gold": 5.0, "finish": "stop", "leaked": false, "tokens": 268, "seconds": 11.1 }, { "i": 1144, "ok": true, "pred": 34.0, "gold": 34.0, "finish": "stop", "leaked": false, "tokens": 176, "seconds": 7.4 }, { "i": 1170, "ok": true, "pred": 2.0, "gold": 2.0, "finish": "stop", "leaked": false, "tokens": 177, "seconds": 7.4 }, { "i": 1196, "ok": true, "pred": 75.0, "gold": 75.0, "finish": "stop", "leaked": false, "tokens": 191, "seconds": 8.0 }, { "i": 1222, "ok": true, "pred": 350.0, "gold": 350.0, "finish": "stop", "leaked": false, "tokens": 210, "seconds": 8.7 }, { "i": 1248, "ok": true, "pred": 750.0, "gold": 750.0, "finish": "stop", "leaked": false, "tokens": 188, "seconds": 8.0 }, { "i": 1274, "ok": true, "pred": 38.0, "gold": 38.0, "finish": "stop", "leaked": false, "tokens": 211, "seconds": 8.8 } ] } }, "hermes_agent": { "this_build": { "passed": 4, "total": 4, "effort": "medium", "tasks": { "explain": { "ok": true, "seconds": 120.2, "exit_code": 0, "api_calls": 5, "tool_calls": 4, "tools": [ { "name": "terminal", "input": "{\"command\": \"pwd && ..." }, { "name": "terminal", "input": "{\"command\": \"ls -la ..." }, { "name": "terminal", "input": "{\"command\": \"cat /Us..." }, { "name": "terminal", "input": "{\"command\": \"cat /Us..." } ], "final": "Tidepool is a small in-memory job queue where lower-priority numbers are popped first, with ties keeping insertion order (`src/jobqueue.py`). Run the tests with `python3 -m unittest discover -s tests -v` from the project directory \u2014 note, `pop` currently returns `[1]` (the tiebreak counter) instead of the job, so the priority test fails; the fix is `heapq.heappop(self._heap)[2]`.", "tail": "", "stderr": "" }, "nonce": { "ok": true, "seconds": 63.6, "exit_code": 0, "api_calls": 3, "tool_calls": 2, "tools": [ { "name": "search_files", "input": "{\"path\": \"/Users/hos..." }, { "name": "read_file", "input": "{\"path\": \"/Users/hos..." } ], "final": "The access code in notes/secret.txt is **ZX-8E8D2609**.", "tail": "", "stderr": "" }, "fix_bug": { "ok": true, "seconds": 148.8, "exit_code": 0, "api_calls": 8, "tool_calls": 7, "tools": [ { "name": "terminal", "input": "{\"command\": \"git sta..." }, { "name": "terminal", "input": "{\"command\": \"ls -la ..." }, { "name": "terminal", "input": "{\"command\": \"find . ..." }, { "name": "terminal", "input": "{\"command\": \"cat src..." }, { "name": "terminal", "input": "{\"command\": \"python3..." }, { "name": "patch", "input": "{\"new_string\": \" ..." }, { "name": "terminal", "input": "{\"command\": \"python3..." } ], "final": "Fixed: `src/jobqueue.py:16` returned `heappop(...)[1]` \u2014 the insertion counter \u2014 instead of the job at index `[2]`. Tests now pass (2/2, OK).", "tail": "", "stderr": "" }, "write_run": { "ok": true, "seconds": 132.5, "exit_code": 0, "api_calls": 6, "tool_calls": 5, "tools": [ { "name": "terminal", "input": "{\"command\": \"ls -la ..." }, { "name": "terminal", "input": "{\"command\": \"pwd && ..." }, { "name": "terminal", "input": "{\"command\": \"pwd; ls..." }, { "name": "write_file", "input": "{\"content\": \"#!/usr/..." }, { "name": "terminal", "input": "{\"command\": \"python3..." } ], "final": "scripts/wordcount.py prints **20**. It reads README.md and counts whitespace-separated tokens.", "tail": "", "stderr": "" } } }, "original": { "passed": 4, "total": 4, "effort": "medium", "tasks": { "explain": { "ok": true, "seconds": 130.1, "exit_code": 0, "api_calls": 5, "tool_calls": 4, "tools": [ { "name": "terminal", "input": "{\"command\": \"ls -la ..." }, { "name": "terminal", "input": "{\"command\": \"pwd && ..." }, { "name": "terminal", "input": "{\"command\": \"head -4..." }, { "name": "terminal", "input": "{\"command\": \"python3..." } ], "final": "Tidepool is a small in-memory job queue that serves higher-priority jobs first, with insertion order breaking ties. Run the tests with `python3 -m unittest discover -s tests -v` \u2014 but be aware that one test currently fails: `pop()` in `src/jobqueue.py:17` returns the counter (`[1]`) instead of the job (`[2]`), so `test_pop_returns_job` fails (`1 != 'fix outage'`). Want me to fix that line?", "tail": "", "stderr": "" }, "nonce": { "ok": true, "seconds": 75.4, "exit_code": 0, "api_calls": 3, "tool_calls": 2, "tools": [ { "name": "terminal", "input": "{\"command\": \"pwd && ..." }, { "name": "terminal", "input": "{\"command\": \"ls -la ..." } ], "final": "The access code in notes/secret.txt is ZX-F2E46563.", "tail": "", "stderr": "" }, "fix_bug": { "ok": true, "seconds": 170.4, "exit_code": 0, "api_calls": 7, "tool_calls": 7, "tools": [ { "name": "terminal", "input": "{\"command\": \"pwd && ..." }, { "name": "terminal", "input": "{\"command\": \"cd /Use..." }, { "name": "terminal", "input": "{\"command\": \"cd /Use..." }, { "name": "read_file", "input": "{\"path\": \"/Users/hos..." }, { "name": "read_file", "input": "{\"path\": \"/Users/hos..." }, { "name": "terminal", "input": "{\"command\": \"cd /Use..." }, { "name": "terminal", "input": "{\"command\": \"cd /Use..." } ], "final": "Fixed. The bug was in `src/jobqueue.py:16`: `pop()` returned heap index `[1]` (the insertion counter) instead of `[2]` (the job). One-line change to `heappop(...)[2]`.\n\nTests now pass:\n```\ntest_len ... ok\ntest_pop_returns_job ... ok\nRan 2 tests \u2014 OK\n```\n\nNote: there's a `notes/secret.txt` in the scratch directory which I deliberately did not read or use.", "tail": "", "stderr": "" }, "write_run": { "ok": true, "seconds": 168.6, "exit_code": 0, "api_calls": 6, "tool_calls": 5, "tools": [ { "name": "terminal", "input": "{\"command\": \"ls -la ..." }, { "name": "terminal", "input": "{\"command\": \"pwd && ..." }, { "name": "terminal", "input": "{\"command\": \"ls /Use..." }, { "name": "write_file", "input": "{\"content\": \"#!/usr/..." }, { "name": "terminal", "input": "{\"command\": \"python3..." } ], "final": "`scripts/wordcount.py` created and run. It printed **20**.\n\n(The README holds 20 whitespace-separated tokens: `# Tidepool A small in-memory job queue with priorities. ## Run the tests python3 -m unittest discover -s tests -v`.)", "tail": "", "stderr": "" } } } }, "benchmark": { "this_build": { "load_seconds": 0.9, "weights_in_memory_gb": 9.07, "rows": [ { "case": "short", "draft": false, "prompt_tokens": 28, "generation_tokens": 256, "prompt_tps": 99.6, "generation_tps": 33.44, "peak_gb": 9.83, "wall_s": 7.9 }, { "case": "short", "draft": true, "prompt_tokens": 28, "generation_tokens": 256, "prompt_tps": 108.6, "generation_tps": 19.24, "peak_gb": 10.14, "wall_s": 13.6 }, { "case": "ctx1k", "draft": false, "prompt_tokens": 1024, "generation_tokens": 256, "prompt_tps": 429.6, "generation_tps": 29.66, "peak_gb": 12.62, "wall_s": 11.0 }, { "case": "ctx1k", "draft": true, "prompt_tokens": 1024, "generation_tokens": 256, "prompt_tps": 441.2, "generation_tps": 40.2, "peak_gb": 12.74, "wall_s": 8.7 }, { "case": "ctx4k", "draft": false, "prompt_tokens": 4023, "generation_tokens": 256, "prompt_tps": 402.9, "generation_tps": 28.56, "peak_gb": 14.07, "wall_s": 19.0 }, { "case": "ctx4k", "draft": true, "prompt_tokens": 4023, "generation_tokens": 256, "prompt_tps": 393.1, "generation_tps": 23.48, "peak_gb": 14.23, "wall_s": 21.2 } ], "mlx": "0.32.4" }, "original": { "load_seconds": 4.9, "weights_in_memory_gb": 18.19, "rows": [ { "case": "short", "draft": false, "prompt_tokens": 28, "generation_tokens": 256, "prompt_tps": 91.1, "generation_tps": 25.91, "peak_gb": 20.62, "wall_s": 10.2 }, { "case": "short", "draft": true, "prompt_tokens": 28, "generation_tokens": 256, "prompt_tps": 100.4, "generation_tps": 19.46, "peak_gb": 20.98, "wall_s": 13.5 }, { "case": "ctx1k", "draft": false, "prompt_tokens": 1024, "generation_tokens": 256, "prompt_tps": 429.0, "generation_tps": 23.25, "peak_gb": 23.46, "wall_s": 13.4 }, { "case": "ctx1k", "draft": true, "prompt_tokens": 1024, "generation_tokens": 256, "prompt_tps": 434.4, "generation_tps": 31.77, "peak_gb": 23.59, "wall_s": 10.4 }, { "case": "ctx4k", "draft": false, "prompt_tokens": 4023, "generation_tokens": 256, "prompt_tps": 379.1, "generation_tps": 22.33, "peak_gb": 24.92, "wall_s": 22.1 }, { "case": "ctx4k", "draft": true, "prompt_tokens": 4023, "generation_tokens": 256, "prompt_tps": 377.1, "generation_tps": 22.02, "peak_gb": 25.07, "wall_s": 22.3 } ], "mlx": "0.32.4" } }, "memory_and_image_check": { "this_build": { "weights_in_memory_gib": 9.07, "rows": [ { "prompt_tokens": 224, "prefill_step": 1024, "prompt_tps": 337.8, "generation_tps": 33.27, "peak_gib": 9.8 }, { "prompt_tokens": 4023, "prefill_step": 1024, "prompt_tps": 406.3, "generation_tps": 28.81, "peak_gib": 13.38 }, { "prompt_tokens": 4023, "prefill_step": 256, "prompt_tps": 362.9, "generation_tps": 28.94, "peak_gib": 10.6 }, { "prompt_tokens": 8024, "prefill_step": 256, "prompt_tps": 348.0, "generation_tps": 27.42, "peak_gib": 10.81 }, { "prompt_tokens": 16024, "prefill_step": 256, "prompt_tps": 332.2, "generation_tps": 24.67, "peak_gib": 11.25 } ], "vision": [ { "image": "data/images/cat.jpeg", "question": "What animal is in this image, and where is it? One sentence.", "answer": "ct: animal has cat-like face, short legs, sandy/gray fur, snow dusting, ears with tufts. It resembles a lynx/bobcat. Location: on snow-covered ground/platform with birch logs behind. One sentence. Ensure no overclaim? Say \"A lynx (bobcat-like cat) is walking on snow-covered ground in front of birch logs.\" That's one sentence.A lynx-like cat is walking on snow-covered ground in front of birch logs." }, { "image": "data/images/chart.png", "question": "Which quarter sold the most units, how many, and what is the order reference?", "answer": "The user wants answer based on chart. Need inspect image. Chart title: QUARTERLY UNITS SOLD - ORDER REF 4821. Bars: Q1 120, Q2 185, Q3 90, Q4 240. Most units Q4 with 240, order reference 4821. Need answer concise. Ensure mention order ref. No extra.Q4 sold the most units: 240. Order reference: 4821." } ] }, "original": { "weights_in_memory_gib": 18.19, "rows": [ { "prompt_tokens": 224, "prefill_step": 1024, "prompt_tps": 302.1, "generation_tps": 25.82, "peak_gib": 20.27 }, { "prompt_tokens": 4023, "prefill_step": 1024, "prompt_tps": 403.6, "generation_tps": 22.82, "peak_gib": 24.23 }, { "prompt_tokens": 4023, "prefill_step": 256, "prompt_tps": 349.2, "generation_tps": 22.83, "peak_gib": 21.37 }, { "prompt_tokens": 8024, "prefill_step": 256, "prompt_tps": 323.8, "generation_tps": 21.69, "peak_gib": 21.63 }, { "prompt_tokens": 16024, "prefill_step": 256, "prompt_tps": 310.8, "generation_tps": 19.9, "peak_gib": 22.06 } ], "vision": [ { "image": "data/images/cat.jpeg", "question": "What animal is in this image, and where is it? One sentence.", "answer": "at (snow leopard-like) or perhaps a lynx/bobcat? It looks like a Pallas's cat (manul) - stocky, short legs, thick fur, speckled face. It's walking on snow with a wooden/log fence background. Need answer concise.\n\nAnimal: Pallas's cat (manul). Location: on snowy ground in front of a wooden fence/log wall. One sentence.A Pallas\u2019s cat (manul) is walking on snowy ground in front of a wooden log fence." }, { "image": "data/images/chart.png", "question": "Which quarter sold the most units, how many, and what is the order reference?", "answer": "The user is asking about a bar chart. Let me read the values:\n- Q1: 120\n- Q2: 185\n- Q3: 90\n- Q4: 240\n\nThe highest is Q4 with 240 units. The order reference is 4821 (from \"QUARTERLY UNITS SOLD - ORDER REF 4821\").Q4 sold the most units, with 240 units. The order reference is 4821." } ] } }, "cached_decoding_vs_full_forward_original": { "bf16_chunk128": { "tokens": 4001, "sequences": 10, "max_abs_logit_diff": 22.842241287231445, "kl_mean": 0.048451148127050876, "kl_max": 13.973586082458496, "nll_full_forward": 0.7997520344759697, "nll_cached_steps": 0.7905213371034683, "top1_agreement": 0.970757310672332, "seconds": 211 }, "bf16_chunk2048": { "tokens": 4001, "sequences": 10, "max_abs_logit_diff": 21.5530948638916, "kl_mean": 0.04029263491467409, "kl_max": 18.789379119873047, "nll_full_forward": 0.7997520344759697, "nll_cached_steps": 0.7888766982143625, "top1_agreement": 0.970757310672332, "seconds": 209 }, "fp32_chunk128": { "tokens": 1192, "sequences": 4, "max_abs_logit_diff": 15.65640640258789, "kl_mean": 0.006313716402616854, "kl_max": 2.067986488342285, "nll_full_forward": 1.674901450240372, "nll_cached_steps": 1.6847560293722472, "top1_agreement": 0.9966442953020134, "seconds": 80 }, "fp32_chunk2048": { "tokens": 1192, "sequences": 4, "max_abs_logit_diff": 15.656400680541992, "kl_mean": 0.006313734596712877, "kl_max": 2.067990303039551, "nll_full_forward": 1.674901450240372, "nll_cached_steps": 1.6847562597902028, "top1_agreement": 0.9966442953020134, "seconds": 78 } }, "reference_parity": { "description": "Teacher-forced logits on identical token ids. Reference: transformers PR 51050 (commit 6173a2f6) on CPU, expert weights dequantized by the reference op. MLX: mlx-vlm main 111ac668, mlx 0.32.4, M4 Pro.", "mlx_fp32_vs_torch_fp32": { "image_cat": { "tokens": 492, "max_abs_logit_diff": 0.44274044036865234, "mean_abs_logit_diff": 0.00039586810658722305, "kl_mean": 7.587283164404417e-06, "kl_max": 0.003581067210598087, "top1_agreement": 1.0, "next_token_acc_ref": 0.054989816700611, "next_token_acc_test": 0.054989816700611, "nll_ref": 20.089199532342388, "nll_test": 20.089406827516832 }, "image_chart": { "tokens": 356, "max_abs_logit_diff": 8.646623849868774, "mean_abs_logit_diff": 0.042977310928109254, "kl_mean": 0.004497380610732641, "kl_max": 0.33672980618712545, "top1_agreement": 0.9719101123595506, "next_token_acc_ref": 0.09014084507042254, "next_token_acc_test": 0.09014084507042254, "nll_ref": 17.659265003278247, "nll_test": 17.646260290588184 }, "long_context": { "tokens": 1353, "max_abs_logit_diff": 7.7301411628723145, "mean_abs_logit_diff": 0.015888066121287594, "kl_mean": 1.7773926378733907e-05, "kl_max": 0.010085841474386945, "top1_agreement": 1.0, "next_token_acc_ref": 0.9060650887573964, "next_token_acc_test": 0.9060650887573964, "nll_ref": 0.5817908920328585, "nll_test": 0.5817055003276868 }, "multilingual": { "tokens": 65, "max_abs_logit_diff": 0.00010979175567626953, "mean_abs_logit_diff": 8.978016029090604e-06, "kl_mean": 2.0563244512179743e-11, "kl_max": 2.554999550892132e-10, "top1_agreement": 1.0, "next_token_acc_ref": 0.5625, "next_token_acc_test": 0.5625, "nll_ref": 3.7906516032194206, "nll_test": 3.7906535542633177 }, "short_chat": { "tokens": 37, "max_abs_logit_diff": 0.0001277923583984375, "mean_abs_logit_diff": 9.456596459282991e-06, "kl_mean": 2.240959362721911e-11, "kl_max": 1.7461242067926594e-10, "top1_agreement": 1.0, "next_token_acc_ref": 0.5277777777777778, "next_token_acc_test": 0.5277777777777778, "nll_ref": 5.443080670467236, "nll_test": 5.443082489456533 }, "thinking_chat": { "tokens": 102, "max_abs_logit_diff": 0.00010538101196289062, "mean_abs_logit_diff": 9.532523792852414e-06, "kl_mean": 1.8364102705662738e-11, "kl_max": 4.242171022827198e-10, "top1_agreement": 1.0, "next_token_acc_ref": 0.6138613861386139, "next_token_acc_test": 0.6138613861386139, "nll_ref": 3.7741031649505654, "nll_test": 3.774103348220411 }, "tool_roundtrip": { "tokens": 329, "max_abs_logit_diff": 0.000240325927734375, "mean_abs_logit_diff": 1.1768514219006727e-05, "kl_mean": 2.007265502036474e-11, "kl_max": 1.3455370426613493e-09, "top1_agreement": 1.0, "next_token_acc_ref": 0.7134146341463414, "next_token_acc_test": 0.7134146341463414, "nll_ref": 2.7811447828381786, "nll_test": 2.7811444905435425 } }, "torch_bf16_vs_torch_fp32": { "image_cat": { "tokens": 492, "max_abs_logit_diff": 20.917954444885254, "mean_abs_logit_diff": 0.4790812881133246, "kl_mean": 0.2554885655286341, "kl_max": 11.36766312077503, "top1_agreement": 0.8780487804878049, "next_token_acc_ref": 0.054989816700611, "next_token_acc_test": 0.05702647657841141, "nll_ref": 20.089199532342388, "nll_test": 20.049141792477794 }, "image_chart": { "tokens": 356, "max_abs_logit_diff": 23.090723991394043, "mean_abs_logit_diff": 0.6762670648069056, "kl_mean": 0.5612026261553928, "kl_max": 13.180007741732275, "top1_agreement": 0.7443820224719101, "next_token_acc_ref": 0.09014084507042254, "next_token_acc_test": 0.09014084507042254, "nll_ref": 17.659265003278247, "nll_test": 17.667298816635764 }, "long_context": { "tokens": 1353, "max_abs_logit_diff": 24.625191688537598, "mean_abs_logit_diff": 1.5311537717340122, "kl_mean": 0.22785581219171008, "kl_max": 16.08681830122339, "top1_agreement": 0.9556541019955654, "next_token_acc_ref": 0.9060650887573964, "next_token_acc_test": 0.9038461538461539, "nll_ref": 0.5817908920328585, "nll_test": 0.5574441898794273 }, "multilingual": { "tokens": 65, "max_abs_logit_diff": 4.211472511291504, "mean_abs_logit_diff": 0.2116919903498743, "kl_mean": 0.008501311125484017, "kl_max": 0.08177949573595175, "top1_agreement": 0.9692307692307692, "next_token_acc_ref": 0.5625, "next_token_acc_test": 0.578125, "nll_ref": 3.7906516032194206, "nll_test": 3.8383234493145535 }, "short_chat": { "tokens": 37, "max_abs_logit_diff": 3.2938833236694336, "mean_abs_logit_diff": 0.23400576493509498, "kl_mean": 0.013311820955924617, "kl_max": 0.12193033467452069, "top1_agreement": 0.972972972972973, "next_token_acc_ref": 0.5277777777777778, "next_token_acc_test": 0.5277777777777778, "nll_ref": 5.443080670467236, "nll_test": 5.375631014013916 }, "thinking_chat": { "tokens": 102, "max_abs_logit_diff": 14.65269660949707, "mean_abs_logit_diff": 0.26605868399973925, "kl_mean": 0.0358142972599525, "kl_max": 2.5301440916711586, "top1_agreement": 0.9509803921568627, "next_token_acc_ref": 0.6138613861386139, "next_token_acc_test": 0.6039603960396039, "nll_ref": 3.7741031649505654, "nll_test": 3.840478707203926 }, "tool_roundtrip": { "tokens": 329, "max_abs_logit_diff": 23.453279495239258, "mean_abs_logit_diff": 0.6738596847750378, "kl_mean": 0.11465851963454897, "kl_max": 12.569975177111543, "top1_agreement": 0.9361702127659575, "next_token_acc_ref": 0.7134146341463414, "next_token_acc_test": 0.7225609756097561, "nll_ref": 2.7811447828381786, "nll_test": 2.7678284988449815 } }, "mlx_bf16_vs_torch_fp32": { "image_cat": { "tokens": 492, "max_abs_logit_diff": 25.776443481445312, "mean_abs_logit_diff": 0.5739572311203393, "kl_mean": 0.32945401742291003, "kl_max": 13.424443716518757, "top1_agreement": 0.8536585365853658, "next_token_acc_ref": 0.054989816700611, "next_token_acc_test": 0.059063136456211814, "nll_ref": 20.089199532342388, "nll_test": 20.10524947315169 }, "image_chart": { "tokens": 356, "max_abs_logit_diff": 22.61000919342041, "mean_abs_logit_diff": 0.5653393714458804, "kl_mean": 0.356097592533074, "kl_max": 7.309836100553708, "top1_agreement": 0.7556179775280899, "next_token_acc_ref": 0.09014084507042254, "next_token_acc_test": 0.09014084507042254, "nll_ref": 17.659265003278247, "nll_test": 17.608667050969412 }, "long_context": { "tokens": 1353, "max_abs_logit_diff": 24.45859718322754, "mean_abs_logit_diff": 1.473904154273382, "kl_mean": 0.19963105957870853, "kl_max": 15.416439084029014, "top1_agreement": 0.9645232815964523, "next_token_acc_ref": 0.9060650887573964, "next_token_acc_test": 0.9090236686390533, "nll_ref": 0.5817908920328585, "nll_test": 0.5440008410165926 }, "multilingual": { "tokens": 65, "max_abs_logit_diff": 2.285825252532959, "mean_abs_logit_diff": 0.19048134856934984, "kl_mean": 0.007670106536798406, "kl_max": 0.22094075094875654, "top1_agreement": 0.9846153846153847, "next_token_acc_ref": 0.5625, "next_token_acc_test": 0.5625, "nll_ref": 3.7906516032194206, "nll_test": 3.793026564653868 }, "short_chat": { "tokens": 37, "max_abs_logit_diff": 4.4355528354644775, "mean_abs_logit_diff": 0.22052635319363867, "kl_mean": 0.00854530879380758, "kl_max": 0.07499803025163768, "top1_agreement": 0.9459459459459459, "next_token_acc_ref": 0.5277777777777778, "next_token_acc_test": 0.5277777777777778, "nll_ref": 5.443080670467236, "nll_test": 5.343368736261641 }, "thinking_chat": { "tokens": 102, "max_abs_logit_diff": 14.612149477005005, "mean_abs_logit_diff": 0.2570170134252102, "kl_mean": 0.03432711815246316, "kl_max": 2.6583591721311954, "top1_agreement": 0.9705882352941176, "next_token_acc_ref": 0.6138613861386139, "next_token_acc_test": 0.6039603960396039, "nll_ref": 3.7741031649505654, "nll_test": 3.8346220063091554 }, "tool_roundtrip": { "tokens": 329, "max_abs_logit_diff": 20.791677236557007, "mean_abs_logit_diff": 0.7246150221283114, "kl_mean": 0.13842127199136015, "kl_max": 7.050130603024795, "top1_agreement": 0.9361702127659575, "next_token_acc_ref": 0.7134146341463414, "next_token_acc_test": 0.698170731707317, "nll_ref": 2.7811447828381786, "nll_test": 2.8992298776234766 } }, "mlx_bf16_vs_torch_bf16": { "image_cat": { "tokens": 492, "max_abs_logit_diff": 23.71006679534912, "mean_abs_logit_diff": 0.5271237628672776, "kl_mean": 0.2635479704693753, "kl_max": 6.784772286464859, "top1_agreement": 0.8597560975609756, "next_token_acc_ref": 0.05702647657841141, "next_token_acc_test": 0.059063136456211814, "nll_ref": 20.049141792477794, "nll_test": 20.10524947315169 }, "image_chart": { "tokens": 356, "max_abs_logit_diff": 24.48257350921631, "mean_abs_logit_diff": 0.7330006307927256, "kl_mean": 0.6029482114534772, "kl_max": 10.911963588562358, "top1_agreement": 0.7247191011235955, "next_token_acc_ref": 0.09014084507042254, "next_token_acc_test": 0.09014084507042254, "nll_ref": 17.667298816635764, "nll_test": 17.608667050969412 }, "long_context": { "tokens": 1353, "max_abs_logit_diff": 24.829045295715332, "mean_abs_logit_diff": 1.6162585746277247, "kl_mean": 0.2801612289778648, "kl_max": 25.589695259544296, "top1_agreement": 0.9615668883961567, "next_token_acc_ref": 0.9038461538461539, "next_token_acc_test": 0.9090236686390533, "nll_ref": 0.5574441898794273, "nll_test": 0.5440008410165926 }, "multilingual": { "tokens": 65, "max_abs_logit_diff": 3.5512681007385254, "mean_abs_logit_diff": 0.19969853793115186, "kl_mean": 0.005015380128086673, "kl_max": 0.08439173889412566, "top1_agreement": 0.9846153846153847, "next_token_acc_ref": 0.578125, "next_token_acc_test": 0.5625, "nll_ref": 3.8383234493145535, "nll_test": 3.793026564653868 }, "short_chat": { "tokens": 37, "max_abs_logit_diff": 3.655950903892517, "mean_abs_logit_diff": 0.2583573715704287, "kl_mean": 0.015552908775126063, "kl_max": 0.10747477572936717, "top1_agreement": 0.972972972972973, "next_token_acc_ref": 0.5277777777777778, "next_token_acc_test": 0.5277777777777778, "nll_ref": 5.375631014013916, "nll_test": 5.343368736261641 }, "thinking_chat": { "tokens": 102, "max_abs_logit_diff": 4.207983016967773, "mean_abs_logit_diff": 0.19963685965760844, "kl_mean": 0.0081508028170958, "kl_max": 0.2828682003566698, "top1_agreement": 0.9803921568627451, "next_token_acc_ref": 0.6039603960396039, "next_token_acc_test": 0.6039603960396039, "nll_ref": 3.840478707203926, "nll_test": 3.8346220063091554 }, "tool_roundtrip": { "tokens": 329, "max_abs_logit_diff": 22.754143714904785, "mean_abs_logit_diff": 0.6942348508134669, "kl_mean": 0.1607668713797337, "kl_max": 7.755121918082823, "top1_agreement": 0.9240121580547113, "next_token_acc_ref": 0.7225609756097561, "next_token_acc_test": 0.698170731707317, "nll_ref": 2.7678284988449815, "nll_test": 2.8992298776234766 } }, "notes": { "routing_flips": "Where fp32 runs differ, hidden states agree to ~1e-5 relative until a single token's top-8 expert choice flips at a near-tie: long_context first differs at layer visit 32 token 859 (8th/9th routing score margin 1.9e-6), then visit 34 token 1275 (3.5e-6) and visit 39 token 241 (3.1e-6). image_chart first differs at visit 20 token 77.", "dequantization": "MLX dequantization of NVFP4 (layers 0, 13) and MXFP8 (layer 22) experts equals the reference op bit for bit (max abs diff 0.0).", "preprocessing": "Image patches match the reference to 1.2e-7; image grids and prompt token ids are identical; chat-template token ids are identical on all text cases." } }, "per_layer_output_error": { "0": { "w13:3:64": { "gptq": 0.012887698906057739, "rtn": 0.028264907912936545 }, "w13:2:64": { "gptq": 0.049336151572674375, "rtn": 0.12750708317950427 }, "w2:3:64": { "gptq": 0.015656777931638093, "rtn": 0.03319389076493519 }, "w2:2:64": { "gptq": 0.06048335714614615, "rtn": 0.14741300019398665 } }, "1": { "w13:3:64": { "gptq": 0.011802700054363778, "rtn": 0.025790537433406653 }, "w13:2:64": { "gptq": 0.04516684536862453, "rtn": 0.11663405471536598 }, "w2:3:64": { "gptq": 0.01641453559978002, "rtn": 0.03318410301470986 }, "w2:2:64": { "gptq": 0.06349666359881456, "rtn": 0.14814831675886653 } }, "2": { "w13:3:64": { "gptq": 0.011815724312573465, "rtn": 0.026115071002306627 }, "w13:2:64": { "gptq": 0.045276587990840216, "rtn": 0.11779761737034271 }, "w2:3:64": { "gptq": 0.017178491960464465, "rtn": 0.03435173635451321 }, "w2:2:64": { "gptq": 0.06630938304581857, "rtn": 0.15267030548535696 } }, "3": { "w13:3:64": { "gptq": 0.012184065107500825, "rtn": 0.0264636905217352 }, "w13:2:64": { "gptq": 0.0466883827039428, "rtn": 0.11962121154823789 }, "w2:3:64": { "gptq": 0.018285621104312873, "rtn": 0.03451856381783443 }, "w2:2:64": { "gptq": 0.07047653555282281, "rtn": 0.15371292378272855 } }, "4": { "w13:3:64": { "gptq": 0.012464210034104514, "rtn": 0.026564464601376334 }, "w13:2:64": { "gptq": 0.04778888899832448, "rtn": 0.11984409260700374 }, "w2:3:64": { "gptq": 0.01901098009817418, "rtn": 0.03473462659622553 }, "w2:2:64": { "gptq": 0.07319164829533384, "rtn": 0.15440812657866768 } }, "5": { "w13:3:64": { "gptq": 0.01145844188891433, "rtn": 0.025238676157675728 }, "w13:2:64": { "gptq": 0.04380464005824843, "rtn": 0.11382083683444219 }, "w2:3:64": { "gptq": 0.016609748305823607, "rtn": 0.033314921714924374 }, "w2:2:64": { "gptq": 0.06413552679105854, "rtn": 0.14833856203953347 } }, "6": { "w13:3:64": { "gptq": 0.008630106207050013, "rtn": 0.021677847331551762 }, "w13:2:64": { "gptq": 0.03336440886887183, "rtn": 0.098471811560027 }, "w2:3:64": { "gptq": 0.017297109454220555, "rtn": 0.0329146712577646 }, "w2:2:64": { "gptq": 0.06655221483005157, "rtn": 0.1467128978713485 } }, "7": { "w13:3:64": { "gptq": 0.009308482229105724, "rtn": 0.02300102947982573 }, "w13:2:64": { "gptq": 0.036142309747491115, "rtn": 0.10441029484923274 }, "w2:3:64": { "gptq": 0.018696131483273832, "rtn": 0.03377960973702008 }, "w2:2:64": { "gptq": 0.07189859819108688, "rtn": 0.15030866515220506 } }, "8": { "w13:3:64": { "gptq": 0.00970140273000972, "rtn": 0.023736662945633195 }, "w13:2:64": { "gptq": 0.03756622131463952, "rtn": 0.10764342694219707 }, "w2:3:64": { "gptq": 0.01893052109671973, "rtn": 0.03455564228558328 }, "w2:2:64": { "gptq": 0.07267839934317306, "rtn": 0.1539128401762423 } }, "9": { "w13:3:64": { "gptq": 0.009544249372379274, "rtn": 0.024004255634757873 }, "w13:2:64": { "gptq": 0.036904315033598055, "rtn": 0.10858139577985748 }, "w2:3:64": { "gptq": 0.01861156179149983, "rtn": 0.035172145497160565 }, "w2:2:64": { "gptq": 0.07143131999568134, "rtn": 0.1562987522341291 } }, "10": { "w13:3:64": { "gptq": 0.010136449403879732, "rtn": 0.02447905853963705 }, "w13:2:64": { "gptq": 0.039104749307393195, "rtn": 0.11082593899550983 }, "w2:3:64": { "gptq": 0.019276758340826466, "rtn": 0.035136919162328153 }, "w2:2:64": { "gptq": 0.0736179475964238, "rtn": 0.15600411319862784 } }, "11": { "w13:3:64": { "gptq": 0.00973847975737354, "rtn": 0.024046968008915155 }, "w13:2:64": { "gptq": 0.037532198495015906, "rtn": 0.10869107576379265 }, "w2:3:64": { "gptq": 0.01938698195966324, "rtn": 0.03621272793928649 }, "w2:2:64": { "gptq": 0.07390217281140692, "rtn": 0.16105901638744027 } }, "12": { "w13:3:64": { "gptq": 0.009194471343113115, "rtn": 0.02273684322111143 }, "w13:2:64": { "gptq": 0.03564779480386397, "rtn": 0.10313857330143292 }, "w2:3:64": { "gptq": 0.02008837326579736, "rtn": 0.03548716477388728 }, "w2:2:64": { "gptq": 0.07655177852173438, "rtn": 0.15761541023604683 } }, "13": { "w13:3:64": { "gptq": 0.01044786069277626, "rtn": 0.024413591314365678 }, "w13:2:64": { "gptq": 0.040381324838175386, "rtn": 0.11050714810773753 }, "w2:3:64": { "gptq": 0.021217173862265844, "rtn": 0.036407905251155226 }, "w2:2:64": { "gptq": 0.08071875472901732, "rtn": 0.16188429254535236 } }, "14": { "w13:3:64": { "gptq": 0.010248876740760167, "rtn": 0.024586683531814035 }, "w13:2:64": { "gptq": 0.03954587427349752, "rtn": 0.1111096092252099 }, "w2:3:64": { "gptq": 0.020788302220417228, "rtn": 0.037157275436764226 }, "w2:2:64": { "gptq": 0.07906254882958245, "rtn": 0.1651486767835523 } }, "15": { "w13:3:64": { "gptq": 0.010022847419748954, "rtn": 0.024274339397861455 }, "w13:2:64": { "gptq": 0.038597981704112036, "rtn": 0.1097851394777376 }, "w2:3:64": { "gptq": 0.020948054143987628, "rtn": 0.03831832542731186 }, "w2:2:64": { "gptq": 0.07954158805661754, "rtn": 0.17017675119167355 } }, "16": { "w13:3:64": { "gptq": 0.008498339936465907, "rtn": 0.022542785906383197 }, "w13:2:64": { "gptq": 0.03263134778631234, "rtn": 0.10185568361359731 }, "w2:3:64": { "gptq": 0.020534073751669327, "rtn": 0.04082283626413468 }, "w2:2:64": { "gptq": 0.07804667725216022, "rtn": 0.1804656980758368 } }, "17": { "w13:3:64": { "gptq": 0.009502487563696189, "rtn": 0.02370556959794839 }, "w13:2:64": { "gptq": 0.036216959333699165, "rtn": 0.10700858826753805 }, "w2:3:64": { "gptq": 0.020227043716426002, "rtn": 0.04356292902064836 }, "w2:2:64": { "gptq": 0.07657759032425712, "rtn": 0.1907152855875201 } }, "18": { "w13:3:64": { "gptq": 0.012731135847359173, "rtn": 0.02782251663498656 }, "w13:2:64": { "gptq": 0.048424019434402, "rtn": 0.1253200590484997 }, "w2:3:64": { "gptq": 0.01842322559609543, "rtn": 0.03752281212883477 }, "w2:2:64": { "gptq": 0.0701647769331662, "rtn": 0.16632908263507784 } }, "19": { "w13:3:64": { "gptq": 0.013259984145735863, "rtn": 0.02826715333399623 }, "w13:2:64": { "gptq": 0.050449675440751234, "rtn": 0.12738783863054207 }, "w2:3:64": { "gptq": 0.019272166405000277, "rtn": 0.03717022037782505 }, "w2:2:64": { "gptq": 0.07354566699400454, "rtn": 0.1654829052868103 } }, "20": { "w13:4:64": { "gptq": 0.0035938307581711206, "rtn": 0.007118168637384198 }, "w13:3:64": { "gptq": 0.014105890797743957, "rtn": 0.03093346592544241 }, "w13:2:64": { "gptq": 0.053300674122841715, "rtn": 0.13285205393862426 }, "w2:4:64": { "gptq": 0.004915546344155547, "rtn": 0.008680694219172532 }, "w2:3:64": { "gptq": 0.019311162271728324, "rtn": 0.03772180529423481 }, "w2:2:64": { "gptq": 0.07340830898059576, "rtn": 0.1599916808050238 } }, "21": { "w13:4:64": { "gptq": 0.0037124126419644885, "rtn": 0.00734338705895674 }, "w13:3:64": { "gptq": 0.014569280031334947, "rtn": 0.03188605416034486 }, "w13:2:64": { "gptq": 0.055007637306976394, "rtn": 0.1366433478861042 }, "w2:4:64": { "gptq": 0.005143603726133695, "rtn": 0.008794992797545474 }, "w2:3:64": { "gptq": 0.02019433043353457, "rtn": 0.03824136567061181 }, "w2:2:64": { "gptq": 0.07661496879716495, "rtn": 0.16205056792549807 } }, "22": { "w13:4:64": { "gptq": 0.0035216957335476614, "rtn": 0.00713298095857926 }, "w13:3:64": { "gptq": 0.01381453591307655, "rtn": 0.030957691522989786 }, "w13:2:64": { "gptq": 0.052077855686062643, "rtn": 0.13301789032382183 }, "w2:4:64": { "gptq": 0.004941542055169573, "rtn": 0.008916477635004198 }, "w2:3:64": { "gptq": 0.019347042681703397, "rtn": 0.038785959177411136 }, "w2:2:64": { "gptq": 0.073295078682088, "rtn": 0.1641994122618597 } }, "23": { "w13:4:64": { "gptq": 0.003139279646588113, "rtn": 0.006451814291554056 }, "w13:3:64": { "gptq": 0.012311763717043433, "rtn": 0.028066662725762536 }, "w13:2:64": { "gptq": 0.04638598758334907, "rtn": 0.12017644349661992 }, "w2:4:64": { "gptq": 0.005102806170440476, "rtn": 0.009787645443707936 }, "w2:3:64": { "gptq": 0.019850079171067476, "rtn": 0.04258285916317973 }, "w2:2:64": { "gptq": 0.07480900112960819, "rtn": 0.17718135892908898 } } } }
{
"held_out_loss": {
"native": {
"tokens": 40806,
"nll": 1.4480010462071342,
"acc": 0.6699995098759987,
"seconds": 115,
"peak_gb": 22.03,
"weights_gib": null,
"activations": "bf16",
"actquant": null,
"chat": {
"tokens": 6597,
"nll": 0.4662371408908642
},
"code": {
"tokens": 8877,
"nll": 0.8892847396130361
},
"wikitext": {
"tokens": 25332,
"nll": 1.8994624837962406
}
},
"2bit": {
"tokens": 40806,
"nll": 1.5432901726466433,
"acc": 0.6456893594079302,
"seconds": 112,
"peak_gb": 11.49,
"weights_gib": null,
"activations": "bf16",
"actquant": null,
"chat": {
"tokens": 6597,
"nll": 0.6182796457454582
},
"code": {
"tokens": 8877,
"nll": 0.967472937161608
},
"wikitext": {
"tokens": 25332,
"nll": 1.9859644204505187
}
},
"3bit_round_to_nearest": {
"tokens": 40806,
"nll": 1.4701619848213856,
"acc": 0.6639709846591187,
"seconds": 113,
"peak_gb": 19.63,
"weights_gib": 11.910704612731934,
"activations": "bf16",
"actquant": null,
"chat": {
"tokens": 6597,
"nll": 0.49830741327206834
},
"code": {
"tokens": 8877,
"nll": 0.9093072548956838
},
"wikitext": {
"tokens": 25332,
"nll": 1.919792177702378
}
}
},
"numerics_experiments": {
"native_bf16_actquant": {
"tokens": 40806,
"nll": 1.4507249015946835,
"acc": 0.6688967308729108,
"seconds": 151,
"peak_gb": 22.22,
"weights_gib": null,
"activations": "bf16",
"actquant": [
"input",
"hidden"
],
"chat": {
"tokens": 6597,
"nll": 0.470449288324712
},
"code": {
"tokens": 8877,
"nll": 0.8916940795444644
},
"wikitext": {
"tokens": 25332,
"nll": 1.9019089702857381
}
},
"native_fp32": {
"tokens": 40806,
"nll": 1.4489235311713002,
"acc": 0.6690682742733912,
"seconds": 134,