Image-Text-to-Text
MLX
Safetensors
English
inkling2_loop
mlx-vlm
inkling2
Mixture of Experts
quantized
gptq
apple-silicon
tool-calling
agents
conversational
8-bit precision
Instructions to use VisualInference/Inkling-2-Mini-MLX-3bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use VisualInference/Inkling-2-Mini-MLX-3bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("VisualInference/Inkling-2-Mini-MLX-3bit") config = load_config("VisualInference/Inkling-2-Mini-MLX-3bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use VisualInference/Inkling-2-Mini-MLX-3bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-3bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "VisualInference/Inkling-2-Mini-MLX-3bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use VisualInference/Inkling-2-Mini-MLX-3bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-3bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default VisualInference/Inkling-2-Mini-MLX-3bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use VisualInference/Inkling-2-Mini-MLX-3bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-3bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "VisualInference/Inkling-2-Mini-MLX-3bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download extras/results.json from VisualInference/Inkling-2-Mini-MLX-3bit: direct link, hf CLI and curl.
- Browser
- Download file 85.2 kB
-
https://huggingface.co/VisualInference/Inkling-2-Mini-MLX-3bit/resolve/main/extras/results.json
- Command line
-
hf download hf://VisualInference/Inkling-2-Mini-MLX-3bit/extras/results.json
-
curl -L -o results.json https://huggingface.co/VisualInference/Inkling-2-Mini-MLX-3bit/resolve/main/extras/results.json
85.2 kB
{
"held_out_loss": {
"native": {
"tokens": 40806,
"nll": 1.4480010462071342,
"acc": 0.6699995098759987,
"seconds": 115,
"peak_gb": 22.03,
"weights_gib": null,
"activations": "bf16",
"actquant": null,
"chat": {
"tokens": 6597,
"nll": 0.4662371408908642
},
"code": {
"tokens": 8877,
"nll": 0.8892847396130361
},
"wikitext": {
"tokens": 25332,
"nll": 1.8994624837962406
}
},
"3bit": {
"tokens": 40806,
"nll": 1.46238731782892,
"acc": 0.6657844434641964,
"seconds": 113,
"peak_gb": 14.3,
"weights_gib": null,
"activations": "bf16",
"actquant": null,
"chat": {
"tokens": 6597,
"nll": 0.49518932673864985
},
"code": {
"tokens": 8877,
"nll": 0.9058923079944132
},
"wikitext": {
"tokens": 25332,
"nll": 1.9092770758236866
}
},
"3bit_round_to_nearest": {
"tokens": 40806,
"nll": 1.4701619848213856,
"acc": 0.6639709846591187,
"seconds": 113,
"peak_gb": 19.63,
"weights_gib": 11.910704612731934,
"activations": "bf16",
"actquant": null,
"chat": {
"tokens": 6597,
"nll": 0.49830741327206834
},
"code": {
"tokens": 8877,
"nll": 0.9093072548956838
},
"wikitext": {
"tokens": 25332,
"nll": 1.919792177702378
}
}
},
"numerics_experiments": {
"native_bf16_actquant": {
"tokens": 40806,
"nll": 1.4507249015946835,
"acc": 0.6688967308729108,
"seconds": 151,
"peak_gb": 22.22,
"weights_gib": null,
"activations": "bf16",
"actquant": [
"input",
"hidden"
],
"chat": {
"tokens": 6597,
"nll": 0.470449288324712
},
"code": {
"tokens": 8877,
"nll": 0.8916940795444644
},
"wikitext": {
"tokens": 25332,
"nll": 1.9019089702857381
}
},
"native_fp32": {
"tokens": 40806,
"nll": 1.4489235311713002,
"acc": 0.6690682742733912,
"seconds": 134,
"peak_gb": 24.1,
"weights_gib": null,
"activations": "fp32",
"actquant": null,
"chat": {
"tokens": 6597,
"nll": 0.4647530796709649
},
"code": {
"tokens": 8877,
"nll": 0.8885994847056913
},
"wikitext": {
"tokens": 25332,
"nll": 1.9015750797668678
}
},
"native_fp32_actquant": {
"tokens": 40806,
"nll": 1.4478448928936214,
"acc": 0.6682105572709895,
"seconds": 163,
"peak_gb": 24.24,
"weights_gib": null,
"activations": "fp32",
"actquant": [
"input",
"hidden"
],
"chat": {
"tokens": 6597,
"nll": 0.4712143190091029
},
"code": {
"tokens": 8877,
"nll": 0.8948558546528027
},
"wikitext": {
"tokens": 25332,
"nll": 1.8959625144150138
}
},
"native_bf16_actquant_input": {
"tokens": 40806,
"nll": 1.4477849032594567,
"acc": 0.6688477184727736,
"seconds": 120,
"peak_gb": 21.99,
"weights_gib": null,
"activations": "bf16",
"actquant": [
"input"
],
"chat": {
"tokens": 6597,
"nll": 0.46967875793049946
},
"code": {
"tokens": 8877,
"nll": 0.8883638166019585
},
"wikitext": {
"tokens": 25332,
"nll": 1.8985407546329665
}
}
},
"tasks": {
"tools": {
"passed": 19,
"total": 19,
"leaked": 0,
"seconds": 87,
"rows": [
{
"kind": "single",
"i": 0,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 2.5,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 1,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 3.3,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 2,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 2.3,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 3,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 2.3,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 4,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 2.7,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 5,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 6.2,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 6,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 4.3,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 7,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 2.8,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 8,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 3.1,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 9,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 2.1,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 10,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 5.8,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 11,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 4.1,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "no_tool",
"i": 0,
"ok": true,
"n_calls": 0,
"finish": "stop",
"leaked": false,
"seconds": 1.9
},
{
"kind": "no_tool",
"i": 1,
"ok": true,
"n_calls": 0,
"finish": "stop",
"leaked": false,
"seconds": 2.7
},
{
"kind": "multi",
"i": 0,
"ok": true,
"rounds": 2,
"n_calls": 2,
"finish": "stop",
"leaked": false,
"seconds": 7.5,
"final": "Lisbon is warmer right now: 19\u00b0C vs Oslo at 3\u00b0C \u2014 about 16\u00b0C warmer.",
"error": null
},
{
"kind": "multi",
"i": 1,
"ok": true,
"rounds": 2,
"n_calls": 1,
"finish": "stop",
"leaked": false,
"seconds": 5.0,
"final": "RETRIES is 3.",
"error": null
},
{
"kind": "multi",
"i": 2,
"ok": true,
"rounds": 2,
"n_calls": 1,
"finish": "stop",
"leaked": false,
"seconds": 6.8,
"final": "docs/ARCHITECTURE.md does not exist \u2014 there is no docs/ directory in this repository. The repo contains only Makefile, README.md, app/, src/, and tests/. I can summarize README.md if you'd like.",
"error": null
},
{
"kind": "multi",
"i": 3,
"ok": true,
"rounds": 2,
"n_calls": 2,
"finish": "stop",
"leaked": false,
"seconds": 12.7,
"final": "(1234 \u00d7 57) \u2212 999 = **69,339**\n\n69,339 \u00f7 7 = **9,905.57142857143** (about **9,905.57**)",
"error": null
},
{
"kind": "multi",
"i": 4,
"ok": true,
"rounds": 2,
"n_calls": 1,
"finish": "stop",
"leaked": false,
"seconds": 9.0,
"final": "Customer C-88 has 2 orders in total. The most expensive is A-1042, totaling $129.50 (shipped, tagged gift). The other order, A-1077, is $48.00 and open.",
"error": null
}
]
},
"json": {
"passed": 9,
"total": 10,
"leaked": 0,
"seconds": 54,
"rows": [
{
"i": 0,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 9.5,
"content": null
},
{
"i": 1,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 2.5,
"content": null
},
{
"i": 2,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 5.2,
"content": null
},
{
"i": 3,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 3.2,
"content": null
},
{
"i": 4,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 6.7,
"content": null
},
{
"i": 5,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 3.7,
"content": null
},
{
"i": 6,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 3.7,
"content": null
},
{
"i": 7,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 2.9,
"content": null
},
{
"i": 8,
"ok": false,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 8.5,
"content": "{\"quote\":\"She said \\\"stop\\\" and left.\\\\\"}"
},
{
"i": 9,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 8.2,
"content": null
}
]
},
"humaneval": {
"passed": 35,
"total": 40,
"leaked": 0,
"seconds": 612,
"rows": [
{
"task": "HumanEval/0",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 299,
"seconds": 9.8
},
{
"task": "HumanEval/4",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 429,
"seconds": 13.8
},
{
"task": "HumanEval/8",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 360,
"seconds": 11.6
},
{
"task": "HumanEval/12",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 365,
"seconds": 11.8
},
{
"task": "HumanEval/16",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 154,
"seconds": 5.1
},
{
"task": "HumanEval/20",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 486,
"seconds": 15.8
},
{
"task": "HumanEval/24",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 350,
"seconds": 11.1
},
{
"task": "HumanEval/28",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 162,
"seconds": 5.4
},
{
"task": "HumanEval/32",
"ok": false,
"err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp0gadlh0d/prog.py\", line 26, in <module>\n check(find_zero)\n ^^^^^^^^^\nNameError: name 'find_zero' is not defined\n",
"finish": "length",
"leaked": false,
"tokens": 900,
"seconds": 29.9
},
{
"task": "HumanEval/36",
"ok": true,
"err": null,
"finish": "length",
"leaked": false,
"tokens": 900,
"seconds": 28.8
},
{
"task": "HumanEval/40",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 572,
"seconds": 18.7
},
{
"task": "HumanEval/44",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 494,
"seconds": 15.8
},
{
"task": "HumanEval/48",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 220,
"seconds": 7.2
},
{
"task": "HumanEval/52",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 199,
"seconds": 6.6
},
{
"task": "HumanEval/56",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 468,
"seconds": 15.0
},
{
"task": "HumanEval/60",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 216,
"seconds": 7.1
},
{
"task": "HumanEval/64",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 509,
"seconds": 16.5
},
{
"task": "HumanEval/68",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 239,
"seconds": 8.5
},
{
"task": "HumanEval/72",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 120,
"seconds": 4.5
},
{
"task": "HumanEval/76",
"ok": false,
"err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmph8086k4_/prog.py\", line 21, in <module>\n check(is_simple_power)\n ^^^^^^^^^^^^^^^\nNameError: name 'is_simple_power' is not defined\n",
"finish": "length",
"leaked": false,
"tokens": 900,
"seconds": 29.1
},
{
"task": "HumanEval/80",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 422,
"seconds": 13.6
},
{
"task": "HumanEval/84",
"ok": false,
"err": " File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmpe6o6tp25/prog.py\", line 3\n \"\"\"Given a positive integer N, return the total sum of its digits in binary.\n ^\nSyntaxError: unterminated triple-quoted string literal (detected at line 28)\n",
"finish": "length",
"leaked": false,
"tokens": 900,
"seconds": 29.1
},
{
"task": "HumanEval/88",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 398,
"seconds": 13.3
},
{
"task": "HumanEval/92",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 490,
"seconds": 15.8
},
{
"task": "HumanEval/96",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 592,
"seconds": 19.1
},
{
"task": "HumanEval/100",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 345,
"seconds": 11.2
},
{
"task": "HumanEval/104",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 360,
"seconds": 11.6
},
{
"task": "HumanEval/108",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 887,
"seconds": 28.6
},
{
"task": "HumanEval/112",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 202,
"seconds": 6.9
},
{
"task": "HumanEval/116",
"ok": false,
"err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp91sbevq_/prog.py\", line 21, in <module>\n check(sort_array)\n ^^^^^^^^^^\nNameError: name 'sort_array' is not defined\n",
"finish": "length",
"leaked": false,
"tokens": 900,
"seconds": 29.5
},
{
"task": "HumanEval/120",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 288,
"seconds": 9.8
},
{
"task": "HumanEval/124",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 705,
"seconds": 23.3
},
{
"task": "HumanEval/128",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 543,
"seconds": 17.4
},
{
"task": "HumanEval/132",
"ok": false,
"err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmpvvmctbl3/prog.py\", line 25, in <module>\n check(is_nested)\n ^^^^^^^^^\nNameError: name 'is_nested' is not defined\n",
"finish": "length",
"leaked": false,
"tokens": 900,
"seconds": 29.3
},
{
"task": "HumanEval/136",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 326,
"seconds": 10.6
},
{
"task": "HumanEval/140",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 496,
"seconds": 15.9
},
{
"task": "HumanEval/144",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 267,
"seconds": 8.9
},
{
"task": "HumanEval/148",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 562,
"seconds": 18.4
},
{
"task": "HumanEval/152",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 246,
"seconds": 8.4
},
{
"task": "HumanEval/156",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 539,
"seconds": 17.3
}
]
},
"gsm8k": {
"passed": 49,
"total": 50,
"leaked": 0,
"seconds": 414,
"rows": [
{
"i": 0,
"ok": true,
"pred": 18.0,
"gold": 18.0,
"finish": "stop",
"leaked": false,
"tokens": 177,
"seconds": 5.8
},
{
"i": 26,
"ok": true,
"pred": 243.0,
"gold": 243.0,
"finish": "stop",
"leaked": false,
"tokens": 236,
"seconds": 7.6
},
{
"i": 52,
"ok": true,
"pred": 15.0,
"gold": 15.0,
"finish": "stop",
"leaked": false,
"tokens": 183,
"seconds": 6.0
},
{
"i": 78,
"ok": true,
"pred": 6.0,
"gold": 6.0,
"finish": "stop",
"leaked": false,
"tokens": 235,
"seconds": 7.6
},
{
"i": 104,
"ok": true,
"pred": 500.0,
"gold": 500.0,
"finish": "stop",
"leaked": false,
"tokens": 180,
"seconds": 5.9
},
{
"i": 130,
"ok": true,
"pred": 30.0,
"gold": 30.0,
"finish": "stop",
"leaked": false,
"tokens": 162,
"seconds": 5.3
},
{
"i": 156,
"ok": true,
"pred": 1400.0,
"gold": 1400.0,
"finish": "stop",
"leaked": false,
"tokens": 137,
"seconds": 4.6
},
{
"i": 182,
"ok": true,
"pred": 23.0,
"gold": 23.0,
"finish": "stop",
"leaked": false,
"tokens": 836,
"seconds": 26.5
},
{
"i": 208,
"ok": true,
"pred": 76.0,
"gold": 76.0,
"finish": "stop",
"leaked": false,
"tokens": 162,
"seconds": 5.4
},
{
"i": 234,
"ok": true,
"pred": 21.0,
"gold": 21.0,
"finish": "stop",
"leaked": false,
"tokens": 509,
"seconds": 16.1
},
{
"i": 260,
"ok": true,
"pred": 11.0,
"gold": 11.0,
"finish": "stop",
"leaked": false,
"tokens": 215,
"seconds": 6.9
},
{
"i": 286,
"ok": true,
"pred": 90.0,
"gold": 90.0,
"finish": "stop",
"leaked": false,
"tokens": 148,
"seconds": 4.9
},
{
"i": 312,
"ok": true,
"pred": 32.0,
"gold": 32.0,
"finish": "stop",
"leaked": false,
"tokens": 113,
"seconds": 3.8
},
{
"i": 338,
"ok": true,
"pred": 60.0,
"gold": 60.0,
"finish": "stop",
"leaked": false,
"tokens": 199,
"seconds": 6.5
},
{
"i": 364,
"ok": true,
"pred": 49.0,
"gold": 49.0,
"finish": "stop",
"leaked": false,
"tokens": 123,
"seconds": 4.1
},
{
"i": 390,
"ok": true,
"pred": 12.0,
"gold": 12.0,
"finish": "stop",
"leaked": false,
"tokens": 118,
"seconds": 4.0
},
{
"i": 416,
"ok": true,
"pred": 56.0,
"gold": 56.0,
"finish": "stop",
"leaked": false,
"tokens": 190,
"seconds": 6.2
},
{
"i": 442,
"ok": true,
"pred": 49.0,
"gold": 49.0,
"finish": "stop",
"leaked": false,
"tokens": 260,
"seconds": 8.4
},
{
"i": 468,
"ok": true,
"pred": 120.0,
"gold": 120.0,
"finish": "stop",
"leaked": false,
"tokens": 195,
"seconds": 6.4
},
{
"i": 494,
"ok": true,
"pred": 21.0,
"gold": 21.0,
"finish": "stop",
"leaked": false,
"tokens": 571,
"seconds": 18.2
},
{
"i": 520,
"ok": true,
"pred": 63.0,
"gold": 63.0,
"finish": "stop",
"leaked": false,
"tokens": 152,
"seconds": 5.2
},
{
"i": 546,
"ok": true,
"pred": 130.0,
"gold": 130.0,
"finish": "stop",
"leaked": false,
"tokens": 329,
"seconds": 10.5
},
{
"i": 572,
"ok": true,
"pred": 100.0,
"gold": 100.0,
"finish": "stop",
"leaked": false,
"tokens": 477,
"seconds": 15.2
},
{
"i": 598,
"ok": true,
"pred": 69.0,
"gold": 69.0,
"finish": "stop",
"leaked": false,
"tokens": 221,
"seconds": 7.1
},
{
"i": 624,
"ok": true,
"pred": 85.0,
"gold": 85.0,
"finish": "stop",
"leaked": false,
"tokens": 232,
"seconds": 7.5
},
{
"i": 650,
"ok": true,
"pred": 142.0,
"gold": 142.0,
"finish": "stop",
"leaked": false,
"tokens": 285,
"seconds": 9.2
},
{
"i": 676,
"ok": true,
"pred": 240.0,
"gold": 240.0,
"finish": "stop",
"leaked": false,
"tokens": 252,
"seconds": 8.1
},
{
"i": 702,
"ok": true,
"pred": 2800.0,
"gold": 2800.0,
"finish": "stop",
"leaked": false,
"tokens": 150,
"seconds": 5.1
},
{
"i": 728,
"ok": true,
"pred": 40.0,
"gold": 40.0,
"finish": "stop",
"leaked": false,
"tokens": 228,
"seconds": 7.3
},
{
"i": 754,
"ok": true,
"pred": 89.0,
"gold": 89.0,
"finish": "stop",
"leaked": false,
"tokens": 583,
"seconds": 18.7
},
{
"i": 780,
"ok": true,
"pred": 25.0,
"gold": 25.0,
"finish": "stop",
"leaked": false,
"tokens": 313,
"seconds": 10.2
},
{
"i": 806,
"ok": true,
"pred": 70.0,
"gold": 70.0,
"finish": "stop",
"leaked": false,
"tokens": 326,
"seconds": 10.4
},
{
"i": 832,
"ok": true,
"pred": 42.0,
"gold": 42.0,
"finish": "stop",
"leaked": false,
"tokens": 232,
"seconds": 7.4
},
{
"i": 858,
"ok": true,
"pred": 3430.0,
"gold": 3430.0,
"finish": "stop",
"leaked": false,
"tokens": 432,
"seconds": 13.6
},
{
"i": 884,
"ok": true,
"pred": 50.0,
"gold": 50.0,
"finish": "stop",
"leaked": false,
"tokens": 120,
"seconds": 4.1
},
{
"i": 910,
"ok": true,
"pred": 700.0,
"gold": 700.0,
"finish": "stop",
"leaked": false,
"tokens": 130,
"seconds": 4.4
},
{
"i": 936,
"ok": true,
"pred": 250.0,
"gold": 250.0,
"finish": "stop",
"leaked": false,
"tokens": 397,
"seconds": 12.8
},
{
"i": 962,
"ok": false,
"pred": 600.0,
"gold": 1000.0,
"finish": "stop",
"leaked": false,
"tokens": 213,
"seconds": 6.9
},
{
"i": 988,
"ok": true,
"pred": 6.0,
"gold": 6.0,
"finish": "stop",
"leaked": false,
"tokens": 154,
"seconds": 5.1
},
{
"i": 1014,
"ok": true,
"pred": 3.0,
"gold": 3.0,
"finish": "stop",
"leaked": false,
"tokens": 406,
"seconds": 13.0
},
{
"i": 1040,
"ok": true,
"pred": 736.0,
"gold": 736.0,
"finish": "stop",
"leaked": false,
"tokens": 199,
"seconds": 6.5
},
{
"i": 1066,
"ok": true,
"pred": 14.0,
"gold": 14.0,
"finish": "stop",
"leaked": false,
"tokens": 225,
"seconds": 7.3
},
{
"i": 1092,
"ok": true,
"pred": 1080.0,
"gold": 1080.0,
"finish": "stop",
"leaked": false,
"tokens": 291,
"seconds": 9.4
},
{
"i": 1118,
"ok": true,
"pred": 5.0,
"gold": 5.0,
"finish": "stop",
"leaked": false,
"tokens": 268,
"seconds": 8.7
},
{
"i": 1144,
"ok": true,
"pred": 34.0,
"gold": 34.0,
"finish": "stop",
"leaked": false,
"tokens": 143,
"seconds": 4.7
},
{
"i": 1170,
"ok": true,
"pred": 2.0,
"gold": 2.0,
"finish": "stop",
"leaked": false,
"tokens": 166,
"seconds": 5.5
},
{
"i": 1196,
"ok": true,
"pred": 75.0,
"gold": 75.0,
"finish": "stop",
"leaked": false,
"tokens": 182,
"seconds": 5.9
},
{
"i": 1222,
"ok": true,
"pred": 350.0,
"gold": 350.0,
"finish": "stop",
"leaked": false,
"tokens": 274,
"seconds": 8.8
},
{
"i": 1248,
"ok": true,
"pred": 750.0,
"gold": 750.0,
"finish": "stop",
"leaked": false,
"tokens": 221,
"seconds": 7.3
},
{
"i": 1274,
"ok": true,
"pred": 38.0,
"gold": 38.0,
"finish": "stop",
"leaked": false,
"tokens": 239,
"seconds": 7.7
}
]
}
},
"tasks_original": {
"tools": {
"passed": 18,
"total": 19,
"leaked": 0,
"seconds": 104,
"rows": [
{
"kind": "single",
"i": 0,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 3.3,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 1,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 4.0,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 2,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 2.9,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 3,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 2.9,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 4,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 3.2,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 5,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 5.3,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 6,
"n_calls": 0,
"finish": "stop",
"leaked": false,
"seconds": 4.5,
"valid_json": false,
"right_tool": false,
"ok": false,
"got": ""
},
{
"kind": "single",
"i": 7,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 3.3,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 8,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 3.9,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 9,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 2.2,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 10,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 7.6,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "single",
"i": 11,
"n_calls": 1,
"finish": "tool_calls",
"leaked": false,
"seconds": 4.7,
"valid_json": true,
"right_tool": true,
"ok": true
},
{
"kind": "no_tool",
"i": 0,
"ok": true,
"n_calls": 0,
"finish": "stop",
"leaked": false,
"seconds": 2.2
},
{
"kind": "no_tool",
"i": 1,
"ok": true,
"n_calls": 0,
"finish": "stop",
"leaked": false,
"seconds": 3.7
},
{
"kind": "multi",
"i": 0,
"ok": true,
"rounds": 2,
"n_calls": 2,
"finish": "stop",
"leaked": false,
"seconds": 9.7,
"final": "Lisbon is warmer right now: 19\u00b0C vs Oslo\u2019s 3\u00b0C \u2014 about 16\u00b0C warmer.",
"error": null
},
{
"kind": "multi",
"i": 1,
"ok": true,
"rounds": 2,
"n_calls": 1,
"finish": "stop",
"leaked": false,
"seconds": 6.2,
"final": "RETRIES is 3.",
"error": null
},
{
"kind": "multi",
"i": 2,
"ok": true,
"rounds": 2,
"n_calls": 1,
"finish": "stop",
"leaked": false,
"seconds": 9.1,
"final": "docs/ARCHITECTURE.md does not exist \u2014 there is no `docs/` directory in this repository. The repo contains only `Makefile`, `README.md`, `app/`, `src/`, and `tests/`. I can summarise the README instead if you'd like.",
"error": null
},
{
"kind": "multi",
"i": 3,
"ok": true,
"rounds": 2,
"n_calls": 2,
"finish": "stop",
"leaked": false,
"seconds": 15.3,
"final": "(1234 \u00d7 57) \u2212 999 = **69,339**\n\n69,339 \u00f7 7 = **9,905.57142857143** (about **9,905.57**)",
"error": null
},
{
"kind": "multi",
"i": 4,
"ok": true,
"rounds": 2,
"n_calls": 1,
"finish": "stop",
"leaked": false,
"seconds": 9.6,
"final": "Customer C-88 has 2 orders in total. The most expensive is A-1042, totaling $129.50.",
"error": null
}
]
},
"json": {
"passed": 9,
"total": 10,
"leaked": 0,
"seconds": 86,
"rows": [
{
"i": 0,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 17.1,
"content": null
},
{
"i": 1,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 3.0,
"content": null
},
{
"i": 2,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 5.2,
"content": null
},
{
"i": 3,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 4.6,
"content": null
},
{
"i": 4,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 8.4,
"content": null
},
{
"i": 5,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 3.9,
"content": null
},
{
"i": 6,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 4.8,
"content": null
},
{
"i": 7,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 3.3,
"content": null
},
{
"i": 8,
"ok": false,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 27.3,
"content": "{\"quote\":\"She said \\\"stop\\\" and left.\\\\\"}"
},
{
"i": 9,
"ok": true,
"valid_json": true,
"finish": "stop",
"leaked": false,
"seconds": 8.0,
"content": null
}
]
},
"humaneval": {
"passed": 35,
"total": 40,
"leaked": 0,
"seconds": 759,
"rows": [
{
"task": "HumanEval/0",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 427,
"seconds": 17.7
},
{
"task": "HumanEval/4",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 285,
"seconds": 11.9
},
{
"task": "HumanEval/8",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 493,
"seconds": 20.3
},
{
"task": "HumanEval/12",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 573,
"seconds": 23.6
},
{
"task": "HumanEval/16",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 227,
"seconds": 9.4
},
{
"task": "HumanEval/20",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 625,
"seconds": 26.1
},
{
"task": "HumanEval/24",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 249,
"seconds": 10.3
},
{
"task": "HumanEval/28",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 165,
"seconds": 7.0
},
{
"task": "HumanEval/32",
"ok": false,
"err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp38s2jvzb/prog.py\", line 26, in <module>\n check(find_zero)\n ^^^^^^^^^\nNameError: name 'find_zero' is not defined\n",
"finish": "length",
"leaked": false,
"tokens": 900,
"seconds": 38.6
},
{
"task": "HumanEval/36",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 640,
"seconds": 26.4
},
{
"task": "HumanEval/40",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 641,
"seconds": 27.0
},
{
"task": "HumanEval/44",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 365,
"seconds": 15.1
},
{
"task": "HumanEval/48",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 210,
"seconds": 8.8
},
{
"task": "HumanEval/52",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 239,
"seconds": 10.0
},
{
"task": "HumanEval/56",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 270,
"seconds": 11.3
},
{
"task": "HumanEval/60",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 269,
"seconds": 11.2
},
{
"task": "HumanEval/64",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 497,
"seconds": 20.6
},
{
"task": "HumanEval/68",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 335,
"seconds": 14.7
},
{
"task": "HumanEval/72",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 154,
"seconds": 7.0
},
{
"task": "HumanEval/76",
"ok": false,
"err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp0mwlmoao/prog.py\", line 21, in <module>\n check(is_simple_power)\n ^^^^^^^^^^^^^^^\nNameError: name 'is_simple_power' is not defined\n",
"finish": "length",
"leaked": false,
"tokens": 900,
"seconds": 37.7
},
{
"task": "HumanEval/80",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 243,
"seconds": 10.2
},
{
"task": "HumanEval/84",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 756,
"seconds": 31.5
},
{
"task": "HumanEval/88",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 266,
"seconds": 11.6
},
{
"task": "HumanEval/92",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 455,
"seconds": 18.9
},
{
"task": "HumanEval/96",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 555,
"seconds": 23.2
},
{
"task": "HumanEval/100",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 353,
"seconds": 14.7
},
{
"task": "HumanEval/104",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 412,
"seconds": 17.1
},
{
"task": "HumanEval/108",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 557,
"seconds": 23.1
},
{
"task": "HumanEval/112",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 288,
"seconds": 12.2
},
{
"task": "HumanEval/116",
"ok": false,
"err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp4huqxua3/prog.py\", line 21, in <module>\n check(sort_array)\n ^^^^^^^^^^\nNameError: name 'sort_array' is not defined\n",
"finish": "length",
"leaked": false,
"tokens": 900,
"seconds": 37.9
},
{
"task": "HumanEval/120",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 331,
"seconds": 14.2
},
{
"task": "HumanEval/124",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 464,
"seconds": 19.8
},
{
"task": "HumanEval/128",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 398,
"seconds": 16.5
},
{
"task": "HumanEval/132",
"ok": false,
"err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp_1cc4urv/prog.py\", line 25, in <module>\n check(is_nested)\n ^^^^^^^^^\nNameError: name 'is_nested' is not defined\n",
"finish": "length",
"leaked": false,
"tokens": 900,
"seconds": 37.8
},
{
"task": "HumanEval/136",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 328,
"seconds": 13.8
},
{
"task": "HumanEval/140",
"ok": false,
"err": "Traceback (most recent call last):\n File \"/private/var/folders/kd/xh61hr5n5c75hx91r3yrt0jw0000gn/T/tmp7bobb2r_/prog.py\", line 16, in <module>\n check(fix_spaces)\n ^^^^^^^^^^\nNameError: name 'fix_spaces' is not defined\n",
"finish": "length",
"leaked": false,
"tokens": 900,
"seconds": 37.6
},
{
"task": "HumanEval/144",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 238,
"seconds": 10.2
},
{
"task": "HumanEval/148",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 582,
"seconds": 24.6
},
{
"task": "HumanEval/152",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 137,
"seconds": 6.3
},
{
"task": "HumanEval/156",
"ok": true,
"err": null,
"finish": "stop",
"leaked": false,
"tokens": 507,
"seconds": 21.0
}
]
},
"gsm8k": {
"passed": 48,
"total": 50,
"leaked": 0,
"seconds": 543,
"rows": [
{
"i": 0,
"ok": true,
"pred": 18.0,
"gold": 18.0,
"finish": "stop",
"leaked": false,
"tokens": 150,
"seconds": 6.4
},
{
"i": 26,
"ok": true,
"pred": 243.0,
"gold": 243.0,
"finish": "stop",
"leaked": false,
"tokens": 192,
"seconds": 8.0
},
{
"i": 52,
"ok": true,
"pred": 15.0,
"gold": 15.0,
"finish": "stop",
"leaked": false,
"tokens": 204,
"seconds": 8.5
},
{
"i": 78,
"ok": true,
"pred": 6.0,
"gold": 6.0,
"finish": "stop",
"leaked": false,
"tokens": 234,
"seconds": 9.8
},
{
"i": 104,
"ok": true,
"pred": 500.0,
"gold": 500.0,
"finish": "stop",
"leaked": false,
"tokens": 180,
"seconds": 7.5
},
{
"i": 130,
"ok": true,
"pred": 30.0,
"gold": 30.0,
"finish": "stop",
"leaked": false,
"tokens": 124,
"seconds": 5.4
},
{
"i": 156,
"ok": true,
"pred": 1400.0,
"gold": 1400.0,
"finish": "stop",
"leaked": false,
"tokens": 164,
"seconds": 7.0
},
{
"i": 182,
"ok": false,
"pred": 24.0,
"gold": 23.0,
"finish": "stop",
"leaked": false,
"tokens": 851,
"seconds": 34.9
},
{
"i": 208,
"ok": true,
"pred": 76.0,
"gold": 76.0,
"finish": "stop",
"leaked": false,
"tokens": 139,
"seconds": 5.9
},
{
"i": 234,
"ok": true,
"pred": 21.0,
"gold": 21.0,
"finish": "stop",
"leaked": false,
"tokens": 613,
"seconds": 25.1
},
{
"i": 260,
"ok": true,
"pred": 11.0,
"gold": 11.0,
"finish": "stop",
"leaked": false,
"tokens": 210,
"seconds": 8.7
},
{
"i": 286,
"ok": true,
"pred": 90.0,
"gold": 90.0,
"finish": "stop",
"leaked": false,
"tokens": 148,
"seconds": 6.2
},
{
"i": 312,
"ok": true,
"pred": 32.0,
"gold": 32.0,
"finish": "stop",
"leaked": false,
"tokens": 113,
"seconds": 4.8
},
{
"i": 338,
"ok": true,
"pred": 60.0,
"gold": 60.0,
"finish": "stop",
"leaked": false,
"tokens": 186,
"seconds": 7.8
},
{
"i": 364,
"ok": true,
"pred": 49.0,
"gold": 49.0,
"finish": "stop",
"leaked": false,
"tokens": 115,
"seconds": 5.0
},
{
"i": 390,
"ok": true,
"pred": 12.0,
"gold": 12.0,
"finish": "stop",
"leaked": false,
"tokens": 295,
"seconds": 12.2
},
{
"i": 416,
"ok": true,
"pred": 56.0,
"gold": 56.0,
"finish": "stop",
"leaked": false,
"tokens": 185,
"seconds": 7.7
},
{
"i": 442,
"ok": true,
"pred": 49.0,
"gold": 49.0,
"finish": "stop",
"leaked": false,
"tokens": 229,
"seconds": 9.6
},
{
"i": 468,
"ok": true,
"pred": 120.0,
"gold": 120.0,
"finish": "stop",
"leaked": false,
"tokens": 225,
"seconds": 9.3
},
{
"i": 494,
"ok": true,
"pred": 21.0,
"gold": 21.0,
"finish": "stop",
"leaked": false,
"tokens": 702,
"seconds": 28.9
},
{
"i": 520,
"ok": true,
"pred": 63.0,
"gold": 63.0,
"finish": "stop",
"leaked": false,
"tokens": 182,
"seconds": 7.8
},
{
"i": 546,
"ok": true,
"pred": 130.0,
"gold": 130.0,
"finish": "stop",
"leaked": false,
"tokens": 382,
"seconds": 15.7
},
{
"i": 572,
"ok": true,
"pred": 100.0,
"gold": 100.0,
"finish": "stop",
"leaked": false,
"tokens": 217,
"seconds": 9.1
},
{
"i": 598,
"ok": true,
"pred": 69.0,
"gold": 69.0,
"finish": "stop",
"leaked": false,
"tokens": 189,
"seconds": 7.8
},
{
"i": 624,
"ok": true,
"pred": 85.0,
"gold": 85.0,
"finish": "stop",
"leaked": false,
"tokens": 192,
"seconds": 8.0
},
{
"i": 650,
"ok": true,
"pred": 142.0,
"gold": 142.0,
"finish": "stop",
"leaked": false,
"tokens": 259,
"seconds": 10.8
},
{
"i": 676,
"ok": true,
"pred": 240.0,
"gold": 240.0,
"finish": "stop",
"leaked": false,
"tokens": 260,
"seconds": 10.8
},
{
"i": 702,
"ok": true,
"pred": 2800.0,
"gold": 2800.0,
"finish": "stop",
"leaked": false,
"tokens": 179,
"seconds": 7.6
},
{
"i": 728,
"ok": true,
"pred": 40.0,
"gold": 40.0,
"finish": "stop",
"leaked": false,
"tokens": 232,
"seconds": 9.6
},
{
"i": 754,
"ok": true,
"pred": 89.0,
"gold": 89.0,
"finish": "stop",
"leaked": false,
"tokens": 612,
"seconds": 25.3
},
{
"i": 780,
"ok": true,
"pred": 25.0,
"gold": 25.0,
"finish": "stop",
"leaked": false,
"tokens": 390,
"seconds": 16.2
},
{
"i": 806,
"ok": true,
"pred": 70.0,
"gold": 70.0,
"finish": "stop",
"leaked": false,
"tokens": 321,
"seconds": 13.3
},
{
"i": 832,
"ok": true,
"pred": 42.0,
"gold": 42.0,
"finish": "stop",
"leaked": false,
"tokens": 321,
"seconds": 13.1
},
{
"i": 858,
"ok": true,
"pred": 3430.0,
"gold": 3430.0,
"finish": "stop",
"leaked": false,
"tokens": 429,
"seconds": 17.5
},
{
"i": 884,
"ok": true,
"pred": 50.0,
"gold": 50.0,
"finish": "stop",
"leaked": false,
"tokens": 102,
"seconds": 4.5
},
{
"i": 910,
"ok": true,
"pred": 700.0,
"gold": 700.0,
"finish": "stop",
"leaked": false,
"tokens": 137,
"seconds": 5.9
},
{
"i": 936,
"ok": true,
"pred": 250.0,
"gold": 250.0,
"finish": "stop",
"leaked": false,
"tokens": 284,
"seconds": 11.9
},
{
"i": 962,
"ok": false,
"pred": 600.0,
"gold": 1000.0,
"finish": "stop",
"leaked": false,
"tokens": 135,
"seconds": 5.8
},
{
"i": 988,
"ok": true,
"pred": 6.0,
"gold": 6.0,
"finish": "stop",
"leaked": false,
"tokens": 581,
"seconds": 23.8
},
{
"i": 1014,
"ok": true,
"pred": 3.0,
"gold": 3.0,
"finish": "stop",
"leaked": false,
"tokens": 381,
"seconds": 15.8
},
{
"i": 1040,
"ok": true,
"pred": 736.0,
"gold": 736.0,
"finish": "stop",
"leaked": false,
"tokens": 199,
"seconds": 8.3
},
{
"i": 1066,
"ok": true,
"pred": 14.0,
"gold": 14.0,
"finish": "stop",
"leaked": false,
"tokens": 196,
"seconds": 8.2
},
{
"i": 1092,
"ok": true,
"pred": 1080.0,
"gold": 1080.0,
"finish": "stop",
"leaked": false,
"tokens": 200,
"seconds": 8.4
},
{
"i": 1118,
"ok": true,
"pred": 5.0,
"gold": 5.0,
"finish": "stop",
"leaked": false,
"tokens": 268,
"seconds": 11.1
},
{
"i": 1144,
"ok": true,
"pred": 34.0,
"gold": 34.0,
"finish": "stop",
"leaked": false,
"tokens": 176,
"seconds": 7.4
},
{
"i": 1170,
"ok": true,
"pred": 2.0,
"gold": 2.0,
"finish": "stop",
"leaked": false,
"tokens": 177,
"seconds": 7.4
},
{
"i": 1196,
"ok": true,
"pred": 75.0,
"gold": 75.0,
"finish": "stop",
"leaked": false,
"tokens": 191,
"seconds": 8.0
},
{
"i": 1222,
"ok": true,
"pred": 350.0,
"gold": 350.0,
"finish": "stop",
"leaked": false,
"tokens": 210,
"seconds": 8.7
},
{
"i": 1248,
"ok": true,
"pred": 750.0,
"gold": 750.0,
"finish": "stop",
"leaked": false,
"tokens": 188,
"seconds": 8.0
},
{
"i": 1274,
"ok": true,
"pred": 38.0,
"gold": 38.0,
"finish": "stop",
"leaked": false,
"tokens": 211,
"seconds": 8.8
}
]
}
},
"hermes_agent": {
"this_build": {
"passed": 4,
"total": 4,
"effort": "medium",
"tasks": {
"explain": {
"ok": true,
"seconds": 92.1,
"exit_code": 0,
"api_calls": 4,
"tool_calls": 3,
"tools": [
{
"name": "terminal",
"input": "{\"command\": \"ls -la ..."
},
{
"name": "terminal",
"input": "{\"command\": \"ls -R s..."
},
{
"name": "terminal",
"input": "{\"command\": \"python3..."
}
],
"final": "Tidepool is a small in-memory priority job queue (src/jobqueue.py, using a heap with priority and insertion-order tie-breaking). Run its tests with `python3 -m unittest discover -s tests -v` \u2014 note that one test currently fails: `test_pop_returns_job` fails because `pop()` returns the insertion counter (index 1 of the heap tuple) instead of the job (index 2).",
"tail": "",
"stderr": ""
},
"nonce": {
"ok": true,
"seconds": 84.9,
"exit_code": 0,
"api_calls": 4,
"tool_calls": 3,
"tools": [
{
"name": "terminal",
"input": "{\"command\": \"ls -la;..."
},
{
"name": "read_file",
"input": "{\"path\": \"notes/secr..."
},
{
"name": "read_file",
"input": "{\"path\": \"README.md\"..."
}
],
"final": "The access code in notes/secret.txt is **ZX-FE4E89E0**.",
"tail": "",
"stderr": ""
},
"fix_bug": {
"ok": true,
"seconds": 159.6,
"exit_code": 0,
"api_calls": 8,
"tool_calls": 8,
"tools": [
{
"name": "terminal",
"input": "{\"command\": \"pwd && ..."
},
{
"name": "terminal",
"input": "{\"command\": \"cat REA..."
},
{
"name": "read_file",
"input": "{\"path\": \"src/jobque..."
},
{
"name": "read_file",
"input": "{\"path\": \"tests/test..."
},
{
"name": "terminal",
"input": "{\"command\": \"python3..."
},
{
"name": "patch",
"input": "{\"new_string\": \" ..."
},
{
"name": "patch",
"input": "{\"new_string\": \" ..."
},
{
"name": "terminal",
"input": "{\"command\": \"python3..."
}
],
"final": "Fixed. `src/jobqueue.py:16` `pop()` returned index `[1]` of the heap entry \u2014 the tie-breaking insertion counter (e.g. `1`) \u2014 instead of `[2]`, the job itself. Tests now pass (2/2).",
"tail": "",
"stderr": ""
},
"write_run": {
"ok": true,
"seconds": 147.4,
"exit_code": 0,
"api_calls": 6,
"tool_calls": 6,
"tools": [
{
"name": "terminal",
"input": "{\"command\": \"pwd && ..."
},
{
"name": "read_file",
"input": "{\"path\": \"README.md\"..."
},
{
"name": "terminal",
"input": "{\"command\": \"ls -la ..."
},
{
"name": "terminal",
"input": "{\"command\": \"mkdir -..."
},
{
"name": "write_file",
"input": "{\"content\": \"#!/usr/..."
},
{
"name": "terminal",
"input": "{\"command\": \"python3..."
}
],
"final": "`scripts/wordcount.py` created and run \u2014 it printed **20**.\n\n(Word count is whitespace-split, so \"#\", \"##\", and the `python3 -m unittest ...` line count as words too.)",
"tail": "",
"stderr": ""
}
}
},
"original": {
"passed": 4,
"total": 4,
"effort": "medium",
"tasks": {
"explain": {
"ok": true,
"seconds": 130.1,
"exit_code": 0,
"api_calls": 5,
"tool_calls": 4,
"tools": [
{
"name": "terminal",
"input": "{\"command\": \"ls -la ..."
},
{
"name": "terminal",
"input": "{\"command\": \"pwd && ..."
},
{
"name": "terminal",
"input": "{\"command\": \"head -4..."
},
{
"name": "terminal",
"input": "{\"command\": \"python3..."
}
],
"final": "Tidepool is a small in-memory job queue that serves higher-priority jobs first, with insertion order breaking ties. Run the tests with `python3 -m unittest discover -s tests -v` \u2014 but be aware that one test currently fails: `pop()` in `src/jobqueue.py:17` returns the counter (`[1]`) instead of the job (`[2]`), so `test_pop_returns_job` fails (`1 != 'fix outage'`). Want me to fix that line?",
"tail": "",
"stderr": ""
},
"nonce": {
"ok": true,
"seconds": 75.4,
"exit_code": 0,
"api_calls": 3,
"tool_calls": 2,
"tools": [
{
"name": "terminal",
"input": "{\"command\": \"pwd && ..."
},
{
"name": "terminal",
"input": "{\"command\": \"ls -la ..."
}
],
"final": "The access code in notes/secret.txt is ZX-F2E46563.",
"tail": "",
"stderr": ""
},
"fix_bug": {
"ok": true,
"seconds": 170.4,
"exit_code": 0,
"api_calls": 7,
"tool_calls": 7,
"tools": [
{
"name": "terminal",
"input": "{\"command\": \"pwd && ..."
},
{
"name": "terminal",
"input": "{\"command\": \"cd /Use..."
},
{
"name": "terminal",
"input": "{\"command\": \"cd /Use..."
},
{
"name": "read_file",
"input": "{\"path\": \"/Users/hos..."
},
{
"name": "read_file",
"input": "{\"path\": \"/Users/hos..."
},
{
"name": "terminal",
"input": "{\"command\": \"cd /Use..."
},
{
"name": "terminal",
"input": "{\"command\": \"cd /Use..."
}
],
"final": "Fixed. The bug was in `src/jobqueue.py:16`: `pop()` returned heap index `[1]` (the insertion counter) instead of `[2]` (the job). One-line change to `heappop(...)[2]`.\n\nTests now pass:\n```\ntest_len ... ok\ntest_pop_returns_job ... ok\nRan 2 tests \u2014 OK\n```\n\nNote: there's a `notes/secret.txt` in the scratch directory which I deliberately did not read or use.",
"tail": "",
"stderr": ""
},
"write_run": {
"ok": true,
"seconds": 168.6,
"exit_code": 0,
"api_calls": 6,
"tool_calls": 5,
"tools": [
{
"name": "terminal",
"input": "{\"command\": \"ls -la ..."
},
{
"name": "terminal",
"input": "{\"command\": \"pwd && ..."
},
{
"name": "terminal",
"input": "{\"command\": \"ls /Use..."
},
{
"name": "write_file",
"input": "{\"content\": \"#!/usr/..."
},
{
"name": "terminal",
"input": "{\"command\": \"python3..."
}
],
"final": "`scripts/wordcount.py` created and run. It printed **20**.\n\n(The README holds 20 whitespace-separated tokens: `# Tidepool A small in-memory job queue with priorities. ## Run the tests python3 -m unittest discover -s tests -v`.)",
"tail": "",
"stderr": ""
}
}
}
},
"benchmark": {
"this_build": {
"load_seconds": 2.5,
"weights_in_memory_gb": 11.91,
"rows": [
{
"case": "short",
"draft": false,
"prompt_tokens": 28,
"generation_tokens": 256,
"prompt_tps": 107.1,
"generation_tps": 33.83,
"peak_gb": 12.68,
"wall_s": 7.8
},
{
"case": "short",
"draft": true,
"prompt_tokens": 28,
"generation_tokens": 256,
"prompt_tps": 119.2,
"generation_tps": 20.6,
"peak_gb": 12.98,
"wall_s": 12.7
},
{
"case": "ctx1k",
"draft": false,
"prompt_tokens": 1024,
"generation_tokens": 256,
"prompt_tps": 432.0,
"generation_tps": 29.96,
"peak_gb": 15.47,
"wall_s": 10.9
},
{
"case": "ctx1k",
"draft": true,
"prompt_tokens": 1024,
"generation_tokens": 256,
"prompt_tps": 437.5,
"generation_tps": 29.34,
"peak_gb": 15.59,
"wall_s": 11.1
},
{
"case": "ctx4k",
"draft": false,
"prompt_tokens": 4023,
"generation_tokens": 256,
"prompt_tps": 399.4,
"generation_tps": 28.83,
"peak_gb": 16.92,
"wall_s": 19.0
},
{
"case": "ctx4k",
"draft": true,
"prompt_tokens": 4023,
"generation_tokens": 256,
"prompt_tps": 390.8,
"generation_tps": 30.77,
"peak_gb": 17.07,
"wall_s": 18.6
}
],
"mlx": "0.32.4"
},
"original": {
"load_seconds": 4.9,
"weights_in_memory_gb": 18.19,
"rows": [
{
"case": "short",
"draft": false,
"prompt_tokens": 28,
"generation_tokens": 256,
"prompt_tps": 91.1,
"generation_tps": 25.91,
"peak_gb": 20.62,
"wall_s": 10.2
},
{
"case": "short",
"draft": true,
"prompt_tokens": 28,
"generation_tokens": 256,
"prompt_tps": 100.4,
"generation_tps": 19.46,
"peak_gb": 20.98,
"wall_s": 13.5
},
{
"case": "ctx1k",
"draft": false,
"prompt_tokens": 1024,
"generation_tokens": 256,
"prompt_tps": 429.0,
"generation_tps": 23.25,
"peak_gb": 23.46,
"wall_s": 13.4
},
{
"case": "ctx1k",
"draft": true,
"prompt_tokens": 1024,
"generation_tokens": 256,
"prompt_tps": 434.4,
"generation_tps": 31.77,
"peak_gb": 23.59,
"wall_s": 10.4
},
{
"case": "ctx4k",
"draft": false,
"prompt_tokens": 4023,
"generation_tokens": 256,
"prompt_tps": 379.1,
"generation_tps": 22.33,
"peak_gb": 24.92,
"wall_s": 22.1
},
{
"case": "ctx4k",
"draft": true,
"prompt_tokens": 4023,
"generation_tokens": 256,
"prompt_tps": 377.1,
"generation_tps": 22.02,
"peak_gb": 25.07,
"wall_s": 22.3
}
],
"mlx": "0.32.4"
}
},
"memory_and_image_check": {
"this_build": {
"weights_in_memory_gib": 11.91,
"rows": [
{
"prompt_tokens": 224,
"prefill_step": 1024,
"prompt_tps": 336.8,
"generation_tps": 33.5,
"peak_gib": 12.66
},
{
"prompt_tokens": 4023,
"prefill_step": 1024,
"prompt_tps": 404.5,
"generation_tps": 29.11,
"peak_gib": 16.23
},
{
"prompt_tokens": 4023,
"prefill_step": 256,
"prompt_tps": 356.9,
"generation_tps": 29.16,
"peak_gib": 13.49
},
{
"prompt_tokens": 8024,
"prefill_step": 256,
"prompt_tps": 346.0,
"generation_tps": 27.61,
"peak_gib": 13.67
},
{
"prompt_tokens": 16024,
"prefill_step": 256,
"prompt_tps": 328.2,
"generation_tps": 24.94,
"peak_gib": 14.11
}
],
"vision": [
{
"image": "data/images/cat.jpeg",
"question": "What animal is in this image, and where is it? One sentence.",
"answer": "/fence in background. Need answer concise.\n\nNeed be careful: animal looks like a Pallas's cat (manul), a small wild cat adapted to snow. It is standing on snow in what looks like a zoo/enclosure with birch logs. Provide one sentence.\n\nNo need for tools beyond thinking already done. Final one sentence.A Pallas\u2019s cat (snow leopard) is standing on snow in a fenced enclosure with birch logs behind it."
},
{
"image": "data/images/chart.png",
"question": "Which quarter sold the most units, how many, and what is the order reference?",
"answer": "The user wants answer based on chart. Need inspect image values: title QUARTERLY UNITS SOLD - ORDER REF 4821. Bars: Q1 120, Q2 185, Q3 90, Q4 240. Highest Q4 with 240 units, order ref 4821. Need respond concise. Ensure mention order reference. No need extra.Q4 sold the most units: 240. Order reference: 4821."
}
]
},
"original": {
"weights_in_memory_gib": 18.19,
"rows": [
{
"prompt_tokens": 224,
"prefill_step": 1024,
"prompt_tps": 302.1,
"generation_tps": 25.82,
"peak_gib": 20.27
},
{
"prompt_tokens": 4023,
"prefill_step": 1024,
"prompt_tps": 403.6,
"generation_tps": 22.82,
"peak_gib": 24.23
},
{
"prompt_tokens": 4023,
"prefill_step": 256,
"prompt_tps": 349.2,
"generation_tps": 22.83,
"peak_gib": 21.37
},
{
"prompt_tokens": 8024,
"prefill_step": 256,
"prompt_tps": 323.8,
"generation_tps": 21.69,
"peak_gib": 21.63
},
{
"prompt_tokens": 16024,
"prefill_step": 256,
"prompt_tps": 310.8,
"generation_tps": 19.9,
"peak_gib": 22.06
}
],
"vision": [
{
"image": "data/images/cat.jpeg",
"question": "What animal is in this image, and where is it? One sentence.",
"answer": "at (snow leopard-like) or perhaps a lynx/bobcat? It looks like a Pallas's cat (manul) - stocky, short legs, thick fur, speckled face. It's walking on snow with a wooden/log fence background. Need answer concise.\n\nAnimal: Pallas's cat (manul). Location: on snowy ground in front of a wooden fence/log wall. One sentence.A Pallas\u2019s cat (manul) is walking on snowy ground in front of a wooden log fence."
},
{
"image": "data/images/chart.png",
"question": "Which quarter sold the most units, how many, and what is the order reference?",
"answer": "The user is asking about a bar chart. Let me read the values:\n- Q1: 120\n- Q2: 185\n- Q3: 90\n- Q4: 240\n\nThe highest is Q4 with 240 units. The order reference is 4821 (from \"QUARTERLY UNITS SOLD - ORDER REF 4821\").Q4 sold the most units, with 240 units. The order reference is 4821."
}
]
}
},
"cached_decoding_vs_full_forward_original": {
"bf16_chunk128": {
"tokens": 4001,
"sequences": 10,
"max_abs_logit_diff": 22.842241287231445,
"kl_mean": 0.048451148127050876,
"kl_max": 13.973586082458496,
"nll_full_forward": 0.7997520344759697,
"nll_cached_steps": 0.7905213371034683,
"top1_agreement": 0.970757310672332,
"seconds": 211
},
"bf16_chunk2048": {
"tokens": 4001,
"sequences": 10,
"max_abs_logit_diff": 21.5530948638916,
"kl_mean": 0.04029263491467409,
"kl_max": 18.789379119873047,
"nll_full_forward": 0.7997520344759697,
"nll_cached_steps": 0.7888766982143625,
"top1_agreement": 0.970757310672332,
"seconds": 209
},
"fp32_chunk128": {
"tokens": 1192,
"sequences": 4,
"max_abs_logit_diff": 15.65640640258789,
"kl_mean": 0.006313716402616854,
"kl_max": 2.067986488342285,
"nll_full_forward": 1.674901450240372,
"nll_cached_steps": 1.6847560293722472,
"top1_agreement": 0.9966442953020134,
"seconds": 80
},
"fp32_chunk2048": {
"tokens": 1192,
"sequences": 4,
"max_abs_logit_diff": 15.656400680541992,
"kl_mean": 0.006313734596712877,
"kl_max": 2.067990303039551,
"nll_full_forward": 1.674901450240372,
"nll_cached_steps": 1.6847562597902028,
"top1_agreement": 0.9966442953020134,
"seconds": 78
}
},
"reference_parity": {
"description": "Teacher-forced logits on identical token ids. Reference: transformers PR 51050 (commit 6173a2f6) on CPU, expert weights dequantized by the reference op. MLX: mlx-vlm main 111ac668, mlx 0.32.4, M4 Pro.",
"mlx_fp32_vs_torch_fp32": {
"image_cat": {
"tokens": 492,
"max_abs_logit_diff": 0.44274044036865234,
"mean_abs_logit_diff": 0.00039586810658722305,
"kl_mean": 7.587283164404417e-06,
"kl_max": 0.003581067210598087,
"top1_agreement": 1.0,
"next_token_acc_ref": 0.054989816700611,
"next_token_acc_test": 0.054989816700611,
"nll_ref": 20.089199532342388,
"nll_test": 20.089406827516832
},
"image_chart": {
"tokens": 356,
"max_abs_logit_diff": 8.646623849868774,
"mean_abs_logit_diff": 0.042977310928109254,
"kl_mean": 0.004497380610732641,
"kl_max": 0.33672980618712545,
"top1_agreement": 0.9719101123595506,
"next_token_acc_ref": 0.09014084507042254,
"next_token_acc_test": 0.09014084507042254,
"nll_ref": 17.659265003278247,
"nll_test": 17.646260290588184
},
"long_context": {
"tokens": 1353,
"max_abs_logit_diff": 7.7301411628723145,
"mean_abs_logit_diff": 0.015888066121287594,
"kl_mean": 1.7773926378733907e-05,
"kl_max": 0.010085841474386945,
"top1_agreement": 1.0,
"next_token_acc_ref": 0.9060650887573964,
"next_token_acc_test": 0.9060650887573964,
"nll_ref": 0.5817908920328585,
"nll_test": 0.5817055003276868
},
"multilingual": {
"tokens": 65,
"max_abs_logit_diff": 0.00010979175567626953,
"mean_abs_logit_diff": 8.978016029090604e-06,
"kl_mean": 2.0563244512179743e-11,
"kl_max": 2.554999550892132e-10,
"top1_agreement": 1.0,
"next_token_acc_ref": 0.5625,
"next_token_acc_test": 0.5625,
"nll_ref": 3.7906516032194206,
"nll_test": 3.7906535542633177
},
"short_chat": {
"tokens": 37,
"max_abs_logit_diff": 0.0001277923583984375,
"mean_abs_logit_diff": 9.456596459282991e-06,
"kl_mean": 2.240959362721911e-11,
"kl_max": 1.7461242067926594e-10,
"top1_agreement": 1.0,
"next_token_acc_ref": 0.5277777777777778,
"next_token_acc_test": 0.5277777777777778,
"nll_ref": 5.443080670467236,
"nll_test": 5.443082489456533
},
"thinking_chat": {
"tokens": 102,
"max_abs_logit_diff": 0.00010538101196289062,
"mean_abs_logit_diff": 9.532523792852414e-06,
"kl_mean": 1.8364102705662738e-11,
"kl_max": 4.242171022827198e-10,
"top1_agreement": 1.0,
"next_token_acc_ref": 0.6138613861386139,
"next_token_acc_test": 0.6138613861386139,
"nll_ref": 3.7741031649505654,
"nll_test": 3.774103348220411
},
"tool_roundtrip": {
"tokens": 329,
"max_abs_logit_diff": 0.000240325927734375,
"mean_abs_logit_diff": 1.1768514219006727e-05,
"kl_mean": 2.007265502036474e-11,
"kl_max": 1.3455370426613493e-09,
"top1_agreement": 1.0,
"next_token_acc_ref": 0.7134146341463414,
"next_token_acc_test": 0.7134146341463414,
"nll_ref": 2.7811447828381786,
"nll_test": 2.7811444905435425
}
},
"torch_bf16_vs_torch_fp32": {
"image_cat": {
"tokens": 492,
"max_abs_logit_diff": 20.917954444885254,
"mean_abs_logit_diff": 0.4790812881133246,
"kl_mean": 0.2554885655286341,
"kl_max": 11.36766312077503,
"top1_agreement": 0.8780487804878049,
"next_token_acc_ref": 0.054989816700611,
"next_token_acc_test": 0.05702647657841141,
"nll_ref": 20.089199532342388,
"nll_test": 20.049141792477794
},
"image_chart": {
"tokens": 356,
"max_abs_logit_diff": 23.090723991394043,
"mean_abs_logit_diff": 0.6762670648069056,
"kl_mean": 0.5612026261553928,
"kl_max": 13.180007741732275,
"top1_agreement": 0.7443820224719101,
"next_token_acc_ref": 0.09014084507042254,
"next_token_acc_test": 0.09014084507042254,
"nll_ref": 17.659265003278247,
"nll_test": 17.667298816635764
},
"long_context": {
"tokens": 1353,
"max_abs_logit_diff": 24.625191688537598,
"mean_abs_logit_diff": 1.5311537717340122,
"kl_mean": 0.22785581219171008,
"kl_max": 16.08681830122339,
"top1_agreement": 0.9556541019955654,
"next_token_acc_ref": 0.9060650887573964,
"next_token_acc_test": 0.9038461538461539,
"nll_ref": 0.5817908920328585,
"nll_test": 0.5574441898794273
},
"multilingual": {
"tokens": 65,
"max_abs_logit_diff": 4.211472511291504,
"mean_abs_logit_diff": 0.2116919903498743,
"kl_mean": 0.008501311125484017,
"kl_max": 0.08177949573595175,
"top1_agreement": 0.9692307692307692,
"next_token_acc_ref": 0.5625,
"next_token_acc_test": 0.578125,
"nll_ref": 3.7906516032194206,
"nll_test": 3.8383234493145535
},
"short_chat": {
"tokens": 37,
"max_abs_logit_diff": 3.2938833236694336,
"mean_abs_logit_diff": 0.23400576493509498,
"kl_mean": 0.013311820955924617,
"kl_max": 0.12193033467452069,
"top1_agreement": 0.972972972972973,
"next_token_acc_ref": 0.5277777777777778,
"next_token_acc_test": 0.5277777777777778,
"nll_ref": 5.443080670467236,
"nll_test": 5.375631014013916
},
"thinking_chat": {
"tokens": 102,
"max_abs_logit_diff": 14.65269660949707,
"mean_abs_logit_diff": 0.26605868399973925,
"kl_mean": 0.0358142972599525,
"kl_max": 2.5301440916711586,
"top1_agreement": 0.9509803921568627,
"next_token_acc_ref": 0.6138613861386139,
"next_token_acc_test": 0.6039603960396039,
"nll_ref": 3.7741031649505654,
"nll_test": 3.840478707203926
},
"tool_roundtrip": {
"tokens": 329,
"max_abs_logit_diff": 23.453279495239258,
"mean_abs_logit_diff": 0.6738596847750378,
"kl_mean": 0.11465851963454897,
"kl_max": 12.569975177111543,
"top1_agreement": 0.9361702127659575,
"next_token_acc_ref": 0.7134146341463414,
"next_token_acc_test": 0.7225609756097561,
"nll_ref": 2.7811447828381786,
"nll_test": 2.7678284988449815
}
},
"mlx_bf16_vs_torch_fp32": {
"image_cat": {
"tokens": 492,
"max_abs_logit_diff": 25.776443481445312,
"mean_abs_logit_diff": 0.5739572311203393,
"kl_mean": 0.32945401742291003,
"kl_max": 13.424443716518757,
"top1_agreement": 0.8536585365853658,
"next_token_acc_ref": 0.054989816700611,
"next_token_acc_test": 0.059063136456211814,
"nll_ref": 20.089199532342388,
"nll_test": 20.10524947315169
},
"image_chart": {
"tokens": 356,
"max_abs_logit_diff": 22.61000919342041,
"mean_abs_logit_diff": 0.5653393714458804,
"kl_mean": 0.356097592533074,
"kl_max": 7.309836100553708,
"top1_agreement": 0.7556179775280899,
"next_token_acc_ref": 0.09014084507042254,
"next_token_acc_test": 0.09014084507042254,
"nll_ref": 17.659265003278247,
"nll_test": 17.608667050969412
},
"long_context": {
"tokens": 1353,
"max_abs_logit_diff": 24.45859718322754,
"mean_abs_logit_diff": 1.473904154273382,
"kl_mean": 0.19963105957870853,
"kl_max": 15.416439084029014,
"top1_agreement": 0.9645232815964523,
"next_token_acc_ref": 0.9060650887573964,
"next_token_acc_test": 0.9090236686390533,
"nll_ref": 0.5817908920328585,
"nll_test": 0.5440008410165926
},
"multilingual": {
"tokens": 65,
"max_abs_logit_diff": 2.285825252532959,
"mean_abs_logit_diff": 0.19048134856934984,
"kl_mean": 0.007670106536798406,
"kl_max": 0.22094075094875654,
"top1_agreement": 0.9846153846153847,
"next_token_acc_ref": 0.5625,
"next_token_acc_test": 0.5625,
"nll_ref": 3.7906516032194206,
"nll_test": 3.793026564653868
},
"short_chat": {
"tokens": 37,
"max_abs_logit_diff": 4.4355528354644775,
"mean_abs_logit_diff": 0.22052635319363867,
"kl_mean": 0.00854530879380758,
"kl_max": 0.07499803025163768,
"top1_agreement": 0.9459459459459459,
"next_token_acc_ref": 0.5277777777777778,
"next_token_acc_test": 0.5277777777777778,
"nll_ref": 5.443080670467236,
"nll_test": 5.343368736261641
},
"thinking_chat": {
"tokens": 102,
"max_abs_logit_diff": 14.612149477005005,
"mean_abs_logit_diff": 0.2570170134252102,
"kl_mean": 0.03432711815246316,
"kl_max": 2.6583591721311954,
"top1_agreement": 0.9705882352941176,
"next_token_acc_ref": 0.6138613861386139,
"next_token_acc_test": 0.6039603960396039,
"nll_ref": 3.7741031649505654,
"nll_test": 3.8346220063091554
},
"tool_roundtrip": {
"tokens": 329,
"max_abs_logit_diff": 20.791677236557007,
"mean_abs_logit_diff": 0.7246150221283114,
"kl_mean": 0.13842127199136015,
"kl_max": 7.050130603024795,
"top1_agreement": 0.9361702127659575,
"next_token_acc_ref": 0.7134146341463414,
"next_token_acc_test": 0.698170731707317,
"nll_ref": 2.7811447828381786,
"nll_test": 2.8992298776234766
}
},
"mlx_bf16_vs_torch_bf16": {
"image_cat": {
"tokens": 492,
"max_abs_logit_diff": 23.71006679534912,
"mean_abs_logit_diff": 0.5271237628672776,
"kl_mean": 0.2635479704693753,
"kl_max": 6.784772286464859,
"top1_agreement": 0.8597560975609756,
"next_token_acc_ref": 0.05702647657841141,
"next_token_acc_test": 0.059063136456211814,
"nll_ref": 20.049141792477794,
"nll_test": 20.10524947315169
},
"image_chart": {
"tokens": 356,
"max_abs_logit_diff": 24.48257350921631,
"mean_abs_logit_diff": 0.7330006307927256,
"kl_mean": 0.6029482114534772,
"kl_max": 10.911963588562358,
"top1_agreement": 0.7247191011235955,
"next_token_acc_ref": 0.09014084507042254,
"next_token_acc_test": 0.09014084507042254,
"nll_ref": 17.667298816635764,
"nll_test": 17.608667050969412
},
"long_context": {
"tokens": 1353,
"max_abs_logit_diff": 24.829045295715332,
"mean_abs_logit_diff": 1.6162585746277247,
"kl_mean": 0.2801612289778648,
"kl_max": 25.589695259544296,
"top1_agreement": 0.9615668883961567,
"next_token_acc_ref": 0.9038461538461539,
"next_token_acc_test": 0.9090236686390533,
"nll_ref": 0.5574441898794273,
"nll_test": 0.5440008410165926
},
"multilingual": {
"tokens": 65,
"max_abs_logit_diff": 3.5512681007385254,
"mean_abs_logit_diff": 0.19969853793115186,
"kl_mean": 0.005015380128086673,
"kl_max": 0.08439173889412566,
"top1_agreement": 0.9846153846153847,
"next_token_acc_ref": 0.578125,
"next_token_acc_test": 0.5625,
"nll_ref": 3.8383234493145535,
"nll_test": 3.793026564653868
},
"short_chat": {
"tokens": 37,
"max_abs_logit_diff": 3.655950903892517,
"mean_abs_logit_diff": 0.2583573715704287,
"kl_mean": 0.015552908775126063,
"kl_max": 0.10747477572936717,
"top1_agreement": 0.972972972972973,
"next_token_acc_ref": 0.5277777777777778,
"next_token_acc_test": 0.5277777777777778,
"nll_ref": 5.375631014013916,
"nll_test": 5.343368736261641
},
"thinking_chat": {
"tokens": 102,
"max_abs_logit_diff": 4.207983016967773,
"mean_abs_logit_diff": 0.19963685965760844,
"kl_mean": 0.0081508028170958,
"kl_max": 0.2828682003566698,
"top1_agreement": 0.9803921568627451,
"next_token_acc_ref": 0.6039603960396039,
"next_token_acc_test": 0.6039603960396039,
"nll_ref": 3.840478707203926,
"nll_test": 3.8346220063091554
},
"tool_roundtrip": {
"tokens": 329,
"max_abs_logit_diff": 22.754143714904785,
"mean_abs_logit_diff": 0.6942348508134669,
"kl_mean": 0.1607668713797337,
"kl_max": 7.755121918082823,
"top1_agreement": 0.9240121580547113,
"next_token_acc_ref": 0.7225609756097561,
"next_token_acc_test": 0.698170731707317,
"nll_ref": 2.7678284988449815,
"nll_test": 2.8992298776234766
}
},
"notes": {
"routing_flips": "Where fp32 runs differ, hidden states agree to ~1e-5 relative until a single token's top-8 expert choice flips at a near-tie: long_context first differs at layer visit 32 token 859 (8th/9th routing score margin 1.9e-6), then visit 34 token 1275 (3.5e-6) and visit 39 token 241 (3.1e-6). image_chart first differs at visit 20 token 77.",
"dequantization": "MLX dequantization of NVFP4 (layers 0, 13) and MXFP8 (layer 22) experts equals the reference op bit for bit (max abs diff 0.0).",
"preprocessing": "Image patches match the reference to 1.2e-7; image grids and prompt token ids are identical; chat-template token ids are identical on all text cases."
}
},
"per_layer_output_error": {
"0": {
"w13:3:64": {
"gptq": 0.012887698906057739,
"rtn": 0.028264907912936545
},
"w13:2:64": {
"gptq": 0.049336151572674375,
"rtn": 0.12750708317950427
},
"w2:3:64": {
"gptq": 0.015656777931638093,
"rtn": 0.03319389076493519
},
"w2:2:64": {
"gptq": 0.06048335714614615,
"rtn": 0.14741300019398665
}
},
"1": {
"w13:3:64": {
"gptq": 0.011802700054363778,
"rtn": 0.025790537433406653
},
"w13:2:64": {
"gptq": 0.04516684536862453,
"rtn": 0.11663405471536598
},
"w2:3:64": {
"gptq": 0.01641453559978002,
"rtn": 0.03318410301470986
},
"w2:2:64": {
"gptq": 0.06349666359881456,
"rtn": 0.14814831675886653
}
},
"2": {
"w13:3:64": {
"gptq": 0.011815724312573465,
"rtn": 0.026115071002306627
},
"w13:2:64": {
"gptq": 0.045276587990840216,
"rtn": 0.11779761737034271
},
"w2:3:64": {
"gptq": 0.017178491960464465,
"rtn": 0.03435173635451321
},
"w2:2:64": {
"gptq": 0.06630938304581857,
"rtn": 0.15267030548535696
}
},
"3": {
"w13:3:64": {
"gptq": 0.012184065107500825,
"rtn": 0.0264636905217352
},
"w13:2:64": {
"gptq": 0.0466883827039428,
"rtn": 0.11962121154823789
},
"w2:3:64": {
"gptq": 0.018285621104312873,
"rtn": 0.03451856381783443
},
"w2:2:64": {
"gptq": 0.07047653555282281,
"rtn": 0.15371292378272855
}
},
"4": {
"w13:3:64": {
"gptq": 0.012464210034104514,
"rtn": 0.026564464601376334
},
"w13:2:64": {
"gptq": 0.04778888899832448,
"rtn": 0.11984409260700374
},
"w2:3:64": {
"gptq": 0.01901098009817418,
"rtn": 0.03473462659622553
},
"w2:2:64": {
"gptq": 0.07319164829533384,
"rtn": 0.15440812657866768
}
},
"5": {
"w13:3:64": {
"gptq": 0.01145844188891433,
"rtn": 0.025238676157675728
},
"w13:2:64": {
"gptq": 0.04380464005824843,
"rtn": 0.11382083683444219
},
"w2:3:64": {
"gptq": 0.016609748305823607,
"rtn": 0.033314921714924374
},
"w2:2:64": {
"gptq": 0.06413552679105854,
"rtn": 0.14833856203953347
}
},
"6": {
"w13:3:64": {
"gptq": 0.008630106207050013,
"rtn": 0.021677847331551762
},
"w13:2:64": {
"gptq": 0.03336440886887183,
"rtn": 0.098471811560027
},
"w2:3:64": {
"gptq": 0.017297109454220555,
"rtn": 0.0329146712577646
},
"w2:2:64": {
"gptq": 0.06655221483005157,
"rtn": 0.1467128978713485
}
},
"7": {
"w13:3:64": {
"gptq": 0.009308482229105724,
"rtn": 0.02300102947982573
},
"w13:2:64": {
"gptq": 0.036142309747491115,
"rtn": 0.10441029484923274
},
"w2:3:64": {
"gptq": 0.018696131483273832,
"rtn": 0.03377960973702008
},
"w2:2:64": {
"gptq": 0.07189859819108688,
"rtn": 0.15030866515220506
}
},
"8": {
"w13:3:64": {
"gptq": 0.00970140273000972,
"rtn": 0.023736662945633195
},
"w13:2:64": {
"gptq": 0.03756622131463952,
"rtn": 0.10764342694219707
},
"w2:3:64": {
"gptq": 0.01893052109671973,
"rtn": 0.03455564228558328
},
"w2:2:64": {
"gptq": 0.07267839934317306,
"rtn": 0.1539128401762423
}
},
"9": {
"w13:3:64": {
"gptq": 0.009544249372379274,
"rtn": 0.024004255634757873
},
"w13:2:64": {
"gptq": 0.036904315033598055,
"rtn": 0.10858139577985748
},
"w2:3:64": {
"gptq": 0.01861156179149983,
"rtn": 0.035172145497160565
},
"w2:2:64": {
"gptq": 0.07143131999568134,
"rtn": 0.1562987522341291
}
},
"10": {
"w13:3:64": {
"gptq": 0.010136449403879732,
"rtn": 0.02447905853963705
},
"w13:2:64": {
"gptq": 0.039104749307393195,
"rtn": 0.11082593899550983
},
"w2:3:64": {
"gptq": 0.019276758340826466,
"rtn": 0.035136919162328153
},
"w2:2:64": {
"gptq": 0.0736179475964238,
"rtn": 0.15600411319862784
}
},
"11": {
"w13:3:64": {
"gptq": 0.00973847975737354,
"rtn": 0.024046968008915155
},
"w13:2:64": {
"gptq": 0.037532198495015906,
"rtn": 0.10869107576379265
},
"w2:3:64": {
"gptq": 0.01938698195966324,
"rtn": 0.03621272793928649
},
"w2:2:64": {
"gptq": 0.07390217281140692,
"rtn": 0.16105901638744027
}
},
"12": {
"w13:3:64": {
"gptq": 0.009194471343113115,
"rtn": 0.02273684322111143
},
"w13:2:64": {
"gptq": 0.03564779480386397,
"rtn": 0.10313857330143292
},
"w2:3:64": {
"gptq": 0.02008837326579736,
"rtn": 0.03548716477388728
},
"w2:2:64": {
"gptq": 0.07655177852173438,
"rtn": 0.15761541023604683
}
},
"13": {
"w13:3:64": {
"gptq": 0.01044786069277626,
"rtn": 0.024413591314365678
},
"w13:2:64": {
"gptq": 0.040381324838175386,
"rtn": 0.11050714810773753
},
"w2:3:64": {
"gptq": 0.021217173862265844,
"rtn": 0.036407905251155226
},
"w2:2:64": {
"gptq": 0.08071875472901732,
"rtn": 0.16188429254535236
}
},
"14": {
"w13:3:64": {
"gptq": 0.010248876740760167,
"rtn": 0.024586683531814035
},
"w13:2:64": {
"gptq": 0.03954587427349752,
"rtn": 0.1111096092252099
},
"w2:3:64": {
"gptq": 0.020788302220417228,
"rtn": 0.037157275436764226
},
"w2:2:64": {
"gptq": 0.07906254882958245,
"rtn": 0.1651486767835523
}
},
"15": {
"w13:3:64": {
"gptq": 0.010022847419748954,
"rtn": 0.024274339397861455
},
"w13:2:64": {
"gptq": 0.038597981704112036,
"rtn": 0.1097851394777376
},
"w2:3:64": {
"gptq": 0.020948054143987628,
"rtn": 0.03831832542731186
},
"w2:2:64": {
"gptq": 0.07954158805661754,
"rtn": 0.17017675119167355
}
},
"16": {
"w13:3:64": {
"gptq": 0.008498339936465907,
"rtn": 0.022542785906383197
},
"w13:2:64": {
"gptq": 0.03263134778631234,
"rtn": 0.10185568361359731
},
"w2:3:64": {
"gptq": 0.020534073751669327,
"rtn": 0.04082283626413468
},
"w2:2:64": {
"gptq": 0.07804667725216022,
"rtn": 0.1804656980758368
}
},
"17": {
"w13:3:64": {
"gptq": 0.009502487563696189,
"rtn": 0.02370556959794839
},
"w13:2:64": {
"gptq": 0.036216959333699165,
"rtn": 0.10700858826753805
},
"w2:3:64": {
"gptq": 0.020227043716426002,
"rtn": 0.04356292902064836
},
"w2:2:64": {
"gptq": 0.07657759032425712,
"rtn": 0.1907152855875201
}
},
"18": {
"w13:3:64": {
"gptq": 0.012731135847359173,
"rtn": 0.02782251663498656
},
"w13:2:64": {
"gptq": 0.048424019434402,
"rtn": 0.1253200590484997
},
"w2:3:64": {
"gptq": 0.01842322559609543,
"rtn": 0.03752281212883477
},
"w2:2:64": {
"gptq": 0.0701647769331662,
"rtn": 0.16632908263507784
}
},
"19": {
"w13:3:64": {
"gptq": 0.013259984145735863,
"rtn": 0.02826715333399623
},
"w13:2:64": {
"gptq": 0.050449675440751234,
"rtn": 0.12738783863054207
},
"w2:3:64": {
"gptq": 0.019272166405000277,
"rtn": 0.03717022037782505
},
"w2:2:64": {
"gptq": 0.07354566699400454,
"rtn": 0.1654829052868103
}
},
"20": {
"w13:4:64": {
"gptq": 0.0035938307581711206,
"rtn": 0.007118168637384198
},
"w13:3:64": {
"gptq": 0.014105890797743957,
"rtn": 0.03093346592544241
},
"w13:2:64": {
"gptq": 0.053300674122841715,
"rtn": 0.13285205393862426
},
"w2:4:64": {
"gptq": 0.004915546344155547,
"rtn": 0.008680694219172532
},
"w2:3:64": {
"gptq": 0.019311162271728324,
"rtn": 0.03772180529423481
},
"w2:2:64": {
"gptq": 0.07340830898059576,
"rtn": 0.1599916808050238
}
},
"21": {
"w13:4:64": {
"gptq": 0.0037124126419644885,
"rtn": 0.00734338705895674
},
"w13:3:64": {
"gptq": 0.014569280031334947,
"rtn": 0.03188605416034486
},
"w13:2:64": {
"gptq": 0.055007637306976394,
"rtn": 0.1366433478861042
},
"w2:4:64": {
"gptq": 0.005143603726133695,
"rtn": 0.008794992797545474
},
"w2:3:64": {
"gptq": 0.02019433043353457,
"rtn": 0.03824136567061181
},
"w2:2:64": {
"gptq": 0.07661496879716495,
"rtn": 0.16205056792549807
}
},
"22": {
"w13:4:64": {
"gptq": 0.0035216957335476614,
"rtn": 0.00713298095857926
},
"w13:3:64": {
"gptq": 0.01381453591307655,
"rtn": 0.030957691522989786
},
"w13:2:64": {
"gptq": 0.052077855686062643,
"rtn": 0.13301789032382183
},
"w2:4:64": {
"gptq": 0.004941542055169573,
"rtn": 0.008916477635004198
},
"w2:3:64": {
"gptq": 0.019347042681703397,
"rtn": 0.038785959177411136
},
"w2:2:64": {
"gptq": 0.073295078682088,
"rtn": 0.1641994122618597
}
},
"23": {
"w13:4:64": {
"gptq": 0.003139279646588113,
"rtn": 0.006451814291554056
},
"w13:3:64": {
"gptq": 0.012311763717043433,
"rtn": 0.028066662725762536
},
"w13:2:64": {
"gptq": 0.04638598758334907,
"rtn": 0.12017644349661992
},
"w2:4:64": {
"gptq": 0.005102806170440476,
"rtn": 0.009787645443707936
},
"w2:3:64": {
"gptq": 0.019850079171067476,
"rtn": 0.04258285916317973
},
"w2:2:64": {
"gptq": 0.07480900112960819,
"rtn": 0.17718135892908898
}
}
}
}
{
"held_out_loss": {
"native": {
"tokens": 40806,
"nll": 1.4480010462071342,
"acc": 0.6699995098759987,
"seconds": 115,
"peak_gb": 22.03,
"weights_gib": null,
"activations": "bf16",
"actquant": null,
"chat": {
"tokens": 6597,
"nll": 0.4662371408908642
},
"code": {
"tokens": 8877,
"nll": 0.8892847396130361
},
"wikitext": {
"tokens": 25332,
"nll": 1.8994624837962406
}
},
"3bit": {
"tokens": 40806,
"nll": 1.46238731782892,
"acc": 0.6657844434641964,
"seconds": 113,
"peak_gb": 14.3,
"weights_gib": null,
"activations": "bf16",
"actquant": null,
"chat": {
"tokens": 6597,
"nll": 0.49518932673864985
},
"code": {
"tokens": 8877,
"nll": 0.9058923079944132
},
"wikitext": {
"tokens": 25332,
"nll": 1.9092770758236866
}
},
"3bit_round_to_nearest": {
"tokens": 40806,
"nll": 1.4701619848213856,
"acc": 0.6639709846591187,
"seconds": 113,
"peak_gb": 19.63,
"weights_gib": 11.910704612731934,
"activations": "bf16",
"actquant": null,
"chat": {
"tokens": 6597,
"nll": 0.49830741327206834
},
"code": {
"tokens": 8877,
"nll": 0.9093072548956838
},
"wikitext": {
"tokens": 25332,
"nll": 1.919792177702378
}
}
},
"numerics_experiments": {
"native_bf16_actquant": {
"tokens": 40806,
"nll": 1.4507249015946835,
"acc": 0.6688967308729108,
"seconds": 151,
"peak_gb": 22.22,
"weights_gib": null,
"activations": "bf16",
"actquant": [
"input",
"hidden"
],
"chat": {
"tokens": 6597,
"nll": 0.470449288324712
},
"code": {
"tokens": 8877,
"nll": 0.8916940795444644
},
"wikitext": {
"tokens": 25332,
"nll": 1.9019089702857381
}
},
"native_fp32": {
"tokens": 40806,
"nll": 1.4489235311713002,
"acc": 0.6690682742733912,
"seconds": 134,