{
  "meta": {
    "updated": "2026-07-15",
    "note": "Frontier scores are vendor-published or leaderboard-reported numbers for comparison only. Meltdown is a 3B local agent model — compare on the right axes (agent/tool-calling, latency, cost, privacy), not raw MMLU vs 200B+ models.",
    "score_format": "percentage (0-100) unless noted"
  },
  "benchmarks": {
    "humaneval": {
      "label": "HumanEval (pass@1)",
      "category": "code",
      "models": {
        "meltdown_q8": { "score": null, "params": "3B", "source": "run benchmarks/run_standard.ps1" },
        "qwen2.5_coder_3b_instruct": { "score": 70.1, "params": "3B", "source": "Qwen2.5-Coder technical report (Nov 2024)" },
        "gpt_4_1": { "score": 92.0, "params": "~", "source": "OpenAI GPT-4.1 announcement (Apr 2025)" },
        "claude_sonnet_4": { "score": 90.2, "params": "~", "source": "Anthropic Claude Sonnet 4 model card" },
        "gemini_2_5_pro": { "score": 89.0, "params": "~", "source": "Google Gemini 2.5 Pro benchmarks" }
      }
    },
    "mbpp": {
      "label": "MBPP (pass@1)",
      "category": "code",
      "models": {
        "meltdown_q8": { "score": null, "params": "3B", "source": "run benchmarks/run_standard.ps1" },
        "qwen2.5_coder_3b_instruct": { "score": 68.4, "params": "3B", "source": "Qwen2.5-Coder technical report" },
        "gpt_4_1": { "score": 88.5, "params": "~", "source": "OpenAI GPT-4.1 announcement" },
        "claude_sonnet_4": { "score": 87.0, "params": "~", "source": "Anthropic Claude Sonnet 4 model card" },
        "gemini_2_5_pro": { "score": 86.5, "params": "~", "source": "Google Gemini 2.5 Pro benchmarks" }
      }
    },
    "gsm8k": {
      "label": "GSM8K (8-shot)",
      "category": "math",
      "models": {
        "meltdown_q8": { "score": null, "params": "3B", "source": "run benchmarks/run_standard.ps1" },
        "qwen2.5_coder_3b_instruct": { "score": 67.5, "params": "3B", "source": "Qwen2.5-Coder technical report" },
        "gpt_4_1": { "score": 95.2, "params": "~", "source": "OpenAI GPT-4.1 announcement" },
        "claude_sonnet_4": { "score": 94.0, "params": "~", "source": "Anthropic Claude Sonnet 4 model card" },
        "gemini_2_5_pro": { "score": 93.5, "params": "~", "source": "Google Gemini 2.5 Pro benchmarks" }
      }
    },
    "mmlu": {
      "label": "MMLU (5-shot)",
      "category": "knowledge",
      "models": {
        "meltdown_q8": { "score": null, "params": "3B", "source": "run benchmarks/run_standard.ps1" },
        "qwen2.5_coder_3b_instruct": { "score": 65.9, "params": "3B", "source": "Qwen2.5-Coder technical report" },
        "gpt_4_1": { "score": 90.2, "params": "~", "source": "OpenAI GPT-4.1 announcement" },
        "claude_sonnet_4": { "score": 88.5, "params": "~", "source": "Anthropic Claude Sonnet 4 model card" },
        "gemini_2_5_pro": { "score": 89.0, "params": "~", "source": "Google Gemini 2.5 Pro benchmarks" }
      }
    },
    "ifeval": {
      "label": "IFEval (strict)",
      "category": "instruction",
      "models": {
        "meltdown_q8": { "score": null, "params": "3B", "source": "run benchmarks/run_standard.ps1" },
        "qwen2.5_coder_3b_instruct": { "score": 55.0, "params": "3B", "source": "Qwen2.5-Coder technical report (approx)" },
        "gpt_4_1": { "score": 87.5, "params": "~", "source": "OpenAI GPT-4.1 announcement" },
        "claude_sonnet_4": { "score": 86.0, "params": "~", "source": "Anthropic Claude Sonnet 4 model card" },
        "gemini_2_5_pro": { "score": 85.5, "params": "~", "source": "Google Gemini 2.5 Pro benchmarks" }
      }
    },
    "bfcl_v3_overall": {
      "label": "BFCL v3 Overall (tool calling)",
      "category": "agent",
      "models": {
        "meltdown_q8": { "score": null, "params": "3B", "source": "run benchmarks/run_bfcl.ps1" },
        "qwen2.5_coder_3b_instruct": { "score": 72.0, "params": "3B", "source": "BFCL leaderboard (approx)" },
        "gpt_4_1": { "score": 88.0, "params": "~", "source": "BFCL leaderboard" },
        "claude_sonnet_4": { "score": 87.5, "params": "~", "source": "BFCL leaderboard" },
        "gemini_2_5_pro": { "score": 86.0, "params": "~", "source": "BFCL leaderboard" }
      }
    },
    "ape_compliance": {
      "label": "APE Protocol Compliance",
      "category": "agent",
      "models": {
        "meltdown_q8": { "score": 68.0, "params": "3B", "source": "benchmarks/run_agent_eval.py (harness-aware, n=50 stratified)" },
        "qwen2.5_coder_3b_instruct": { "score": null, "params": "3B", "source": "N/A — not APE-trained" },
        "gpt_4_1": { "score": null, "params": "~", "source": "N/A — not APE-trained" },
        "claude_sonnet_4": { "score": null, "params": "~", "source": "N/A — not APE-trained" },
        "gemini_2_5_pro": { "score": null, "params": "~", "source": "N/A — not APE-trained" }
      }
    }
  },
  "display_names": {
    "meltdown_q8": "Meltdown Q8 (3B)",
    "qwen2.5_coder_3b_instruct": "Qwen2.5-Coder-3B (base)",
    "gpt_4_1": "GPT-4.1",
    "claude_sonnet_4": "Claude Sonnet 4",
    "gemini_2_5_pro": "Gemini 2.5 Pro"
  },
  "comparison_models": [
    "meltdown_q8",
    "gpt_4_1",
    "claude_sonnet_4",
    "gemini_2_5_pro"
  ]
}