results.json

{
  "benchmark": "ToolUse.1",
  "version": "1.0.0",
  "updated": "2026-10-18",
  "corpus": {
    "cases": 114,
    "call_cases": 78,
    "no_call_cases": 36,
    "by_language": {
      "en": 114
    },
    "by_category": {
      "calculator": 15,
      "get_weather": 8,
      "web_search": 8,
      "translate": 8,
      "convert_currency": 8,
      "convert_units": 8,
      "set_timer": 6,
      "save_note": 6,
      "get_time": 6,
      "negative": 30,
      "clarification": 5,
      "multi_step": 6
    },
    "sha256": "17a6ef84c22648d83add6f56a03b24f896420837f4e69039c22d61cf8ad2c3e1"
  },
  "rows": [
    {
      "model": "oracle",
      "link": "",
      "protocol": "all",
      "score": 1.0,
      "score_ci95": [
        0.9889,
        1.0
      ],
      "correct_call": 1.0,
      "abstention": 1.0,
      "spurious_call_rate": 0.0,
      "n_cases": 342,
      "notes": "reads the expectations out of the corpus — grader reference",
      "self_check": true,
      "by_protocol": {
        "json": 1.0,
        "plain": 1.0,
        "xml": 1.0
      },
      "config": {
        "backend": "harness self-check",
        "results_file": "selftest__oracle__all.json"
      }
    },
    {
      "model": "HardcoreAI/Nova-1.2-0.8B",
      "link": "",
      "protocol": "all",
      "score": 0.7719,
      "score_ci95": [
        0.7246,
        0.8132
      ],
      "correct_call": 0.7179,
      "abstention": 0.8889,
      "spurious_call_rate": 0.0463,
      "n_cases": 342,
      "notes": "Nova-1.2-0.8B | LoRA on Nova-1.1 | greedy | thinking off | batch 8 | English-only ToolUse.1",
      "self_check": false,
      "by_protocol": {
        "json": 0.8246,
        "plain": 0.7544,
        "xml": 0.7368
      },
      "config": {
        "backend": "external",
        "results_file": "HardcoreAI__Nova-1.2-0.8B__all.json"
      }
    },
    {
      "model": "HardcoreAI/Nova-1.1-0.8B",
      "link": "",
      "protocol": "all",
      "score": 0.6491,
      "score_ci95": [
        0.5971,
        0.6978
      ],
      "correct_call": 0.5983,
      "abstention": 0.7593,
      "spurious_call_rate": 0.1667,
      "n_cases": 342,
      "notes": "Nova-1.1-0.8B | greedy | thinking off | batch 8 | English-only ToolUse.1",
      "self_check": false,
      "by_protocol": {
        "json": 0.6754,
        "plain": 0.614,
        "xml": 0.6579
      },
      "config": {
        "backend": "external",
        "results_file": "HardcoreAI__Nova-1.1-0.8B__all.json"
      }
    },
    {
      "model": "Qwen/Qwen3.5-0.8B",
      "link": "",
      "protocol": "all",
      "score": 0.5965,
      "score_ci95": [
        0.5437,
        0.6471
      ],
      "correct_call": 0.4829,
      "abstention": 0.8426,
      "spurious_call_rate": 0.0648,
      "n_cases": 342,
      "notes": "Qwen3.5-0.8B | greedy | thinking off | batch 8 | English-only ToolUse.1",
      "self_check": false,
      "by_protocol": {
        "json": 0.6754,
        "plain": 0.4298,
        "xml": 0.6842
      },
      "config": {
        "backend": "external",
        "results_file": "Qwen__Qwen3.5-0.8B__all.json"
      }
    }
  ]
}