eval_data.py

# eval_data.py
# PBH Applied Systems — Agent Arena model registry.
#
# Scores: every quant_eval figure the Arena shows comes from arena_metrics.json, which the site's page factory
# writes from the published quant_eval Public Corpus (doi:10.5281/zenodo.22009419) and from verified quant_eval run
# bundles of showcase models published as summaries (each build names its source). A model is matched by the
# SHA-256 of the GGUF file it actually serves, read from the Hugging Face repo at startup (LFS metadata, no
# download): only a file a v7.22 run evaluated shows results, so replacing a repo's file switches its results on
# with no change here. Nothing here is typed by hand. Notes on models without a v7.22 run are v7.21 observations.

import json
import os

with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "arena_metrics.json"), encoding="utf-8") as _f:
    METRICS = json.load(_f)
HARNESS = METRICS["harness_family"]                 # "v7.22"
FAMILY_ORDER = METRICS["family_order"]
ALPHA = METRICS["alpha"]

# ---------------------------------------------------------------------------
# Per-family fixture descriptions
# ---------------------------------------------------------------------------

FAMILY_DESCRIPTIONS = {
    "json_multistep": (
        "Multi-step planning with self-check and oracle verification. "
        "Hardest family — all four signals must pass: schema_ok, "
        "checks_consistent_ok, stop_semantics_ok, oracle_equiv_ok."
    ),
    "stateful_followup": (
        "Two-turn state tracking. Turn 2 only evaluated given correct Turn 1. "
        "Tests multi-turn memory under production conditions."
    ),
    "toolcall_only": (
        "Bare schema-only tool call: strict tool name + args check. "
        "No prose, no explanation — just schema-valid JSON. "
        "Where quantization most commonly degrades structured dispatch."
    ),
    "mixed_brief_json": (
        "Hybrid output: natural language answer + valid JSON block in same response. "
        "Both parts must be present and correct simultaneously."
    ),
    "toolcall": (
        "Tool call embedded in a broader response. More forgiving than toolcall_only. "
        "Tests inline tool dispatch with surrounding context."
    ),
    "json": (
        "Single-step structured JSON with constraint rules. Bucket-scored."
    ),
    "fuzz": (
        "Property-based regression across structured placement correctness. "
        "Bucket-scored. Detects inconsistencies under input variation."
    ),
    "mcq": (
        "Multiple-choice extraction with exact answer signal. "
        "Bucket-scored. A-bias is a known characteristic in some models."
    ),
}

# ---------------------------------------------------------------------------
# Model registry — Q4_K_M variants only. Deployment facts (repo, file, SHA-256, context, VRAM) are kept here;
# results are attached below from arena_metrics.json by SHA-256.
# vram_gb: from model card Key Characteristics.
# ---------------------------------------------------------------------------

MODELS = {
    "qwen2.5-3b": {
        "display_name": "Qwen2.5-3B-Instruct Q4_K_M",
        "short_name": "Qwen2.5-3B",
        "family": "Qwen2.5",
        "params": "3B",
        "context_window": 32768,
        "file_size_gb": 1.93,
        "vram_gb": 4.0,
        "avg_inference_sec": 0.390,
        "hf_repo": "pbhappliedsystems/qwen-2.5-3B-instruct-gguf-Q4-K-M",
        "hf_filename": "qwen-2.5-3B-instruct-gguf-Q4-K-M.gguf",
        "sha256": "9ab3bc9beaddaec3700d5cc754b52e1501a3fd172bc7fc3ee3eb8e1d388ee043",
        "run_id": "20260221_041137",
        "license": "Qwen Research License (non-commercial)",
        "solo_only": False,
        "thinking_mode": False,
        "known_issues": [
            "A-bias on MCQ: mcq_02 and mcq_05 both produce 'A' (wrong). "
            "Add CoT prompting for MCQ pipelines.",
            "json_multistep: 0.200 pass rate — checks_consistent_ok fails on all "
            "cases except ms_easy_01.",
        ],
        "series_notes": (
            "Smallest and fastest model in series (0.390 sec/case, 1.93 GB). "
            "Runs on 4 GB VRAM or CPU. Strong coherence relative to size. "
            "Reasoning is weakest in the evaluated series."
        ),
    },

    "qwen2.5-7b": {
        "display_name": "Qwen2.5-7B-Instruct Q4_K_M",
        "short_name": "Qwen2.5-7B",
        "family": "Qwen2.5",
        "params": "7B",
        "context_window": 32768,
        "file_size_gb": 4.68,
        "vram_gb": 6.0,
        "avg_inference_sec": 0.554,
        "hf_repo": "pbhappliedsystems/qwen-2.5-7B-instruct-gguf-Q4-K-M",
        "hf_filename": "qwen-2.5-7B-instruct-gguf-Q4-K-M.gguf",
        "sha256": "863656d217841f5d3fb180d9dca4e4bbdaa071bde25885fa0d27fe7188a2cc85",
        "run_id": "20260221_024911",
        "license": "Qwen Research License (non-commercial)",
        "solo_only": False,
        "thinking_mode": False,
        "known_issues": [
            "toolcall_only: 0/2 pass — wrong schema key names "
            "('numbers' array instead of 'args' object).",
            "EOS token contamination on toolcall final answers — "
            "strip <|im_end|> before downstream processing.",
        ],
        "series_notes": (
            "Major capability step over 3B: reasoning +0.574. "
            "checks_consistent_ok goes 0.200 → 1.000. "
            "Fastest non-3B model at 0.554 sec/case."
        ),
    },

    "qwen2.5-14b-1m": {
        "display_name": "Qwen2.5-14B-Instruct-1M Q4_K_M",
        "short_name": "Qwen2.5-14B-1M",
        "family": "Qwen2.5",
        "params": "14B",
        "context_window": 1_000_000,
        "file_size_gb": 8.99,
        "vram_gb": 12.0,
        "avg_inference_sec": 2.683,
        "hf_repo": "pbhappliedsystems/qwen-2.5-14B-instruct-1m-gguf-Q4-K-M",
        "hf_filename": "qwen-2.5-14B-instruct-1m-gguf-Q4-K-M.gguf",
        "sha256": "5ad529ff2b1b192f31c8a638fe8756a0c628904e2ded797c11f9194216976973",
        "run_id": "20260210_235131",
        "license": "Apache 2.0",
        "solo_only": False,
        "thinking_mode": False,
        "known_issues": [
            "toolcall_only: args_ok=0.000 — 'input'/{x,y} wrapper instead of 'args'/{a,b}. "
            "Specify exact key names in system prompt.",
            "EOS token contamination on toolcall final answers — "
            "strip <|im_end|> before downstream processing.",
        ],
        "series_notes": (
            "#1 reasoning and #1 instruction-following in the evaluated series. "
            "Zero quantization degradation across all behavioral families — "
            "F16 and Q4_K_M produce identical pass rates on every fixture. "
            "1M context window. For deployment: set n_ctx to actual context needed; "
            "full 1M context requires ~80 GB VRAM."
        ),
    },

    "qwen2.5-32b": {
        "display_name": "Qwen2.5-32B-Instruct Q4_K_M",
        "short_name": "Qwen2.5-32B",
        "family": "Qwen2.5",
        "params": "32B",
        "context_window": 32768,
        "file_size_gb": 19.9,
        "vram_gb": 24.0,
        "avg_inference_sec": 9.282,
        "hf_repo": "pbhappliedsystems/qwen-2.5-32B-instruct-gguf-Q4-K-M",
        "hf_filename": "qwen-2.5-32B-instruct-gguf-Q4-K-M.gguf",
        "sha256": "6f810a332a884410aa65cc1b5a128a8603f083b36465acfbbf67a08f50a4d3e3",
        "run_id": "20260221_144732",
        "license": "Apache 2.0",
        "solo_only": False,  # H200 141GB VRAM — all pairs feasible
        "thinking_mode": False,
        "known_issues": [
            "json_multistep: 0.600 pass rate — counterintuitively underperforms 7B and 14B-1M. "
            "ms_hard_01 fails with checks_consistent_ok=0 and oracle_equiv_ok=0.",
            "toolcall_only: args_ok=0.000 — uses 'params'/{a,b} instead of 'args'/{a,b}. "
            "Arg value names are correct; only outer wrapper key fails. "
            "Fixable with explicit key-name system prompt.",
            "EOS token contamination on toolcall final answers — "
            "strip <|im_end|> before downstream processing.",
        ],
        "series_notes": (
            "Largest evaluated model (19.9 GB, ~24 GB VRAM). "
            "Single-runner evaluation — no F16 baseline possible at this file size. "
            "Counterintuitively underperforms 7B and 14B-1M on json_multistep. "
            "MCQ: 5/5 perfect. stateful_followup: 1.000. mixed_brief_json: 1.000."
        ),
    },

    "ministral-14b-instruct": {
        "display_name": "Ministral-3-14B-Instruct-2512 Q4_K_M",
        "short_name": "Ministral-14B",
        "family": "Ministral",
        "params": "14B",
        "context_window": 32768,
        "file_size_gb": 8.24,
        "vram_gb": 11.0,
        "avg_inference_sec": 3.77,
        "hf_repo": "pbhappliedsystems/ministral-3-14b-instruct-2512-gguf-Q4-K-M",
        "hf_filename": "ministral-3-14b-instruct-2512-gguf-Q4-K-M.gguf",
        "sha256": "a23910514ee512aa28db8dddd390c26a73b9c318dcdec374ae02d722d9658749",   # file in the repo before the v7.22 upload
        "run_id": "20260209_170235",
        "license": "Apache 2.0",
        "solo_only": False,
        "thinking_mode": False,
        "known_issues": [
            "toolcall_only: F16=1.000 → Q4_K_M=0.000. Complete degradation on bare "
            "tool-call schema under quantization. Do not deploy in bare tool-call "
            "pipelines without schema enforcement.",
        ],
        "series_notes": (
            "Strong all-around scores. Critical finding: toolcall_only drops from "
            "1.000 (F16) to 0.000 (Q4_K_M) — the most severe quantization degradation "
            "event in the evaluated series on that family."
        ),
    },

    "ministral-14b-reasoning": {
        "display_name": "Ministral-3-14B-Reasoning-2512 Q4_K_M",
        "short_name": "Ministral-14B-R",
        "family": "Ministral",
        "params": "14B",
        "context_window": 32768,
        "file_size_gb": 8.24,
        "vram_gb": 11.0,
        "avg_inference_sec": 1.18,
        "hf_repo": "pbhappliedsystems/ministral-3-14b-reasoning-2512-gguf-Q4-K-M",
        "hf_filename": "ministral-3-14b-reasoning-2512-gguf-Q4-K-M.gguf",
        "sha256": "e7171d96748ddc948fd6d9edb3d1c6e3f9ba6b855ff964aee98519788da330c2",
        "run_id": "20260209_233252",
        "license": "Apache 2.0",
        "solo_only": False,
        "thinking_mode": False,
        "known_issues": [
            "Q4_K_M compresses chain-of-thought from F16's 65.67 sec/case to 1.18 sec/case "
            "(55.7x faster). Whether this is a feature or regression depends on use case.",
            "mcq_02: F16 fails due to markdown fence wrapping; Q4_K_M suppresses fencing "
            "but selects wrong answer.",
        ],
        "series_notes": (
            "Fastest non-3B model in the series at 1.18 sec/case Q4_K_M. "
            "Quantization dramatically compresses the reasoning chain vs F16 (65.67 sec). "
            "Use when speed matters and abbreviated reasoning is acceptable."
        ),
    },

    "phi4-reasoning-plus": {
        "display_name": "Phi-4-reasoning-plus Q4_K_M",
        "short_name": "Phi-4-R+",
        "family": "Phi-4",
        "params": "14B",
        "context_window": 16384,
        "file_size_gb": 9.05,
        "vram_gb": 12.0,
        "avg_inference_sec": 25.84,
        "hf_repo": "pbhappliedsystems/phi-4-reasoning-plus-gguf-Q4-K-M",
        "hf_filename": "phi-4-reasoning-plus-gguf-Q4-K-M.gguf",
        "sha256": "2fe74424b03433d11ccf3f2ce8da404810fa7eb9a269135b1f14bf0d88566e4d",
        "run_id": "20260222_170914",
        "license": "MIT",
        "solo_only": False,
        "thinking_mode": False,
        "known_issues": [
            "Systematic EOS token contamination: <|im_end|> appears as literal text. "
            "Strip before ALL downstream processing.",
            "json_multistep: 4/5 cases produce only <|im_end|> as entire response.",
            "mcq: all 5 cases fail with EOS token output — bucket_score=0.000.",
            "toolcall_only: 0/2 pass — prose output instead of JSON schema.",
        ],
        "series_notes": (
            "Lowest reasoning (0.3648) and coherence (0.4921) in the evaluated series. "
            "Systematic EOS token contamination drives failures across planning, MCQ, "
            "and tool dispatch families. Demonstrates what rigorous pre-deployment "
            "evaluation surfaces that casual testing does not."
        ),
    },

    "mistral-nemo": {
        "display_name": "Mistral-Nemo-Instruct-2407 Q4_K_M",
        "short_name": "Mistral-Nemo",
        "family": "Mistral",
        "params": "12B",
        "context_window": 128000,
        "file_size_gb": 7.48,
        "vram_gb": 10.0,
        "avg_inference_sec": 1.42,
        "hf_repo": "pbhappliedsystems/mistral-nemo-instruct-2407-gguf-Q4-K-M",
        "hf_filename": "mistral-nemo-instruct-2407-gguf-Q4-K-M.gguf",
        "sha256": "d3b7c7950abee2677290c1a3412940a0e1e3bc98449b42f34bc1d7ab6d4917b9",
        "run_id": "20260211_022944",
        "license": "Apache 2.0",
        "solo_only": False,
        "thinking_mode": False,
        "known_issues": [
            "MCQ A-bias: mcq_02 and mcq_05 both produce 'A' (wrong).",
            "json_multistep: ms_hard_01 fails all four gating signals simultaneously.",
            "toolcall_only: args_ok=0.000 — add schema enforcement.",
            "toolcall tool_02: final answer wrong despite correct tool dispatch — "
            "validate post-execution.",
        ],
        "series_notes": (
            "128K context window (Tekken tokenizer) — second largest in series "
            "after Qwen2.5-14B-1M's 1M. Multilingual: 9 languages. "
            "Strong instruction-following at 0.9329."
        ),
    },

    "qwen3.6-27b": {
        "display_name": "Qwen3.6-27B Q4_K_M",
        "short_name": "Qwen3.6-27B",
        "family": "Qwen3",
        "params": "27B",
        "context_window": 32768,
        "file_size_gb": 16.5,
        "vram_gb": 22.0,
        "avg_inference_sec": 1.938,
        "hf_repo": "pbhappliedsystems/qwen3.6-27B-gguf-Q4-K-M",
        "hf_filename": "qwen3.6-27B-gguf-Q4-K-M.gguf",
        "sha256": "c863357b1b532a02c47ca363ab666dd623470a152a291dac6619ed7ce751d8c8",
        "run_id": "20260426_163540",
        "license": "Apache 2.0",
        "solo_only": False,
        "thinking_mode": True,
        "known_issues": [
            "Hybrid thinking mode: <think> blocks generated on medium/hard tasks. "
            "json_multistep medium and hard cases fail with schema_ok=0 because the "
            "extraction layer receives the think block before the JSON. "
            "Strip <think>...</think> blocks before extraction, or use /no_think "
            "in user message to suppress thinking mode for structured output tasks.",
            "toolcall_only: args_ok=0.000 — uses 'arguments' instead of 'args'. "
            "tool_name key IS correct without enforcement (only model in series to do so). "
            "Specify 'args' explicitly in system prompt to resolve.",
            "EOS token contamination on toolcall final answers — "
            "strip <|im_end|> before downstream processing.",
        ],
        "series_notes": (
            "First Qwen3-series model in the evaluated series. "
            "Hybrid adaptive thinking mode is the defining behavioral characteristic. "
            "json_multistep 0.400 is a pipeline compatibility finding, not a capability "
            "regression — easy cases pass cleanly; medium/hard require think-block stripping. "
            "Only model in the series to produce correct 'tool_name' key without enforcement. "
            "stateful_followup: 1.000. mixed_brief_json: 1.000. MCQ: 5/5 perfect. "
            "fuzz: 20/20 pass."
        ),
    },
}

# ---------------------------------------------------------------------------
# VRAM budget — ZeroGPU Nvidia H200 (141 GB HBM3e)
# All models in the evaluated series can be paired without restriction.
# ---------------------------------------------------------------------------

ZEROGPU_VRAM_GB = 141.0
VRAM_SAFETY_CEILING_GB = 130.0


def pair_is_feasible(key_a: str, key_b: str) -> tuple[bool, str]:
    """
    Returns (feasible: bool, reason: str).
    Checks for duplicate selection and combined VRAM against H200 ceiling.
    """
    if key_a == key_b:
        return False, "Select two different models for comparison."
    combined = MODELS[key_a]["vram_gb"] + MODELS[key_b]["vram_gb"]
    if combined > VRAM_SAFETY_CEILING_GB:
        return False, (
            f"Combined VRAM estimate ({combined:.1f} GB) exceeds safe ceiling "
            f"({VRAM_SAFETY_CEILING_GB} GB)."
        )
    return True, f"Estimated combined VRAM: {combined:.1f} GB / {ZEROGPU_VRAM_GB} GB"

# ---------------------------------------------------------------------------
# Attach published results by SHA-256 (Plan S-19). Option (a), owner-approved scope:
#  - a model whose served file a v7.22 run evaluated gets that run's results, and its v7.21 notes and v7.21
#    speed are dropped, so no model shows two harness versions side by side;
#  - every other model keeps its v7.21 notes, labelled, and shows no scores.
# ---------------------------------------------------------------------------

def _served_sha256():
    """SHA-256 of the file each repo serves now, from its LFS metadata. A model whose lookup fails keeps the
    SHA-256 declared above, so a Hub outage can only hide results, never attach them to the wrong file."""
    try:
        from huggingface_hub import HfApi
        api = HfApi()
        out = {}
        for key, m in MODELS.items():
            try:
                info = api.get_paths_info(m["hf_repo"], [m["hf_filename"]])
                lfs = getattr(info[0], "lfs", None) if info else None
                sha = getattr(lfs, "sha256", None) or (lfs.get("sha256") if isinstance(lfs, dict) else None)
                if sha:
                    out[key] = sha
            except Exception:
                pass
        return out
    except Exception:
        return {}


for _key, _sha in _served_sha256().items():
    MODELS[_key]["sha256"] = _sha

V721 = "Observed in quant_eval v7.21: "
for _key, _m in MODELS.items():
    _r = METRICS["builds"].get(_m["sha256"])
    _m["results"] = _r
    if _r:
        _m["known_issues"] = []
        _m["series_notes"] = ""
        _m["file_size_gb"] = round(_r["size_bytes"] / 1e9, 2)
        _m["avg_inference_sec"] = _r["sec_per_case"]
        _m["speed_note"] = f"{HARNESS}, {_r['accelerator']}"
    else:
        _m["known_issues"] = [V721 + i for i in _m["known_issues"]]
        _m["series_notes"] = (V721 + _m["series_notes"]) if _m["series_notes"] else ""
        _m["speed_note"] = "v7.21, RTX 4090"