# eval_data.py # PBH Applied Systems — Agent Arena model registry. # # Scores: every quant_eval figure the Arena shows comes from arena_metrics.json, which the site's page factory # writes from the published quant_eval Public Corpus (doi:10.5281/zenodo.22009419) and from verified quant_eval run # bundles of showcase models published as summaries (each build names its source). A model is matched by the # SHA-256 of the GGUF file it actually serves, read from the Hugging Face repo at startup (LFS metadata, no # download): only a file a v7.22 run evaluated shows results, so replacing a repo's file switches its results on # with no change here. Nothing here is typed by hand. Notes on models without a v7.22 run are v7.21 observations. import json import os with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "arena_metrics.json"), encoding="utf-8") as _f: METRICS = json.load(_f) HARNESS = METRICS["harness_family"] # "v7.22" FAMILY_ORDER = METRICS["family_order"] ALPHA = METRICS["alpha"] # --------------------------------------------------------------------------- # Per-family fixture descriptions # --------------------------------------------------------------------------- FAMILY_DESCRIPTIONS = { "json_multistep": ( "Multi-step planning with self-check and oracle verification. " "Hardest family — all four signals must pass: schema_ok, " "checks_consistent_ok, stop_semantics_ok, oracle_equiv_ok." ), "stateful_followup": ( "Two-turn state tracking. Turn 2 only evaluated given correct Turn 1. " "Tests multi-turn memory under production conditions." ), "toolcall_only": ( "Bare schema-only tool call: strict tool name + args check. " "No prose, no explanation — just schema-valid JSON. " "Where quantization most commonly degrades structured dispatch." ), "mixed_brief_json": ( "Hybrid output: natural language answer + valid JSON block in same response. " "Both parts must be present and correct simultaneously." ), "toolcall": ( "Tool call embedded in a broader response. More forgiving than toolcall_only. " "Tests inline tool dispatch with surrounding context." ), "json": ( "Single-step structured JSON with constraint rules. Bucket-scored." ), "fuzz": ( "Property-based regression across structured placement correctness. " "Bucket-scored. Detects inconsistencies under input variation." ), "mcq": ( "Multiple-choice extraction with exact answer signal. " "Bucket-scored. A-bias is a known characteristic in some models." ), } # --------------------------------------------------------------------------- # Model registry — Q4_K_M variants only. Deployment facts (repo, file, SHA-256, context, VRAM) are kept here; # results are attached below from arena_metrics.json by SHA-256. # vram_gb: from model card Key Characteristics. # --------------------------------------------------------------------------- MODELS = { "qwen2.5-3b": { "display_name": "Qwen2.5-3B-Instruct Q4_K_M", "short_name": "Qwen2.5-3B", "family": "Qwen2.5", "params": "3B", "context_window": 32768, "file_size_gb": 1.93, "vram_gb": 4.0, "avg_inference_sec": 0.390, "hf_repo": "pbhappliedsystems/qwen-2.5-3B-instruct-gguf-Q4-K-M", "hf_filename": "qwen-2.5-3B-instruct-gguf-Q4-K-M.gguf", "sha256": "9ab3bc9beaddaec3700d5cc754b52e1501a3fd172bc7fc3ee3eb8e1d388ee043", "run_id": "20260221_041137", "license": "Qwen Research License (non-commercial)", "solo_only": False, "thinking_mode": False, "known_issues": [ "A-bias on MCQ: mcq_02 and mcq_05 both produce 'A' (wrong). " "Add CoT prompting for MCQ pipelines.", "json_multistep: 0.200 pass rate — checks_consistent_ok fails on all " "cases except ms_easy_01.", ], "series_notes": ( "Smallest and fastest model in series (0.390 sec/case, 1.93 GB). " "Runs on 4 GB VRAM or CPU. Strong coherence relative to size. " "Reasoning is weakest in the evaluated series." ), }, "qwen2.5-7b": { "display_name": "Qwen2.5-7B-Instruct Q4_K_M", "short_name": "Qwen2.5-7B", "family": "Qwen2.5", "params": "7B", "context_window": 32768, "file_size_gb": 4.68, "vram_gb": 6.0, "avg_inference_sec": 0.554, "hf_repo": "pbhappliedsystems/qwen-2.5-7B-instruct-gguf-Q4-K-M", "hf_filename": "qwen-2.5-7B-instruct-gguf-Q4-K-M.gguf", "sha256": "863656d217841f5d3fb180d9dca4e4bbdaa071bde25885fa0d27fe7188a2cc85", "run_id": "20260221_024911", "license": "Qwen Research License (non-commercial)", "solo_only": False, "thinking_mode": False, "known_issues": [ "toolcall_only: 0/2 pass — wrong schema key names " "('numbers' array instead of 'args' object).", "EOS token contamination on toolcall final answers — " "strip <|im_end|> before downstream processing.", ], "series_notes": ( "Major capability step over 3B: reasoning +0.574. " "checks_consistent_ok goes 0.200 → 1.000. " "Fastest non-3B model at 0.554 sec/case." ), }, "qwen2.5-14b-1m": { "display_name": "Qwen2.5-14B-Instruct-1M Q4_K_M", "short_name": "Qwen2.5-14B-1M", "family": "Qwen2.5", "params": "14B", "context_window": 1_000_000, "file_size_gb": 8.99, "vram_gb": 12.0, "avg_inference_sec": 2.683, "hf_repo": "pbhappliedsystems/qwen-2.5-14B-instruct-1m-gguf-Q4-K-M", "hf_filename": "qwen-2.5-14B-instruct-1m-gguf-Q4-K-M.gguf", "sha256": "5ad529ff2b1b192f31c8a638fe8756a0c628904e2ded797c11f9194216976973", "run_id": "20260210_235131", "license": "Apache 2.0", "solo_only": False, "thinking_mode": False, "known_issues": [ "toolcall_only: args_ok=0.000 — 'input'/{x,y} wrapper instead of 'args'/{a,b}. " "Specify exact key names in system prompt.", "EOS token contamination on toolcall final answers — " "strip <|im_end|> before downstream processing.", ], "series_notes": ( "#1 reasoning and #1 instruction-following in the evaluated series. " "Zero quantization degradation across all behavioral families — " "F16 and Q4_K_M produce identical pass rates on every fixture. " "1M context window. For deployment: set n_ctx to actual context needed; " "full 1M context requires ~80 GB VRAM." ), }, "qwen2.5-32b": { "display_name": "Qwen2.5-32B-Instruct Q4_K_M", "short_name": "Qwen2.5-32B", "family": "Qwen2.5", "params": "32B", "context_window": 32768, "file_size_gb": 19.9, "vram_gb": 24.0, "avg_inference_sec": 9.282, "hf_repo": "pbhappliedsystems/qwen-2.5-32B-instruct-gguf-Q4-K-M", "hf_filename": "qwen-2.5-32B-instruct-gguf-Q4-K-M.gguf", "sha256": "6f810a332a884410aa65cc1b5a128a8603f083b36465acfbbf67a08f50a4d3e3", "run_id": "20260221_144732", "license": "Apache 2.0", "solo_only": False, # H200 141GB VRAM — all pairs feasible "thinking_mode": False, "known_issues": [ "json_multistep: 0.600 pass rate — counterintuitively underperforms 7B and 14B-1M. " "ms_hard_01 fails with checks_consistent_ok=0 and oracle_equiv_ok=0.", "toolcall_only: args_ok=0.000 — uses 'params'/{a,b} instead of 'args'/{a,b}. " "Arg value names are correct; only outer wrapper key fails. " "Fixable with explicit key-name system prompt.", "EOS token contamination on toolcall final answers — " "strip <|im_end|> before downstream processing.", ], "series_notes": ( "Largest evaluated model (19.9 GB, ~24 GB VRAM). " "Single-runner evaluation — no F16 baseline possible at this file size. " "Counterintuitively underperforms 7B and 14B-1M on json_multistep. " "MCQ: 5/5 perfect. stateful_followup: 1.000. mixed_brief_json: 1.000." ), }, "ministral-14b-instruct": { "display_name": "Ministral-3-14B-Instruct-2512 Q4_K_M", "short_name": "Ministral-14B", "family": "Ministral", "params": "14B", "context_window": 32768, "file_size_gb": 8.24, "vram_gb": 11.0, "avg_inference_sec": 3.77, "hf_repo": "pbhappliedsystems/ministral-3-14b-instruct-2512-gguf-Q4-K-M", "hf_filename": "ministral-3-14b-instruct-2512-gguf-Q4-K-M.gguf", "sha256": "a23910514ee512aa28db8dddd390c26a73b9c318dcdec374ae02d722d9658749", # file in the repo before the v7.22 upload "run_id": "20260209_170235", "license": "Apache 2.0", "solo_only": False, "thinking_mode": False, "known_issues": [ "toolcall_only: F16=1.000 → Q4_K_M=0.000. Complete degradation on bare " "tool-call schema under quantization. Do not deploy in bare tool-call " "pipelines without schema enforcement.", ], "series_notes": ( "Strong all-around scores. Critical finding: toolcall_only drops from " "1.000 (F16) to 0.000 (Q4_K_M) — the most severe quantization degradation " "event in the evaluated series on that family." ), }, "ministral-14b-reasoning": { "display_name": "Ministral-3-14B-Reasoning-2512 Q4_K_M", "short_name": "Ministral-14B-R", "family": "Ministral", "params": "14B", "context_window": 32768, "file_size_gb": 8.24, "vram_gb": 11.0, "avg_inference_sec": 1.18, "hf_repo": "pbhappliedsystems/ministral-3-14b-reasoning-2512-gguf-Q4-K-M", "hf_filename": "ministral-3-14b-reasoning-2512-gguf-Q4-K-M.gguf", "sha256": "e7171d96748ddc948fd6d9edb3d1c6e3f9ba6b855ff964aee98519788da330c2", "run_id": "20260209_233252", "license": "Apache 2.0", "solo_only": False, "thinking_mode": False, "known_issues": [ "Q4_K_M compresses chain-of-thought from F16's 65.67 sec/case to 1.18 sec/case " "(55.7x faster). Whether this is a feature or regression depends on use case.", "mcq_02: F16 fails due to markdown fence wrapping; Q4_K_M suppresses fencing " "but selects wrong answer.", ], "series_notes": ( "Fastest non-3B model in the series at 1.18 sec/case Q4_K_M. " "Quantization dramatically compresses the reasoning chain vs F16 (65.67 sec). " "Use when speed matters and abbreviated reasoning is acceptable." ), }, "phi4-reasoning-plus": { "display_name": "Phi-4-reasoning-plus Q4_K_M", "short_name": "Phi-4-R+", "family": "Phi-4", "params": "14B", "context_window": 16384, "file_size_gb": 9.05, "vram_gb": 12.0, "avg_inference_sec": 25.84, "hf_repo": "pbhappliedsystems/phi-4-reasoning-plus-gguf-Q4-K-M", "hf_filename": "phi-4-reasoning-plus-gguf-Q4-K-M.gguf", "sha256": "2fe74424b03433d11ccf3f2ce8da404810fa7eb9a269135b1f14bf0d88566e4d", "run_id": "20260222_170914", "license": "MIT", "solo_only": False, "thinking_mode": False, "known_issues": [ "Systematic EOS token contamination: <|im_end|> appears as literal text. " "Strip before ALL downstream processing.", "json_multistep: 4/5 cases produce only <|im_end|> as entire response.", "mcq: all 5 cases fail with EOS token output — bucket_score=0.000.", "toolcall_only: 0/2 pass — prose output instead of JSON schema.", ], "series_notes": ( "Lowest reasoning (0.3648) and coherence (0.4921) in the evaluated series. " "Systematic EOS token contamination drives failures across planning, MCQ, " "and tool dispatch families. Demonstrates what rigorous pre-deployment " "evaluation surfaces that casual testing does not." ), }, "mistral-nemo": { "display_name": "Mistral-Nemo-Instruct-2407 Q4_K_M", "short_name": "Mistral-Nemo", "family": "Mistral", "params": "12B", "context_window": 128000, "file_size_gb": 7.48, "vram_gb": 10.0, "avg_inference_sec": 1.42, "hf_repo": "pbhappliedsystems/mistral-nemo-instruct-2407-gguf-Q4-K-M", "hf_filename": "mistral-nemo-instruct-2407-gguf-Q4-K-M.gguf", "sha256": "d3b7c7950abee2677290c1a3412940a0e1e3bc98449b42f34bc1d7ab6d4917b9", "run_id": "20260211_022944", "license": "Apache 2.0", "solo_only": False, "thinking_mode": False, "known_issues": [ "MCQ A-bias: mcq_02 and mcq_05 both produce 'A' (wrong).", "json_multistep: ms_hard_01 fails all four gating signals simultaneously.", "toolcall_only: args_ok=0.000 — add schema enforcement.", "toolcall tool_02: final answer wrong despite correct tool dispatch — " "validate post-execution.", ], "series_notes": ( "128K context window (Tekken tokenizer) — second largest in series " "after Qwen2.5-14B-1M's 1M. Multilingual: 9 languages. " "Strong instruction-following at 0.9329." ), }, "qwen3.6-27b": { "display_name": "Qwen3.6-27B Q4_K_M", "short_name": "Qwen3.6-27B", "family": "Qwen3", "params": "27B", "context_window": 32768, "file_size_gb": 16.5, "vram_gb": 22.0, "avg_inference_sec": 1.938, "hf_repo": "pbhappliedsystems/qwen3.6-27B-gguf-Q4-K-M", "hf_filename": "qwen3.6-27B-gguf-Q4-K-M.gguf", "sha256": "c863357b1b532a02c47ca363ab666dd623470a152a291dac6619ed7ce751d8c8", "run_id": "20260426_163540", "license": "Apache 2.0", "solo_only": False, "thinking_mode": True, "known_issues": [ "Hybrid thinking mode: blocks generated on medium/hard tasks. " "json_multistep medium and hard cases fail with schema_ok=0 because the " "extraction layer receives the think block before the JSON. " "Strip ... blocks before extraction, or use /no_think " "in user message to suppress thinking mode for structured output tasks.", "toolcall_only: args_ok=0.000 — uses 'arguments' instead of 'args'. " "tool_name key IS correct without enforcement (only model in series to do so). " "Specify 'args' explicitly in system prompt to resolve.", "EOS token contamination on toolcall final answers — " "strip <|im_end|> before downstream processing.", ], "series_notes": ( "First Qwen3-series model in the evaluated series. " "Hybrid adaptive thinking mode is the defining behavioral characteristic. " "json_multistep 0.400 is a pipeline compatibility finding, not a capability " "regression — easy cases pass cleanly; medium/hard require think-block stripping. " "Only model in the series to produce correct 'tool_name' key without enforcement. " "stateful_followup: 1.000. mixed_brief_json: 1.000. MCQ: 5/5 perfect. " "fuzz: 20/20 pass." ), }, } # --------------------------------------------------------------------------- # VRAM budget — ZeroGPU Nvidia H200 (141 GB HBM3e) # All models in the evaluated series can be paired without restriction. # --------------------------------------------------------------------------- ZEROGPU_VRAM_GB = 141.0 VRAM_SAFETY_CEILING_GB = 130.0 def pair_is_feasible(key_a: str, key_b: str) -> tuple[bool, str]: """ Returns (feasible: bool, reason: str). Checks for duplicate selection and combined VRAM against H200 ceiling. """ if key_a == key_b: return False, "Select two different models for comparison." combined = MODELS[key_a]["vram_gb"] + MODELS[key_b]["vram_gb"] if combined > VRAM_SAFETY_CEILING_GB: return False, ( f"Combined VRAM estimate ({combined:.1f} GB) exceeds safe ceiling " f"({VRAM_SAFETY_CEILING_GB} GB)." ) return True, f"Estimated combined VRAM: {combined:.1f} GB / {ZEROGPU_VRAM_GB} GB" # --------------------------------------------------------------------------- # Attach published results by SHA-256 (Plan S-19). Option (a), owner-approved scope: # - a model whose served file a v7.22 run evaluated gets that run's results, and its v7.21 notes and v7.21 # speed are dropped, so no model shows two harness versions side by side; # - every other model keeps its v7.21 notes, labelled, and shows no scores. # --------------------------------------------------------------------------- def _served_sha256(): """SHA-256 of the file each repo serves now, from its LFS metadata. A model whose lookup fails keeps the SHA-256 declared above, so a Hub outage can only hide results, never attach them to the wrong file.""" try: from huggingface_hub import HfApi api = HfApi() out = {} for key, m in MODELS.items(): try: info = api.get_paths_info(m["hf_repo"], [m["hf_filename"]]) lfs = getattr(info[0], "lfs", None) if info else None sha = getattr(lfs, "sha256", None) or (lfs.get("sha256") if isinstance(lfs, dict) else None) if sha: out[key] = sha except Exception: pass return out except Exception: return {} for _key, _sha in _served_sha256().items(): MODELS[_key]["sha256"] = _sha V721 = "Observed in quant_eval v7.21: " for _key, _m in MODELS.items(): _r = METRICS["builds"].get(_m["sha256"]) _m["results"] = _r if _r: _m["known_issues"] = [] _m["series_notes"] = "" _m["file_size_gb"] = round(_r["size_bytes"] / 1e9, 2) _m["avg_inference_sec"] = _r["sec_per_case"] _m["speed_note"] = f"{HARNESS}, {_r['accelerator']}" else: _m["known_issues"] = [V721 + i for i in _m["known_issues"]] _m["series_notes"] = (V721 + _m["series_notes"]) if _m["series_notes"] else "" _m["speed_note"] = "v7.21, RTX 4090"