Spaces:
Running on Zero
Running on Zero
Arena: v7.22 results incl. Ministral-3-14B-Instruct-2512; match served files by repo SHA-256
1f75d12 verified | # eval_data.py | |
| # PBH Applied Systems — Agent Arena model registry. | |
| # | |
| # Scores: every quant_eval figure the Arena shows comes from arena_metrics.json, which the site's page factory | |
| # writes from the published quant_eval Public Corpus (doi:10.5281/zenodo.22009419) and from verified quant_eval run | |
| # bundles of showcase models published as summaries (each build names its source). A model is matched by the | |
| # SHA-256 of the GGUF file it actually serves, read from the Hugging Face repo at startup (LFS metadata, no | |
| # download): only a file a v7.22 run evaluated shows results, so replacing a repo's file switches its results on | |
| # with no change here. Nothing here is typed by hand. Notes on models without a v7.22 run are v7.21 observations. | |
| import json | |
| import os | |
| with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "arena_metrics.json"), encoding="utf-8") as _f: | |
| METRICS = json.load(_f) | |
| HARNESS = METRICS["harness_family"] # "v7.22" | |
| FAMILY_ORDER = METRICS["family_order"] | |
| ALPHA = METRICS["alpha"] | |
| # --------------------------------------------------------------------------- | |
| # Per-family fixture descriptions | |
| # --------------------------------------------------------------------------- | |
| FAMILY_DESCRIPTIONS = { | |
| "json_multistep": ( | |
| "Multi-step planning with self-check and oracle verification. " | |
| "Hardest family — all four signals must pass: schema_ok, " | |
| "checks_consistent_ok, stop_semantics_ok, oracle_equiv_ok." | |
| ), | |
| "stateful_followup": ( | |
| "Two-turn state tracking. Turn 2 only evaluated given correct Turn 1. " | |
| "Tests multi-turn memory under production conditions." | |
| ), | |
| "toolcall_only": ( | |
| "Bare schema-only tool call: strict tool name + args check. " | |
| "No prose, no explanation — just schema-valid JSON. " | |
| "Where quantization most commonly degrades structured dispatch." | |
| ), | |
| "mixed_brief_json": ( | |
| "Hybrid output: natural language answer + valid JSON block in same response. " | |
| "Both parts must be present and correct simultaneously." | |
| ), | |
| "toolcall": ( | |
| "Tool call embedded in a broader response. More forgiving than toolcall_only. " | |
| "Tests inline tool dispatch with surrounding context." | |
| ), | |
| "json": ( | |
| "Single-step structured JSON with constraint rules. Bucket-scored." | |
| ), | |
| "fuzz": ( | |
| "Property-based regression across structured placement correctness. " | |
| "Bucket-scored. Detects inconsistencies under input variation." | |
| ), | |
| "mcq": ( | |
| "Multiple-choice extraction with exact answer signal. " | |
| "Bucket-scored. A-bias is a known characteristic in some models." | |
| ), | |
| } | |
| # --------------------------------------------------------------------------- | |
| # Model registry — Q4_K_M variants only. Deployment facts (repo, file, SHA-256, context, VRAM) are kept here; | |
| # results are attached below from arena_metrics.json by SHA-256. | |
| # vram_gb: from model card Key Characteristics. | |
| # --------------------------------------------------------------------------- | |
| MODELS = { | |
| "qwen2.5-3b": { | |
| "display_name": "Qwen2.5-3B-Instruct Q4_K_M", | |
| "short_name": "Qwen2.5-3B", | |
| "family": "Qwen2.5", | |
| "params": "3B", | |
| "context_window": 32768, | |
| "file_size_gb": 1.93, | |
| "vram_gb": 4.0, | |
| "avg_inference_sec": 0.390, | |
| "hf_repo": "pbhappliedsystems/qwen-2.5-3B-instruct-gguf-Q4-K-M", | |
| "hf_filename": "qwen-2.5-3B-instruct-gguf-Q4-K-M.gguf", | |
| "sha256": "9ab3bc9beaddaec3700d5cc754b52e1501a3fd172bc7fc3ee3eb8e1d388ee043", | |
| "run_id": "20260221_041137", | |
| "license": "Qwen Research License (non-commercial)", | |
| "solo_only": False, | |
| "thinking_mode": False, | |
| "known_issues": [ | |
| "A-bias on MCQ: mcq_02 and mcq_05 both produce 'A' (wrong). " | |
| "Add CoT prompting for MCQ pipelines.", | |
| "json_multistep: 0.200 pass rate — checks_consistent_ok fails on all " | |
| "cases except ms_easy_01.", | |
| ], | |
| "series_notes": ( | |
| "Smallest and fastest model in series (0.390 sec/case, 1.93 GB). " | |
| "Runs on 4 GB VRAM or CPU. Strong coherence relative to size. " | |
| "Reasoning is weakest in the evaluated series." | |
| ), | |
| }, | |
| "qwen2.5-7b": { | |
| "display_name": "Qwen2.5-7B-Instruct Q4_K_M", | |
| "short_name": "Qwen2.5-7B", | |
| "family": "Qwen2.5", | |
| "params": "7B", | |
| "context_window": 32768, | |
| "file_size_gb": 4.68, | |
| "vram_gb": 6.0, | |
| "avg_inference_sec": 0.554, | |
| "hf_repo": "pbhappliedsystems/qwen-2.5-7B-instruct-gguf-Q4-K-M", | |
| "hf_filename": "qwen-2.5-7B-instruct-gguf-Q4-K-M.gguf", | |
| "sha256": "863656d217841f5d3fb180d9dca4e4bbdaa071bde25885fa0d27fe7188a2cc85", | |
| "run_id": "20260221_024911", | |
| "license": "Qwen Research License (non-commercial)", | |
| "solo_only": False, | |
| "thinking_mode": False, | |
| "known_issues": [ | |
| "toolcall_only: 0/2 pass — wrong schema key names " | |
| "('numbers' array instead of 'args' object).", | |
| "EOS token contamination on toolcall final answers — " | |
| "strip <|im_end|> before downstream processing.", | |
| ], | |
| "series_notes": ( | |
| "Major capability step over 3B: reasoning +0.574. " | |
| "checks_consistent_ok goes 0.200 → 1.000. " | |
| "Fastest non-3B model at 0.554 sec/case." | |
| ), | |
| }, | |
| "qwen2.5-14b-1m": { | |
| "display_name": "Qwen2.5-14B-Instruct-1M Q4_K_M", | |
| "short_name": "Qwen2.5-14B-1M", | |
| "family": "Qwen2.5", | |
| "params": "14B", | |
| "context_window": 1_000_000, | |
| "file_size_gb": 8.99, | |
| "vram_gb": 12.0, | |
| "avg_inference_sec": 2.683, | |
| "hf_repo": "pbhappliedsystems/qwen-2.5-14B-instruct-1m-gguf-Q4-K-M", | |
| "hf_filename": "qwen-2.5-14B-instruct-1m-gguf-Q4-K-M.gguf", | |
| "sha256": "5ad529ff2b1b192f31c8a638fe8756a0c628904e2ded797c11f9194216976973", | |
| "run_id": "20260210_235131", | |
| "license": "Apache 2.0", | |
| "solo_only": False, | |
| "thinking_mode": False, | |
| "known_issues": [ | |
| "toolcall_only: args_ok=0.000 — 'input'/{x,y} wrapper instead of 'args'/{a,b}. " | |
| "Specify exact key names in system prompt.", | |
| "EOS token contamination on toolcall final answers — " | |
| "strip <|im_end|> before downstream processing.", | |
| ], | |
| "series_notes": ( | |
| "#1 reasoning and #1 instruction-following in the evaluated series. " | |
| "Zero quantization degradation across all behavioral families — " | |
| "F16 and Q4_K_M produce identical pass rates on every fixture. " | |
| "1M context window. For deployment: set n_ctx to actual context needed; " | |
| "full 1M context requires ~80 GB VRAM." | |
| ), | |
| }, | |
| "qwen2.5-32b": { | |
| "display_name": "Qwen2.5-32B-Instruct Q4_K_M", | |
| "short_name": "Qwen2.5-32B", | |
| "family": "Qwen2.5", | |
| "params": "32B", | |
| "context_window": 32768, | |
| "file_size_gb": 19.9, | |
| "vram_gb": 24.0, | |
| "avg_inference_sec": 9.282, | |
| "hf_repo": "pbhappliedsystems/qwen-2.5-32B-instruct-gguf-Q4-K-M", | |
| "hf_filename": "qwen-2.5-32B-instruct-gguf-Q4-K-M.gguf", | |
| "sha256": "6f810a332a884410aa65cc1b5a128a8603f083b36465acfbbf67a08f50a4d3e3", | |
| "run_id": "20260221_144732", | |
| "license": "Apache 2.0", | |
| "solo_only": False, # H200 141GB VRAM — all pairs feasible | |
| "thinking_mode": False, | |
| "known_issues": [ | |
| "json_multistep: 0.600 pass rate — counterintuitively underperforms 7B and 14B-1M. " | |
| "ms_hard_01 fails with checks_consistent_ok=0 and oracle_equiv_ok=0.", | |
| "toolcall_only: args_ok=0.000 — uses 'params'/{a,b} instead of 'args'/{a,b}. " | |
| "Arg value names are correct; only outer wrapper key fails. " | |
| "Fixable with explicit key-name system prompt.", | |
| "EOS token contamination on toolcall final answers — " | |
| "strip <|im_end|> before downstream processing.", | |
| ], | |
| "series_notes": ( | |
| "Largest evaluated model (19.9 GB, ~24 GB VRAM). " | |
| "Single-runner evaluation — no F16 baseline possible at this file size. " | |
| "Counterintuitively underperforms 7B and 14B-1M on json_multistep. " | |
| "MCQ: 5/5 perfect. stateful_followup: 1.000. mixed_brief_json: 1.000." | |
| ), | |
| }, | |
| "ministral-14b-instruct": { | |
| "display_name": "Ministral-3-14B-Instruct-2512 Q4_K_M", | |
| "short_name": "Ministral-14B", | |
| "family": "Ministral", | |
| "params": "14B", | |
| "context_window": 32768, | |
| "file_size_gb": 8.24, | |
| "vram_gb": 11.0, | |
| "avg_inference_sec": 3.77, | |
| "hf_repo": "pbhappliedsystems/ministral-3-14b-instruct-2512-gguf-Q4-K-M", | |
| "hf_filename": "ministral-3-14b-instruct-2512-gguf-Q4-K-M.gguf", | |
| "sha256": "a23910514ee512aa28db8dddd390c26a73b9c318dcdec374ae02d722d9658749", # file in the repo before the v7.22 upload | |
| "run_id": "20260209_170235", | |
| "license": "Apache 2.0", | |
| "solo_only": False, | |
| "thinking_mode": False, | |
| "known_issues": [ | |
| "toolcall_only: F16=1.000 → Q4_K_M=0.000. Complete degradation on bare " | |
| "tool-call schema under quantization. Do not deploy in bare tool-call " | |
| "pipelines without schema enforcement.", | |
| ], | |
| "series_notes": ( | |
| "Strong all-around scores. Critical finding: toolcall_only drops from " | |
| "1.000 (F16) to 0.000 (Q4_K_M) — the most severe quantization degradation " | |
| "event in the evaluated series on that family." | |
| ), | |
| }, | |
| "ministral-14b-reasoning": { | |
| "display_name": "Ministral-3-14B-Reasoning-2512 Q4_K_M", | |
| "short_name": "Ministral-14B-R", | |
| "family": "Ministral", | |
| "params": "14B", | |
| "context_window": 32768, | |
| "file_size_gb": 8.24, | |
| "vram_gb": 11.0, | |
| "avg_inference_sec": 1.18, | |
| "hf_repo": "pbhappliedsystems/ministral-3-14b-reasoning-2512-gguf-Q4-K-M", | |
| "hf_filename": "ministral-3-14b-reasoning-2512-gguf-Q4-K-M.gguf", | |
| "sha256": "e7171d96748ddc948fd6d9edb3d1c6e3f9ba6b855ff964aee98519788da330c2", | |
| "run_id": "20260209_233252", | |
| "license": "Apache 2.0", | |
| "solo_only": False, | |
| "thinking_mode": False, | |
| "known_issues": [ | |
| "Q4_K_M compresses chain-of-thought from F16's 65.67 sec/case to 1.18 sec/case " | |
| "(55.7x faster). Whether this is a feature or regression depends on use case.", | |
| "mcq_02: F16 fails due to markdown fence wrapping; Q4_K_M suppresses fencing " | |
| "but selects wrong answer.", | |
| ], | |
| "series_notes": ( | |
| "Fastest non-3B model in the series at 1.18 sec/case Q4_K_M. " | |
| "Quantization dramatically compresses the reasoning chain vs F16 (65.67 sec). " | |
| "Use when speed matters and abbreviated reasoning is acceptable." | |
| ), | |
| }, | |
| "phi4-reasoning-plus": { | |
| "display_name": "Phi-4-reasoning-plus Q4_K_M", | |
| "short_name": "Phi-4-R+", | |
| "family": "Phi-4", | |
| "params": "14B", | |
| "context_window": 16384, | |
| "file_size_gb": 9.05, | |
| "vram_gb": 12.0, | |
| "avg_inference_sec": 25.84, | |
| "hf_repo": "pbhappliedsystems/phi-4-reasoning-plus-gguf-Q4-K-M", | |
| "hf_filename": "phi-4-reasoning-plus-gguf-Q4-K-M.gguf", | |
| "sha256": "2fe74424b03433d11ccf3f2ce8da404810fa7eb9a269135b1f14bf0d88566e4d", | |
| "run_id": "20260222_170914", | |
| "license": "MIT", | |
| "solo_only": False, | |
| "thinking_mode": False, | |
| "known_issues": [ | |
| "Systematic EOS token contamination: <|im_end|> appears as literal text. " | |
| "Strip before ALL downstream processing.", | |
| "json_multistep: 4/5 cases produce only <|im_end|> as entire response.", | |
| "mcq: all 5 cases fail with EOS token output — bucket_score=0.000.", | |
| "toolcall_only: 0/2 pass — prose output instead of JSON schema.", | |
| ], | |
| "series_notes": ( | |
| "Lowest reasoning (0.3648) and coherence (0.4921) in the evaluated series. " | |
| "Systematic EOS token contamination drives failures across planning, MCQ, " | |
| "and tool dispatch families. Demonstrates what rigorous pre-deployment " | |
| "evaluation surfaces that casual testing does not." | |
| ), | |
| }, | |
| "mistral-nemo": { | |
| "display_name": "Mistral-Nemo-Instruct-2407 Q4_K_M", | |
| "short_name": "Mistral-Nemo", | |
| "family": "Mistral", | |
| "params": "12B", | |
| "context_window": 128000, | |
| "file_size_gb": 7.48, | |
| "vram_gb": 10.0, | |
| "avg_inference_sec": 1.42, | |
| "hf_repo": "pbhappliedsystems/mistral-nemo-instruct-2407-gguf-Q4-K-M", | |
| "hf_filename": "mistral-nemo-instruct-2407-gguf-Q4-K-M.gguf", | |
| "sha256": "d3b7c7950abee2677290c1a3412940a0e1e3bc98449b42f34bc1d7ab6d4917b9", | |
| "run_id": "20260211_022944", | |
| "license": "Apache 2.0", | |
| "solo_only": False, | |
| "thinking_mode": False, | |
| "known_issues": [ | |
| "MCQ A-bias: mcq_02 and mcq_05 both produce 'A' (wrong).", | |
| "json_multistep: ms_hard_01 fails all four gating signals simultaneously.", | |
| "toolcall_only: args_ok=0.000 — add schema enforcement.", | |
| "toolcall tool_02: final answer wrong despite correct tool dispatch — " | |
| "validate post-execution.", | |
| ], | |
| "series_notes": ( | |
| "128K context window (Tekken tokenizer) — second largest in series " | |
| "after Qwen2.5-14B-1M's 1M. Multilingual: 9 languages. " | |
| "Strong instruction-following at 0.9329." | |
| ), | |
| }, | |
| "qwen3.6-27b": { | |
| "display_name": "Qwen3.6-27B Q4_K_M", | |
| "short_name": "Qwen3.6-27B", | |
| "family": "Qwen3", | |
| "params": "27B", | |
| "context_window": 32768, | |
| "file_size_gb": 16.5, | |
| "vram_gb": 22.0, | |
| "avg_inference_sec": 1.938, | |
| "hf_repo": "pbhappliedsystems/qwen3.6-27B-gguf-Q4-K-M", | |
| "hf_filename": "qwen3.6-27B-gguf-Q4-K-M.gguf", | |
| "sha256": "c863357b1b532a02c47ca363ab666dd623470a152a291dac6619ed7ce751d8c8", | |
| "run_id": "20260426_163540", | |
| "license": "Apache 2.0", | |
| "solo_only": False, | |
| "thinking_mode": True, | |
| "known_issues": [ | |
| "Hybrid thinking mode: <think> blocks generated on medium/hard tasks. " | |
| "json_multistep medium and hard cases fail with schema_ok=0 because the " | |
| "extraction layer receives the think block before the JSON. " | |
| "Strip <think>...</think> blocks before extraction, or use /no_think " | |
| "in user message to suppress thinking mode for structured output tasks.", | |
| "toolcall_only: args_ok=0.000 — uses 'arguments' instead of 'args'. " | |
| "tool_name key IS correct without enforcement (only model in series to do so). " | |
| "Specify 'args' explicitly in system prompt to resolve.", | |
| "EOS token contamination on toolcall final answers — " | |
| "strip <|im_end|> before downstream processing.", | |
| ], | |
| "series_notes": ( | |
| "First Qwen3-series model in the evaluated series. " | |
| "Hybrid adaptive thinking mode is the defining behavioral characteristic. " | |
| "json_multistep 0.400 is a pipeline compatibility finding, not a capability " | |
| "regression — easy cases pass cleanly; medium/hard require think-block stripping. " | |
| "Only model in the series to produce correct 'tool_name' key without enforcement. " | |
| "stateful_followup: 1.000. mixed_brief_json: 1.000. MCQ: 5/5 perfect. " | |
| "fuzz: 20/20 pass." | |
| ), | |
| }, | |
| } | |
| # --------------------------------------------------------------------------- | |
| # VRAM budget — ZeroGPU Nvidia H200 (141 GB HBM3e) | |
| # All models in the evaluated series can be paired without restriction. | |
| # --------------------------------------------------------------------------- | |
| ZEROGPU_VRAM_GB = 141.0 | |
| VRAM_SAFETY_CEILING_GB = 130.0 | |
| def pair_is_feasible(key_a: str, key_b: str) -> tuple[bool, str]: | |
| """ | |
| Returns (feasible: bool, reason: str). | |
| Checks for duplicate selection and combined VRAM against H200 ceiling. | |
| """ | |
| if key_a == key_b: | |
| return False, "Select two different models for comparison." | |
| combined = MODELS[key_a]["vram_gb"] + MODELS[key_b]["vram_gb"] | |
| if combined > VRAM_SAFETY_CEILING_GB: | |
| return False, ( | |
| f"Combined VRAM estimate ({combined:.1f} GB) exceeds safe ceiling " | |
| f"({VRAM_SAFETY_CEILING_GB} GB)." | |
| ) | |
| return True, f"Estimated combined VRAM: {combined:.1f} GB / {ZEROGPU_VRAM_GB} GB" | |
| # --------------------------------------------------------------------------- | |
| # Attach published results by SHA-256 (Plan S-19). Option (a), owner-approved scope: | |
| # - a model whose served file a v7.22 run evaluated gets that run's results, and its v7.21 notes and v7.21 | |
| # speed are dropped, so no model shows two harness versions side by side; | |
| # - every other model keeps its v7.21 notes, labelled, and shows no scores. | |
| # --------------------------------------------------------------------------- | |
| def _served_sha256(): | |
| """SHA-256 of the file each repo serves now, from its LFS metadata. A model whose lookup fails keeps the | |
| SHA-256 declared above, so a Hub outage can only hide results, never attach them to the wrong file.""" | |
| try: | |
| from huggingface_hub import HfApi | |
| api = HfApi() | |
| out = {} | |
| for key, m in MODELS.items(): | |
| try: | |
| info = api.get_paths_info(m["hf_repo"], [m["hf_filename"]]) | |
| lfs = getattr(info[0], "lfs", None) if info else None | |
| sha = getattr(lfs, "sha256", None) or (lfs.get("sha256") if isinstance(lfs, dict) else None) | |
| if sha: | |
| out[key] = sha | |
| except Exception: | |
| pass | |
| return out | |
| except Exception: | |
| return {} | |
| for _key, _sha in _served_sha256().items(): | |
| MODELS[_key]["sha256"] = _sha | |
| V721 = "Observed in quant_eval v7.21: " | |
| for _key, _m in MODELS.items(): | |
| _r = METRICS["builds"].get(_m["sha256"]) | |
| _m["results"] = _r | |
| if _r: | |
| _m["known_issues"] = [] | |
| _m["series_notes"] = "" | |
| _m["file_size_gb"] = round(_r["size_bytes"] / 1e9, 2) | |
| _m["avg_inference_sec"] = _r["sec_per_case"] | |
| _m["speed_note"] = f"{HARNESS}, {_r['accelerator']}" | |
| else: | |
| _m["known_issues"] = [V721 + i for i in _m["known_issues"]] | |
| _m["series_notes"] = (V721 + _m["series_notes"]) if _m["series_notes"] else "" | |
| _m["speed_note"] = "v7.21, RTX 4090" | |