| { |
| "schema": "br-voice-reasoner-evaluation-protocol.v1", |
| "evaluation_date_utc": "2026-09-02", |
| "benchmark": { |
| "name": "VoiceBench", |
| "upstream_repository": "https://github.com/matthewcym/VoiceBench", |
| "snapshot_identity": "SHA256 manifest", |
| "git_revision": null, |
| "revision_note": "The evaluated local snapshot did not retain Git metadata; release-relevant files are identified by SHA256." |
| }, |
| "subset_samples": { |
| "wildvoice": 1000, |
| "bbh": 1000, |
| "alpacaeval_full": 636, |
| "mmsu": 3074, |
| "openbookqa": 455, |
| "ifeval": 345, |
| "advbench": 520, |
| "commoneval": 200, |
| "sdqa_usa": 553, |
| "total": 7783 |
| }, |
| "generation": { |
| "reasoning_enabled": true, |
| "temperature": 0.6, |
| "top_p": 0.95, |
| "top_k": 20, |
| "maximum_tokens": 16384, |
| "seed_policy": "A deterministic per-sample primary seed is derived from the input and base seed 1234.", |
| "empty_final_policy": { |
| "primary_attempts": 3, |
| "maximum_consecutive_fallback_seeds": 8, |
| "selection": "First non-empty visible final answer; no ranking or best-of selection.", |
| "samples_entering_seed_fallback": 5, |
| "samples_recovered": 4, |
| "samples_exhausted": 1, |
| "exhausted_record": "No answer.", |
| "exhausted_scoring": "incorrect" |
| } |
| }, |
| "model_judge": { |
| "requested_model": "openai/gpt-4o-mini", |
| "model_kind": "alias", |
| "accessed_utc": "2026-09-02", |
| "gateway": "OpenRouter", |
| "provider_only": "OpenAI", |
| "allow_provider_fallbacks": false, |
| "judgments_per_sample": 3, |
| "vote_mode": "three separate requests without a seed", |
| "temperature": 0.5, |
| "top_p": 0.95, |
| "maximum_tokens": 1024 |
| }, |
| "aggregation": "Open-ended ratings are normalized to 0-100. Overall is the arithmetic mean of nine 0-100 subset scores.", |
| "sha256": { |
| "main.py": "cb7c86d7149b76d50e3316d09a377fb7cfa6549855ad2922f86b7dd980416f3e", |
| "evaluate.py": "0b610247b1373f2c256a4f183c2dfcbe7c562b576823cf4ce1396bf036ba1f42", |
| "src/models/qwen3_omni_thinking.py": "f44bcbc3c9b866c13b4db33ff156026dcf136f7fcfbd223ea006e600986543d8", |
| "qwen3omni_thinking/run_voicebench.sh": "4f040e0f695a12ee6a9fe4e1164c6fe3e57582160d5afca99d975f6ce7a7226c", |
| "qwen3omni_thinking/audit_outputs.py": "3ae9335dff1d568d19dc395bdf2b352b6a81685a9128edccdeca3882bdee7fb0", |
| "qwen3omni_thinking/score_local.sh": "332d701640ee1d76b3d7566e7250c55055546b3aa8186cd443531886635511ac", |
| "qwen3omni_thinking/score_thinking_mcq.py": "505018a7ae166f71f1a1d19b99887f69adc7fc802f91ceca930dbf2349f858eb", |
| "qwen3omni_thinking/api_judge_openrouter.py": "5b721566c1095e5dcf83b909f6ea667d127aed2533315b73f0819d168071ec33", |
| "qwen3omni_thinking/repro_utils.py": "9b36582a40c858f9ae6f5c64f7230dd7cf7db2e868d4f2bd77c446e70ea1a046" |
| } |
| } |
|
|
