File size: 9,096 Bytes
5be8f06 270de32 5be8f06 877d10c 5be8f06 270de32 5be8f06 d258bfe 5be8f06 d258bfe 5be8f06 d258bfe 5be8f06 d258bfe 5be8f06 270de32 5be8f06 270de32 5be8f06 270de32 5be8f06 d258bfe 5be8f06 d258bfe 5be8f06 877d10c 5be8f06 877d10c 5be8f06 270de32 5be8f06 877d10c 5be8f06 877d10c 5be8f06 877d10c 5be8f06 89d9250 5be8f06 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 | """
Automated integrity and statistical invariant test suite for the canonical
Armiger & Armiger Ultra chart and benchmark pipeline.
"""
from __future__ import annotations
import hashlib
import json
import pathlib
import sys
import numpy as np
import pytest
BENCH_DIR = pathlib.Path(__file__).resolve().parent
REPO_ROOT = BENCH_DIR.parent
if str(BENCH_DIR) not in sys.path:
sys.path.insert(0, str(BENCH_DIR))
def _load_vectors(path: pathlib.Path):
raw = json.loads(path.read_text(encoding="utf-8"))
dtypes = raw["_meta"]["dtypes"]
return {k: np.array(v, dtype=np.dtype(dtypes[k])) for k, v in raw["vectors"].items()}
from generate_canonical_charts import ( # noqa: E402
BOOTSTRAP_RESAMPLES,
BOOTSTRAP_SEED,
HAS_MATPLOTLIB,
assert_identical_denominator,
assert_non_circular_teacher_eval,
build_all_charts_and_manifest,
compute_paired_panel_stats,
verify_png_chunks_clean,
)
def test_identical_denominator_lock_enforced():
"""Verify that any denominator mismatch across models on a panel raises AssertionError."""
v_1025 = np.ones(1025, dtype=np.float32)
v_1009 = np.ones(1009, dtype=np.float32) # Dropped 16 'unknown' items
v_2835 = np.ones(2835, dtype=np.float32)
v_2825 = np.ones(2825, dtype=np.float32) # Dropped 10 'unknown' items
v_545 = np.ones(545, dtype=np.float32)
v_849 = np.ones(849, dtype=np.float32) # Mismatched audio language split
assert assert_identical_denominator("ok_1025", {"ultra": v_1025, "gpt": v_1025}, expected_n=1025) == 1025
with pytest.raises(AssertionError, match="Identical Denominator Lock violated"):
assert_identical_denominator("bad_beyond_unknown_drop", {"ultra": v_1025, "gpt": v_1009})
with pytest.raises(AssertionError, match="Identical Denominator Lock violated"):
assert_identical_denominator("bad_oct15_unknown_drop", {"ultra": v_2835, "mimmo": v_2825})
with pytest.raises(AssertionError, match="Identical Denominator Lock violated"):
assert_identical_denominator("bad_audio_split_mix", {"ultra": v_545, "gpt": v_849})
with pytest.raises(AssertionError, match="Expected N=636"):
assert_identical_denominator("bad_expected_n", {"ultra": np.ones(650), "gpt": np.ones(650)}, expected_n=636)
v_nan = v_1025.copy()
v_nan[0] = np.nan
with pytest.raises(AssertionError, match="NaN"):
assert_identical_denominator("bad_nan", {"ultra": v_1025, "gpt": v_nan})
def test_anti_circularity_lock_enforced():
"""Verify that scoring a teacher model against its own circular majority vote raises AssertionError."""
d = _load_vectors(BENCH_DIR / "canonical_eval_vectors.json")
with pytest.raises(AssertionError, match="Anti-Circularity Lock violated"):
assert_non_circular_teacher_eval("OpenAI GPT (Fast Tier)", eval_mode="circular_majority", self_vote_weight=0.333)
with pytest.raises(AssertionError, match="Anti-Circularity Lock violated"):
assert_non_circular_teacher_eval("Google Gemini Flash", eval_mode="leave_one_teacher_out", self_vote_weight=0.5)
gpt_audit = assert_non_circular_teacher_eval(
"OpenAI GPT (Fast Tier)",
eval_mode="leave_one_teacher_out",
self_vote_weight=0.0,
noncirc_vec=d["beyond_cat11_corr_gpt_noncirc"],
circ_vec=d["beyond_cat11_corr_gpt_circ"],
)
assert gpt_audit["non_circular_pct"] == 95.12
assert gpt_audit["circular_self_vote_pct"] == 96.59
assert gpt_audit["circularity_inflation_pp"] > 1.0
gem_audit = assert_non_circular_teacher_eval(
"Google Gemini Flash",
eval_mode="leave_one_teacher_out",
self_vote_weight=0.0,
noncirc_vec=d["beyond_cat11_corr_gemini_noncirc"],
circ_vec=d["beyond_cat11_corr_gemini_circ"],
)
assert gem_audit["non_circular_pct"] == 93.80
assert gem_audit["circular_self_vote_pct"] == 95.61
assert gem_audit["circularity_inflation_pp"] > 1.5
def test_eval_vectors_privacy_and_dtypes():
"""Verify that canonical_eval_vectors.json contains only numeric int16/float32 arrays and zero strings."""
npz_path = BENCH_DIR / "canonical_eval_vectors.json"
assert npz_path.is_file()
d = _load_vectors(npz_path)
assert len(list(d.keys())) >= 140
for k in list(d.keys()):
arr = d[k]
assert arr.dtype in (np.int16, np.float32), f"Array {k} has disallowed dtype {arr.dtype}"
assert arr.ndim == 1, f"Array {k} must be 1D, got shape {arr.shape}"
assert not np.isnan(arr).any(), f"Array {k} contains NaNs"
def test_paired_bootstrap_and_mcnemar_reproducibility():
"""Verify that paired bootstrap 95% CIs and McNemar p-values are 100% deterministic."""
d = _load_vectors(BENCH_DIR / "canonical_eval_vectors.json")
vecs = {
"Armiger Ultra": (d["oct15_aud545_pred_ultra"] == d["oct15_aud545_y_en"]).astype(np.float32),
"Armiger INT8": (d["oct15_aud545_pred_int8_hyb"] == d["oct15_aud545_y_en"]).astype(np.float32),
"GuessIt": (d["oct15_aud545_pred_guessit"] == d["oct15_aud545_y_en"]).astype(np.float32),
"OpenAI GPT (Fast Tier)": (d["oct15_aud545_pred_gpt"] == d["oct15_aud545_y_en"]).astype(np.float32),
}
res1 = compute_paired_panel_stats("test_aud545", vecs, ref_model="Armiger Ultra", expected_n=545)
res2 = compute_paired_panel_stats("test_aud545", vecs, ref_model="Armiger Ultra", expected_n=545)
assert res1 == res2
for m, st in res1["models"].items():
assert st["n"] == 545
assert st["ci95_lo"] <= st["point_pct"] <= st["ci95_hi"]
assert 0.0 <= st["p_value_vs_ref"] <= 1.0
assert res1["models"]["Armiger Ultra"]["point_pct"] == 97.80
assert res1["models"]["GuessIt"]["point_pct"] == 94.86
assert res1["models"]["OpenAI GPT (Fast Tier)"]["point_pct"] == 94.68
assert res1["models"]["OpenAI GPT (Fast Tier)"]["p_value_vs_ref"] < 0.01
def test_manifest_sha256_and_png_chunks(tmp_path: pathlib.Path):
"""Verify CHARTS_MANIFEST.json provenance hashes, PNG chunk hygiene, and byte-exact determinism."""
manifest_path = BENCH_DIR / "CHARTS_MANIFEST.json"
assert manifest_path.is_file(), "CHARTS_MANIFEST.json missing"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
assert manifest["bootstrap_seed"] == BOOTSTRAP_SEED
assert manifest["bootstrap_resamples"] == BOOTSTRAP_RESAMPLES
for k, v in manifest["invariants_verified"].items():
assert v is True, f"Invariant {k} not verified"
# Verify all provenance SHA-256 hashes match disk
for rel_path, expected_sha in manifest["provenance_sha256"].items():
target = (BENCH_DIR / rel_path) if (BENCH_DIR / rel_path).is_file() else (REPO_ROOT / rel_path)
assert target.is_file(), f"Provenance file missing: {rel_path}"
actual_sha = hashlib.sha256(target.read_bytes()).hexdigest()
assert actual_sha == expected_sha, f"SHA-256 mismatch for {rel_path}: {actual_sha} != {expected_sha}"
# Verify all 4 PNGs have only IHDR, IDAT, IEND chunks and RGB mode
for chart_name, chart_meta in manifest["charts"].items():
png_path = REPO_ROOT / "assets" / chart_name
assert png_path.is_file(), f"Chart missing: {png_path}"
chunks = verify_png_chunks_clean(png_path)
assert chunks[0] == "IHDR" and chunks[-1] == "IEND"
assert set(chunks).issubset({"IHDR", "IDAT", "IEND"})
# Re-run pipeline into tmp_path and verify statistical + PNG determinism
tmp_assets = tmp_path / "assets"
tmp_manifest = tmp_path / "CHARTS_MANIFEST.json"
render_mode = bool(HAS_MATPLOTLIB and sys.platform == "darwin")
re_manifest = build_all_charts_and_manifest(
BENCH_DIR / "canonical_eval_vectors.json",
tmp_assets,
tmp_manifest,
render_pngs=render_mode,
)
assert re_manifest["invariants_verified"] == manifest["invariants_verified"]
assert re_manifest["anti_circularity_audit"] == manifest["anti_circularity_audit"]
assert re_manifest["five_split_category_summary"] == manifest["five_split_category_summary"]
for chart_name, chart_meta in manifest["charts"].items():
assert re_manifest["charts"][chart_name] == chart_meta, (
f"Non-deterministic manifest or PNG output for {chart_name}"
)
def test_readme_tables_match_charts_manifest():
"""Verify README.md contains canonical CHARTS_MANIFEST.json metrics and zero stale numbers."""
readme_path = REPO_ROOT / "README.md"
assert readme_path.is_file()
text = readme_path.read_text(encoding="utf-8")
# Canonical numbers that must appear in README
for required_token in ("95.49%", "97.30%", "96.53%", "97.80%", "96.83%"):
assert required_token in text, f"Missing canonical metric {required_token} in {readme_path}"
# Stale pre-audit numbers that must never reappear in README
for stale_token in ("97.13%", "96.35%", "72.35%", "35.29%", "97.55%", "true" + "spec", "tru" + "spec", "tc" + "2000", "multilang" + "3000"):
assert stale_token not in text, f"Stale metric {stale_token} found in {readme_path}"
|