Download scripts/eval.py from SZLHOLDINGS/szl-nemo: direct link, hf CLI and curl.
- Browser
- Download file 1.55 kB
-
https://huggingface.co/SZLHOLDINGS/szl-nemo/resolve/main/scripts/eval.py
- Command line
-
hf download hf://SZLHOLDINGS/szl-nemo/scripts/eval.py
-
curl -L -o eval.py https://huggingface.co/SZLHOLDINGS/szl-nemo/resolve/main/scripts/eval.py
1.55 kB
| #!/usr/bin/env python3 | |
| """Re-verify the szl-nemo recipe-conformance scorer. | |
| 1. sha256 the shipped model.joblib against TRAINING_RECEIPT.json (refuse on mismatch). | |
| 2. Deterministically regenerate the seeded dataset via scripts/forge.py and compare | |
| re-measured fidelity to the receipt (tolerance ±0.02 across library versions). | |
| Run from repo root: python scripts/eval.py""" | |
| import hashlib, json, subprocess, sys, tempfile, os, shutil | |
| root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| receipt = json.load(open(f"{root}/TRAINING_RECEIPT.json")) | |
| got = hashlib.sha256(open(f"{root}/model.joblib", "rb").read()).hexdigest() | |
| want = receipt["model"]["sha256"] | |
| print(f"model.joblib sha256 {'MATCHES receipt' if got == want else 'MISMATCH — refuse'}: {got[:16]}…") | |
| if got != want: | |
| sys.exit(1) | |
| with tempfile.TemporaryDirectory() as td: | |
| copy = f"{td}/repo" | |
| shutil.copytree(root, copy, ignore=shutil.ignore_patterns(".cache")) | |
| out = subprocess.run([sys.executable, f"{copy}/scripts/forge.py"], | |
| capture_output=True, text=True, cwd=copy) | |
| print(out.stdout[-500:] if out.returncode == 0 else out.stderr[-800:]) | |
| if out.returncode: | |
| sys.exit(1) | |
| re_receipt = json.load(open(f"{copy}/TRAINING_RECEIPT.json")) | |
| d = abs(re_receipt["metrics_MEASURED"]["fidelity_vs_rule_checker"] | |
| - receipt["metrics_MEASURED"]["fidelity_vs_rule_checker"]) | |
| print(f"re-measured fidelity delta vs receipt: {d:.4f} ({'OK <=0.02' if d <= 0.02 else 'FAIL'})") | |
| sys.exit(0 if d <= 0.02 else 1) | |