Buckets:
| """WO37 — portfolio eval: fas-g2p / urd-diac / urd-g2p shipped artifacts | |
| on their own held-out test splits. | |
| Metrics: char-level CER (raw) + word accuracy (all chars correct). | |
| Artifacts: GitHub Release URLs from models.yaml, sha-verified (parts | |
| concatenated for multi-part). CPU ONNX inference (the runtime's | |
| provider contract); hours-scale is expected and fine. | |
| Usage (cpu-performance, token): | |
| python /ckpt/run-056-portfolio-eval/portfolio_eval.py | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import io | |
| import json | |
| import os | |
| import time | |
| from pathlib import Path | |
| OUT = Path("/ckpt/run-056-portfolio-eval") | |
| OUT.mkdir(parents=True, exist_ok=True) | |
| ARTIFACTS = { | |
| "fas-g2p-1.0": { | |
| "zip": "fas-g2p-1.0-fp32.zip", | |
| "parts": [["fas-g2p-1.0-fp32.zip.part-00", | |
| "6c475e53f3c7ef3edcef5a24d2d81f51d13ddf4c417de98440943c9a32b03e55"], | |
| ["fas-g2p-1.0-fp32.zip.part-01", | |
| "c5656c649412f41e201c567e002514d31390b636d4650d9dcb6dfed1973fb3d5"]], | |
| "test": "/data-rf/data-v3/test.jsonl", | |
| "src_key": "src", "tgt_key": "tgt", | |
| }, | |
| "urd-diac-1.0": { | |
| "zip": "urd-diac-1.0-fp32.zip", | |
| "url": ("https://github.com/interscript/interscript-models/releases/" | |
| "download/urd-diac-1.0/urd-diac-1.0-fp32.zip"), | |
| "sha256": "c299ea44a4da4851a52283f8e42d2fb6f8c7a9c443ba7eb0270f97372e10b8ea", | |
| "test": "/data-ru/data-diacrit/test.jsonl", | |
| "src_key": "src", "tgt_key": "tgt", | |
| }, | |
| "urd-g2p-1.0": { | |
| "zip": "urd-g2p-1.0-fp32.zip", | |
| "url": ("https://github.com/interscript/interscript-models/releases/" | |
| "download/urd-g2p-1.0/urd-g2p-1.0-fp32.zip"), | |
| "sha256": "9f0a248c7116aa7f6b9ac125fb9684a3556f3b8ce8b64567c05a5e7e7a018ebf", | |
| "test": "/data-ru/data/test.jsonl", | |
| "src_key": "src", "tgt_key": "tgt", | |
| }, | |
| } | |
| def fetch_artifact(name: str, spec: dict) -> Path: | |
| dest = OUT / spec["zip"] | |
| if dest.exists() and dest.stat().st_size > 0: | |
| return dest | |
| if "parts" in spec: | |
| buf = b"" | |
| for fname, sha in spec["parts"]: | |
| p = OUT / fname | |
| if not p.exists() or p.stat().st_size == 0: | |
| import subprocess | |
| url = ("https://github.com/interscript/interscript-models/" | |
| f"releases/download/{name}/{fname}") | |
| subprocess.run(["curl", "-sL", "-o", str(p), url], check=True) | |
| got = hashlib.sha256(p.read_bytes()).hexdigest() | |
| assert got == sha, f"{fname} sha mismatch {got}" | |
| buf += p.read_bytes() | |
| dest.write_bytes(buf) | |
| return dest | |
| import subprocess | |
| subprocess.run(["curl", "-sL", "-o", str(dest), spec["url"]], check=True) | |
| got = hashlib.sha256(dest.read_bytes()).hexdigest() | |
| assert got == spec["sha256"], f"{name} sha mismatch {got}" | |
| return dest | |
| def lev(a: str, b: str) -> int: | |
| prev = list(range(len(b) + 1)) | |
| for i, ca in enumerate(a, 1): | |
| cur = [i] | |
| for j, cb in enumerate(b, 1): | |
| cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb))) | |
| prev = cur | |
| return prev[-1] | |
| def eval_lang(name: str, spec: dict) -> dict: | |
| import sys | |
| sys.path.insert(0, "/code/ml-py") | |
| from interscript.ml.model import Model | |
| t0 = time.time() | |
| zip_p = fetch_artifact(name, spec) | |
| model = Model.load(zip_p) | |
| print(f"[{name}] artifact loaded {time.time()-t0:.0f}s", flush=True) | |
| rows = [json.loads(l) for l in Path(spec["test"]).read_text( | |
| encoding="utf-8").splitlines() if l.strip()] | |
| cap = int(os.environ.get("EVAL_CAP", "3000")) | |
| if len(rows) > cap: | |
| rows = rows[:cap] | |
| cers = [] | |
| correct_words = 0 | |
| n = 0 | |
| for i, r in enumerate(rows): | |
| src = (r.get(spec["src_key"]) or "").strip() | |
| gold = (r.get(spec["tgt_key"]) or "").strip() | |
| if not src or not gold: | |
| continue | |
| pred = model.translate(src) | |
| cers.append(lev(pred, gold) / max(1, len(gold))) | |
| if pred == gold: | |
| correct_words += 1 | |
| n += 1 | |
| if n % 250 == 0: | |
| print(f"[{name}] {n}/{len(rows)} cer={sum(cers)/len(cers):.4f} " | |
| f"({time.time()-t0:.0f}s)", flush=True) | |
| res = {"lang": name, "n": n, "cer": round(sum(cers) / max(1, len(cers)), 4), | |
| "word_acc": round(correct_words / max(1, n), 4), | |
| "wall_secs": round(time.time() - t0, 1)} | |
| print(json.dumps(res), flush=True) | |
| return res | |
| def main() -> None: | |
| results = {} | |
| for name, spec in ARTIFACTS.items(): | |
| results[name] = eval_lang(name, spec) | |
| (OUT / "eval_results.json").write_text( | |
| json.dumps(results, indent=1)) | |
| (OUT / "EVAL_DONE").write_text("ok\n") | |
| print("[done]", flush=True) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 4.85 kB
- Xet hash:
- b1f7c383f47b8319f58d2345f283e4dd91c0f2b221cbd1429ed60169de277985
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.