Buckets:
| """IMF v1 zip loading: sha256 verification + extraction.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import zipfile | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| import yaml | |
| class ModelFormatError(ValueError): | |
| """The zip is not a valid IMF v1 artifact (or fails integrity).""" | |
| class Manifest: | |
| id: str | |
| task: str | |
| decoder: str | |
| precision: str | |
| opset: int | |
| sha256: dict[str, str] | |
| def load_manifest(zip_path: Path | str) -> Manifest: | |
| with zipfile.ZipFile(zip_path) as zf: | |
| names = zf.namelist() | |
| for required in ("metadata.yaml", "encoder.onnx", "decoder.onnx"): | |
| if required not in names: | |
| raise ModelFormatError(f"missing required file: {required}") | |
| raw = yaml.safe_load(zf.read("metadata.yaml")) | |
| if raw.get("format") != "imf-v1": | |
| raise ModelFormatError(f"unsupported format: {raw.get('format')!r}") | |
| if raw.get("tokenizer") != "bytes": | |
| raise ModelFormatError( | |
| f"tokenizer {raw.get('tokenizer')!r}: this runtime is byte-level only" | |
| ) | |
| return Manifest( | |
| id=raw["id"], | |
| task=raw["task"], | |
| decoder=raw.get("decoder", "plain"), | |
| precision=raw.get("precision", "fp32"), | |
| opset=int(raw.get("opset", 14)), | |
| sha256=dict(raw.get("sha256", {})), | |
| ) | |
| def verify_and_read(zip_path: Path | str) -> dict[str, bytes]: | |
| """Read .onnx members after verifying each sha256 against the | |
| manifest — the corrupt-download failure mode fails loudly here.""" | |
| manifest = load_manifest(zip_path) | |
| graphs: dict[str, bytes] = {} | |
| with zipfile.ZipFile(zip_path) as zf: | |
| for name in [n for n in zf.namelist() if n.endswith(".onnx")]: | |
| member = zf.read(name) | |
| recorded = manifest.sha256.get(name) | |
| if recorded is None: | |
| raise ModelFormatError(f"{name} is not covered by metadata sha256") | |
| actual = hashlib.sha256(member).hexdigest() | |
| if actual != recorded: | |
| raise ModelFormatError( | |
| f"{name} sha256 mismatch: zip has {actual}, " | |
| f"metadata says {recorded}" | |
| ) | |
| graphs[name] = member | |
| return graphs | |
Xet Storage Details
- Size:
- 2.32 kB
- Xet hash:
- 27f108ebff9ca7f0b57719f7df9d0a4217b0e86e5932c6366c61c665e978afc3
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.