#!/usr/bin/env python3 """Verify integrity of every local file against the recorded Hub hashes. * downloaded source loading files -> compared to the Hub's git blob SHA-1 (git object id = sha1("blob {size}\\0" + content)) or LFS SHA-256; * NOEON-produced weights -> compared to the SHA-256 in weights/sha256.json. Usage: python scripts/verify_integrity.py """ from __future__ import annotations import hashlib import json import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent def git_blob_sha1(p: Path) -> str: data = p.read_bytes() h = hashlib.sha1() h.update(f"blob {len(data)}\0".encode()) h.update(data) return h.hexdigest() def sha256(p: Path) -> str: h = hashlib.sha256() with open(p, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def main() -> int: man_path = ROOT / "weights" / "source_files.json" manifest = json.loads(man_path.read_text()) base = ROOT / "weights" / "source_loading_files" checked = ok = missing = mismatch = 0 problems = [] for key, entry in manifest["sources"].items(): for f in entry.get("files", []): local = base / key / f["path"] if not local.exists(): continue checked += 1 if f.get("lfs_sha256"): want = f["lfs_sha256"] got = sha256(local) elif f.get("blob_sha1"): want = f["blob_sha1"] got = git_blob_sha1(local) else: continue if got == want: ok += 1 else: mismatch += 1 problems.append((str(local), want, got)) print(f"source loading files: checked {checked}, verified {ok}, " f"mismatch {mismatch}") for p, w, g in problems: print(f" MISMATCH {p}\n want {w}\n got {g}") # NOEON-produced weights sh = ROOT / "weights" / "sha256.json" if sh.exists(): rec = json.loads(sh.read_text()) for rel, want in rec.items(): p = ROOT / rel if not p.exists(): missing += 1 print(f" MISSING {rel}") continue got = sha256(p) if got == want: ok += 1 else: mismatch += 1 print(f" MISMATCH {rel}") return 1 if (mismatch or missing) else 0 if __name__ == "__main__": raise SystemExit(main())