Download scripts/verify_integrity.py from Frankenstein-Labs/NOEON: direct link, hf CLI and curl.
- Browser
- Download file 2.56 kB
-
https://huggingface.co/Frankenstein-Labs/NOEON/resolve/main/scripts/verify_integrity.py
- Command line
-
hf download hf://Frankenstein-Labs/NOEON/scripts/verify_integrity.py
-
curl -L -o verify_integrity.py https://huggingface.co/Frankenstein-Labs/NOEON/resolve/main/scripts/verify_integrity.py
2.56 kB
| #!/usr/bin/env python3 | |
| """Verify integrity of every local file against the recorded Hub hashes. | |
| * downloaded source loading files -> compared to the Hub's git blob SHA-1 | |
| (git object id = sha1("blob {size}\\0" + content)) or LFS SHA-256; | |
| * NOEON-produced weights -> compared to the SHA-256 in weights/sha256.json. | |
| Usage: | |
| python scripts/verify_integrity.py | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import sys | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent.parent | |
| def git_blob_sha1(p: Path) -> str: | |
| data = p.read_bytes() | |
| h = hashlib.sha1() | |
| h.update(f"blob {len(data)}\0".encode()) | |
| h.update(data) | |
| return h.hexdigest() | |
| def sha256(p: Path) -> str: | |
| h = hashlib.sha256() | |
| with open(p, "rb") as f: | |
| for chunk in iter(lambda: f.read(1 << 20), b""): | |
| h.update(chunk) | |
| return h.hexdigest() | |
| def main() -> int: | |
| man_path = ROOT / "weights" / "source_files.json" | |
| manifest = json.loads(man_path.read_text()) | |
| base = ROOT / "weights" / "source_loading_files" | |
| checked = ok = missing = mismatch = 0 | |
| problems = [] | |
| for key, entry in manifest["sources"].items(): | |
| for f in entry.get("files", []): | |
| local = base / key / f["path"] | |
| if not local.exists(): | |
| continue | |
| checked += 1 | |
| if f.get("lfs_sha256"): | |
| want = f["lfs_sha256"] | |
| got = sha256(local) | |
| elif f.get("blob_sha1"): | |
| want = f["blob_sha1"] | |
| got = git_blob_sha1(local) | |
| else: | |
| continue | |
| if got == want: | |
| ok += 1 | |
| else: | |
| mismatch += 1 | |
| problems.append((str(local), want, got)) | |
| print(f"source loading files: checked {checked}, verified {ok}, " | |
| f"mismatch {mismatch}") | |
| for p, w, g in problems: | |
| print(f" MISMATCH {p}\n want {w}\n got {g}") | |
| # NOEON-produced weights | |
| sh = ROOT / "weights" / "sha256.json" | |
| if sh.exists(): | |
| rec = json.loads(sh.read_text()) | |
| for rel, want in rec.items(): | |
| p = ROOT / rel | |
| if not p.exists(): | |
| missing += 1 | |
| print(f" MISSING {rel}") | |
| continue | |
| got = sha256(p) | |
| if got == want: | |
| ok += 1 | |
| else: | |
| mismatch += 1 | |
| print(f" MISMATCH {rel}") | |
| return 1 if (mismatch or missing) else 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |