NOEON / scripts /verify_integrity.py
Frankenstein-Labs's picture
NOEON: full model, code, configs, tokenizers, manifests
419f250 verified
Raw History Blame Contribute Delete
2.56 kB
#!/usr/bin/env python3
"""Verify integrity of every local file against the recorded Hub hashes.
* downloaded source loading files -> compared to the Hub's git blob SHA-1
(git object id = sha1("blob {size}\\0" + content)) or LFS SHA-256;
* NOEON-produced weights -> compared to the SHA-256 in weights/sha256.json.
Usage:
python scripts/verify_integrity.py
"""
from __future__ import annotations
import hashlib
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
def git_blob_sha1(p: Path) -> str:
data = p.read_bytes()
h = hashlib.sha1()
h.update(f"blob {len(data)}\0".encode())
h.update(data)
return h.hexdigest()
def sha256(p: Path) -> str:
h = hashlib.sha256()
with open(p, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def main() -> int:
man_path = ROOT / "weights" / "source_files.json"
manifest = json.loads(man_path.read_text())
base = ROOT / "weights" / "source_loading_files"
checked = ok = missing = mismatch = 0
problems = []
for key, entry in manifest["sources"].items():
for f in entry.get("files", []):
local = base / key / f["path"]
if not local.exists():
continue
checked += 1
if f.get("lfs_sha256"):
want = f["lfs_sha256"]
got = sha256(local)
elif f.get("blob_sha1"):
want = f["blob_sha1"]
got = git_blob_sha1(local)
else:
continue
if got == want:
ok += 1
else:
mismatch += 1
problems.append((str(local), want, got))
print(f"source loading files: checked {checked}, verified {ok}, "
f"mismatch {mismatch}")
for p, w, g in problems:
print(f" MISMATCH {p}\n want {w}\n got {g}")
# NOEON-produced weights
sh = ROOT / "weights" / "sha256.json"
if sh.exists():
rec = json.loads(sh.read_text())
for rel, want in rec.items():
p = ROOT / rel
if not p.exists():
missing += 1
print(f" MISSING {rel}")
continue
got = sha256(p)
if got == want:
ok += 1
else:
mismatch += 1
print(f" MISMATCH {rel}")
return 1 if (mismatch or missing) else 0
if __name__ == "__main__":
raise SystemExit(main())