scripts/integrity.py

"""File integrity and paired evaluation checks (no model dependencies)."""
import argparse
import hashlib
import json
from pathlib import Path


def sha256(path):
    with Path(path).open('rb') as f:
        return hashlib.file_digest(f, 'sha256').hexdigest()


def verify_files(root, rows):
    root = Path(root).resolve()
    verified = []
    for row in rows:
        path = (root / row['path']).resolve()
        if not path.is_relative_to(root):
            raise ValueError(f'Path outside snapshot: {row["path"]}')
        if path.stat().st_size != row['size']:
            raise ValueError(f'Size mismatch: {row["path"]}')
        digest = sha256(path)
        if digest != row['sha256']:
            raise ValueError(f'SHA256 mismatch: {row["path"]}')
        verified.append(dict(row))
    return verified


def validate_pair(a, b):
    keys = ('case_id', 'seed', 'prompt', 'width', 'height', 'steps', 'cfg',
            'offload', 'input_sha256', 'kv_cache')
    for key in keys:
        if key not in a or key not in b or a[key] != b[key]:
            raise ValueError(f'Incomparable pair: {key}')


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--model', default='models/bf16')
    ap.add_argument('--manifest', default='artifacts/upstream/huggingface-model-info.json')
    ap.add_argument('--output', default='artifacts/download-verification.json')
    args = ap.parse_args()
    info = json.loads(Path(args.manifest).read_text(encoding='utf-8'))
    rows = [{'path': x['rfilename'], 'size': x['size'], 'sha256': x['lfs']['sha256']}
            for x in info['siblings'] if x['rfilename'].endswith('.safetensors')]
    if len(rows) != 7:
        raise ValueError('Expected all seven official weight files')
    results = verify_files(args.model, rows)
    Path(args.output).parent.mkdir(parents=True, exist_ok=True)
    Path(args.output).write_text(json.dumps({'revision': info['sha'], 'files': results},
                                          indent=2), encoding='utf-8')
    print(f'Verified {len(results)} weight files against {info["sha"]}', flush=True)


if __name__ == '__main__':
    main()