File size: 2,151 Bytes
9116984 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | """File integrity and paired evaluation checks (no model dependencies)."""
import argparse
import hashlib
import json
from pathlib import Path
def sha256(path):
with Path(path).open('rb') as f:
return hashlib.file_digest(f, 'sha256').hexdigest()
def verify_files(root, rows):
root = Path(root).resolve()
verified = []
for row in rows:
path = (root / row['path']).resolve()
if not path.is_relative_to(root):
raise ValueError(f'Path outside snapshot: {row["path"]}')
if path.stat().st_size != row['size']:
raise ValueError(f'Size mismatch: {row["path"]}')
digest = sha256(path)
if digest != row['sha256']:
raise ValueError(f'SHA256 mismatch: {row["path"]}')
verified.append(dict(row))
return verified
def validate_pair(a, b):
keys = ('case_id', 'seed', 'prompt', 'width', 'height', 'steps', 'cfg',
'offload', 'input_sha256', 'kv_cache', 'source_seed')
for key in keys:
if key not in a or key not in b or a[key] != b[key]:
raise ValueError(f'Incomparable pair: {key}')
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--model', default='models/bf16')
ap.add_argument('--manifest', default='artifacts/upstream/huggingface-model-info.json')
ap.add_argument('--output', default='artifacts/download-verification.json')
args = ap.parse_args()
info = json.loads(Path(args.manifest).read_text(encoding='utf-8'))
rows = [{'path': x['rfilename'], 'size': x['size'], 'sha256': x['lfs']['sha256']}
for x in info['siblings'] if x['rfilename'].endswith('.safetensors')]
if len(rows) != 7:
raise ValueError('Expected all seven official weight files')
results = verify_files(args.model, rows)
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text(json.dumps({'revision': info['sha'], 'files': results},
indent=2), encoding='utf-8')
print(f'Verified {len(results)} weight files against {info["sha"]}', flush=True)
if __name__ == '__main__':
main()
|