#!/usr/bin/env python """Verify a local copy of the PULSE dataset against the Hugging Face file list. For every file in the list (``scripts/fetch_hf_manifest.py`` output), LFS files are compared by sha256 and plain git files by their git blob id (sha1 of ``"blob \\0" + content``). Files present locally but not on the Hub are reported as extra. Exit status 1 if anything differs. python scripts/verify_data.py --root $PULSE_ROOT \ --manifest manifests/hf_PULSE_v1.1_files.tsv --workers 16 --out verify_report.json ``--expect-different`` and ``--ignore-extra`` take glob patterns for files that are known to differ (e.g. a de-anonymised README) or stray local files. """ from __future__ import annotations import argparse import fnmatch import hashlib import json import os import sys import time from concurrent.futures import ProcessPoolExecutor CHUNK = 8 << 20 def read_manifest(path): header, rows = {}, [] with open(path, encoding="utf-8") as f: for line in f: line = line.rstrip("\n") if line.startswith("#"): for kv in line[1:].strip().split("\t"): if "=" in kv: k, v = kv.split("=", 1) header[k.strip()] = v.strip() continue if line.startswith("path\t") or not line: continue p, size, oid, sha = (line.split("\t") + ["", "", "", ""])[:4] rows.append({"path": p, "size": int(size), "git_oid": oid, "lfs_sha256": sha}) return header, rows def hash_file(args): root, row = args p = os.path.join(root, row["path"]) try: size = os.path.getsize(p) except OSError: return row["path"], "missing", None, None if size != row["size"]: return row["path"], "size_mismatch", size, None if row["lfs_sha256"]: h = hashlib.sha256() kind = "sha256" else: h = hashlib.sha1(f"blob {size}\0".encode()) kind = "git_sha1" with open(p, "rb") as f: for chunk in iter(lambda: f.read(CHUNK), b""): h.update(chunk) digest = h.hexdigest() expected = row["lfs_sha256"] or row["git_oid"] return row["path"], ("ok" if digest == expected else f"{kind}_mismatch"), size, digest def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--root", default=os.environ.get("PULSE_ROOT")) ap.add_argument("--manifest", required=True) ap.add_argument("--workers", type=int, default=8) ap.add_argument("--out", default=None, help="JSON report path") ap.add_argument("--expect-different", action="append", default=[], help="glob of Hub files allowed to differ (reported separately)") ap.add_argument("--ignore-extra", action="append", default=[], help="glob of local-only files to ignore (reported separately)") args = ap.parse_args(argv) if not args.root or not os.path.isdir(args.root): raise SystemExit("--root (or $PULSE_ROOT) must be the dataset root") header, rows = read_manifest(args.manifest) t0 = time.time() with ProcessPoolExecutor(max_workers=args.workers) as ex: results = list(ex.map(hash_file, [(args.root, r) for r in rows], chunksize=4)) status = {} for path, st, size, digest in results: status[path] = {"status": st, "size": size, "digest": digest} hub = {r["path"] for r in rows} extra, ignored_extra = [], [] for dp, dns, fns in os.walk(args.root): dns[:] = [d for d in dns if d != ".git" and d != ".cache"] for fn in fns: rel = os.path.relpath(os.path.join(dp, fn), args.root).replace(os.sep, "/") if rel in hub: continue if any(fnmatch.fnmatch(rel, g) for g in args.ignore_extra): ignored_extra.append(rel) else: extra.append(rel) def allowed(p): return any(fnmatch.fnmatch(p, g) for g in args.expect_different) bad = {p: s for p, s in status.items() if s["status"] != "ok" and not allowed(p)} expected_diff = {p: s for p, s in status.items() if s["status"] != "ok" and allowed(p)} counts = {} for s in status.values(): counts[s["status"]] = counts.get(s["status"], 0) + 1 n_bytes = sum(s["size"] or 0 for s in status.values()) report = { "hub": header, "root_name": os.path.basename(os.path.normpath(args.root)), "n_hub_files": len(rows), "status_counts": counts, "n_ok": counts.get("ok", 0), "problems": bad, "expected_differences": expected_diff, "extra_local_files": sorted(extra), "ignored_extra_local_files": sorted(ignored_extra), "bytes_checked": n_bytes, "seconds": round(time.time() - t0, 1), "verdict": "PASS" if not bad and not extra else "FAIL", } if args.out: with open(args.out, "w", encoding="utf-8", newline="\n") as f: json.dump(report, f, indent=1) f.write("\n") print(f"{header.get('repo')}@{header.get('revision')} ({header.get('commit', '')[:10]}): " f"{len(rows)} files, {n_bytes / 1e9:.2f} GB checked in {report['seconds']} s") print(f"status: {counts}") print(f"problems: {len(bad)}; expected differences: {len(expected_diff)} " f"{sorted(expected_diff)}; extra local files: {len(extra)}; ignored extra: " f"{len(ignored_extra)}") for p, s in sorted(bad.items())[:50]: print(f" {s['status']:16s} {p}") for p in sorted(extra)[:50]: print(f" extra {p}") print(f"verdict: {report['verdict']}") return 0 if report["verdict"] == "PASS" else 1 if __name__ == "__main__": sys.exit(main())