File size: 5,806 Bytes
dffbc1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 | #!/usr/bin/env python
"""Verify a local copy of the PULSE dataset against the Hugging Face file list.
For every file in the list (``scripts/fetch_hf_manifest.py`` output), LFS files
are compared by sha256 and plain git files by their git blob id
(sha1 of ``"blob <size>\\0" + content``). Files present locally but not on the
Hub are reported as extra. Exit status 1 if anything differs.
python scripts/verify_data.py --root $PULSE_ROOT \
--manifest manifests/hf_PULSE_v1.1_files.tsv --workers 16 --out verify_report.json
``--expect-different`` and ``--ignore-extra`` take glob patterns for files that
are known to differ (e.g. a de-anonymised README) or stray local files.
"""
from __future__ import annotations
import argparse
import fnmatch
import hashlib
import json
import os
import sys
import time
from concurrent.futures import ProcessPoolExecutor
CHUNK = 8 << 20
def read_manifest(path):
header, rows = {}, []
with open(path, encoding="utf-8") as f:
for line in f:
line = line.rstrip("\n")
if line.startswith("#"):
for kv in line[1:].strip().split("\t"):
if "=" in kv:
k, v = kv.split("=", 1)
header[k.strip()] = v.strip()
continue
if line.startswith("path\t") or not line:
continue
p, size, oid, sha = (line.split("\t") + ["", "", "", ""])[:4]
rows.append({"path": p, "size": int(size), "git_oid": oid, "lfs_sha256": sha})
return header, rows
def hash_file(args):
root, row = args
p = os.path.join(root, row["path"])
try:
size = os.path.getsize(p)
except OSError:
return row["path"], "missing", None, None
if size != row["size"]:
return row["path"], "size_mismatch", size, None
if row["lfs_sha256"]:
h = hashlib.sha256()
kind = "sha256"
else:
h = hashlib.sha1(f"blob {size}\0".encode())
kind = "git_sha1"
with open(p, "rb") as f:
for chunk in iter(lambda: f.read(CHUNK), b""):
h.update(chunk)
digest = h.hexdigest()
expected = row["lfs_sha256"] or row["git_oid"]
return row["path"], ("ok" if digest == expected else f"{kind}_mismatch"), size, digest
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--root", default=os.environ.get("PULSE_ROOT"))
ap.add_argument("--manifest", required=True)
ap.add_argument("--workers", type=int, default=8)
ap.add_argument("--out", default=None, help="JSON report path")
ap.add_argument("--expect-different", action="append", default=[],
help="glob of Hub files allowed to differ (reported separately)")
ap.add_argument("--ignore-extra", action="append", default=[],
help="glob of local-only files to ignore (reported separately)")
args = ap.parse_args(argv)
if not args.root or not os.path.isdir(args.root):
raise SystemExit("--root (or $PULSE_ROOT) must be the dataset root")
header, rows = read_manifest(args.manifest)
t0 = time.time()
with ProcessPoolExecutor(max_workers=args.workers) as ex:
results = list(ex.map(hash_file, [(args.root, r) for r in rows], chunksize=4))
status = {}
for path, st, size, digest in results:
status[path] = {"status": st, "size": size, "digest": digest}
hub = {r["path"] for r in rows}
extra, ignored_extra = [], []
for dp, dns, fns in os.walk(args.root):
dns[:] = [d for d in dns if d != ".git" and d != ".cache"]
for fn in fns:
rel = os.path.relpath(os.path.join(dp, fn), args.root).replace(os.sep, "/")
if rel in hub:
continue
if any(fnmatch.fnmatch(rel, g) for g in args.ignore_extra):
ignored_extra.append(rel)
else:
extra.append(rel)
def allowed(p):
return any(fnmatch.fnmatch(p, g) for g in args.expect_different)
bad = {p: s for p, s in status.items() if s["status"] != "ok" and not allowed(p)}
expected_diff = {p: s for p, s in status.items() if s["status"] != "ok" and allowed(p)}
counts = {}
for s in status.values():
counts[s["status"]] = counts.get(s["status"], 0) + 1
n_bytes = sum(s["size"] or 0 for s in status.values())
report = {
"hub": header, "root_name": os.path.basename(os.path.normpath(args.root)),
"n_hub_files": len(rows), "status_counts": counts,
"n_ok": counts.get("ok", 0), "problems": bad, "expected_differences": expected_diff,
"extra_local_files": sorted(extra), "ignored_extra_local_files": sorted(ignored_extra),
"bytes_checked": n_bytes, "seconds": round(time.time() - t0, 1),
"verdict": "PASS" if not bad and not extra else "FAIL",
}
if args.out:
with open(args.out, "w", encoding="utf-8", newline="\n") as f:
json.dump(report, f, indent=1)
f.write("\n")
print(f"{header.get('repo')}@{header.get('revision')} ({header.get('commit', '')[:10]}): "
f"{len(rows)} files, {n_bytes / 1e9:.2f} GB checked in {report['seconds']} s")
print(f"status: {counts}")
print(f"problems: {len(bad)}; expected differences: {len(expected_diff)} "
f"{sorted(expected_diff)}; extra local files: {len(extra)}; ignored extra: "
f"{len(ignored_extra)}")
for p, s in sorted(bad.items())[:50]:
print(f" {s['status']:16s} {p}")
for p in sorted(extra)[:50]:
print(f" extra {p}")
print(f"verdict: {report['verdict']}")
return 0 if report["verdict"] == "PASS" else 1
if __name__ == "__main__":
sys.exit(main())
|