PULSE-code / scripts /verify_data.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
5.81 kB
#!/usr/bin/env python
"""Verify a local copy of the PULSE dataset against the Hugging Face file list.
For every file in the list (``scripts/fetch_hf_manifest.py`` output), LFS files
are compared by sha256 and plain git files by their git blob id
(sha1 of ``"blob <size>\\0" + content``). Files present locally but not on the
Hub are reported as extra. Exit status 1 if anything differs.
python scripts/verify_data.py --root $PULSE_ROOT \
--manifest manifests/hf_PULSE_v1.1_files.tsv --workers 16 --out verify_report.json
``--expect-different`` and ``--ignore-extra`` take glob patterns for files that
are known to differ (e.g. a de-anonymised README) or stray local files.
"""
from __future__ import annotations
import argparse
import fnmatch
import hashlib
import json
import os
import sys
import time
from concurrent.futures import ProcessPoolExecutor
CHUNK = 8 << 20
def read_manifest(path):
header, rows = {}, []
with open(path, encoding="utf-8") as f:
for line in f:
line = line.rstrip("\n")
if line.startswith("#"):
for kv in line[1:].strip().split("\t"):
if "=" in kv:
k, v = kv.split("=", 1)
header[k.strip()] = v.strip()
continue
if line.startswith("path\t") or not line:
continue
p, size, oid, sha = (line.split("\t") + ["", "", "", ""])[:4]
rows.append({"path": p, "size": int(size), "git_oid": oid, "lfs_sha256": sha})
return header, rows
def hash_file(args):
root, row = args
p = os.path.join(root, row["path"])
try:
size = os.path.getsize(p)
except OSError:
return row["path"], "missing", None, None
if size != row["size"]:
return row["path"], "size_mismatch", size, None
if row["lfs_sha256"]:
h = hashlib.sha256()
kind = "sha256"
else:
h = hashlib.sha1(f"blob {size}\0".encode())
kind = "git_sha1"
with open(p, "rb") as f:
for chunk in iter(lambda: f.read(CHUNK), b""):
h.update(chunk)
digest = h.hexdigest()
expected = row["lfs_sha256"] or row["git_oid"]
return row["path"], ("ok" if digest == expected else f"{kind}_mismatch"), size, digest
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--root", default=os.environ.get("PULSE_ROOT"))
ap.add_argument("--manifest", required=True)
ap.add_argument("--workers", type=int, default=8)
ap.add_argument("--out", default=None, help="JSON report path")
ap.add_argument("--expect-different", action="append", default=[],
help="glob of Hub files allowed to differ (reported separately)")
ap.add_argument("--ignore-extra", action="append", default=[],
help="glob of local-only files to ignore (reported separately)")
args = ap.parse_args(argv)
if not args.root or not os.path.isdir(args.root):
raise SystemExit("--root (or $PULSE_ROOT) must be the dataset root")
header, rows = read_manifest(args.manifest)
t0 = time.time()
with ProcessPoolExecutor(max_workers=args.workers) as ex:
results = list(ex.map(hash_file, [(args.root, r) for r in rows], chunksize=4))
status = {}
for path, st, size, digest in results:
status[path] = {"status": st, "size": size, "digest": digest}
hub = {r["path"] for r in rows}
extra, ignored_extra = [], []
for dp, dns, fns in os.walk(args.root):
dns[:] = [d for d in dns if d != ".git" and d != ".cache"]
for fn in fns:
rel = os.path.relpath(os.path.join(dp, fn), args.root).replace(os.sep, "/")
if rel in hub:
continue
if any(fnmatch.fnmatch(rel, g) for g in args.ignore_extra):
ignored_extra.append(rel)
else:
extra.append(rel)
def allowed(p):
return any(fnmatch.fnmatch(p, g) for g in args.expect_different)
bad = {p: s for p, s in status.items() if s["status"] != "ok" and not allowed(p)}
expected_diff = {p: s for p, s in status.items() if s["status"] != "ok" and allowed(p)}
counts = {}
for s in status.values():
counts[s["status"]] = counts.get(s["status"], 0) + 1
n_bytes = sum(s["size"] or 0 for s in status.values())
report = {
"hub": header, "root_name": os.path.basename(os.path.normpath(args.root)),
"n_hub_files": len(rows), "status_counts": counts,
"n_ok": counts.get("ok", 0), "problems": bad, "expected_differences": expected_diff,
"extra_local_files": sorted(extra), "ignored_extra_local_files": sorted(ignored_extra),
"bytes_checked": n_bytes, "seconds": round(time.time() - t0, 1),
"verdict": "PASS" if not bad and not extra else "FAIL",
}
if args.out:
with open(args.out, "w", encoding="utf-8", newline="\n") as f:
json.dump(report, f, indent=1)
f.write("\n")
print(f"{header.get('repo')}@{header.get('revision')} ({header.get('commit', '')[:10]}): "
f"{len(rows)} files, {n_bytes / 1e9:.2f} GB checked in {report['seconds']} s")
print(f"status: {counts}")
print(f"problems: {len(bad)}; expected differences: {len(expected_diff)} "
f"{sorted(expected_diff)}; extra local files: {len(extra)}; ignored extra: "
f"{len(ignored_extra)}")
for p, s in sorted(bad.items())[:50]:
print(f" {s['status']:16s} {p}")
for p in sorted(extra)[:50]:
print(f" extra {p}")
print(f"verdict: {report['verdict']}")
return 0 if report["verdict"] == "PASS" else 1
if __name__ == "__main__":
sys.exit(main())