Download scripts/verify_data.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 5.81 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/scripts/verify_data.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/scripts/verify_data.py
-
curl -L -o verify_data.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/scripts/verify_data.py
5.81 kB
| #!/usr/bin/env python | |
| """Verify a local copy of the PULSE dataset against the Hugging Face file list. | |
| For every file in the list (``scripts/fetch_hf_manifest.py`` output), LFS files | |
| are compared by sha256 and plain git files by their git blob id | |
| (sha1 of ``"blob <size>\\0" + content``). Files present locally but not on the | |
| Hub are reported as extra. Exit status 1 if anything differs. | |
| python scripts/verify_data.py --root $PULSE_ROOT \ | |
| --manifest manifests/hf_PULSE_v1.1_files.tsv --workers 16 --out verify_report.json | |
| ``--expect-different`` and ``--ignore-extra`` take glob patterns for files that | |
| are known to differ (e.g. a de-anonymised README) or stray local files. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import fnmatch | |
| import hashlib | |
| import json | |
| import os | |
| import sys | |
| import time | |
| from concurrent.futures import ProcessPoolExecutor | |
| CHUNK = 8 << 20 | |
| def read_manifest(path): | |
| header, rows = {}, [] | |
| with open(path, encoding="utf-8") as f: | |
| for line in f: | |
| line = line.rstrip("\n") | |
| if line.startswith("#"): | |
| for kv in line[1:].strip().split("\t"): | |
| if "=" in kv: | |
| k, v = kv.split("=", 1) | |
| header[k.strip()] = v.strip() | |
| continue | |
| if line.startswith("path\t") or not line: | |
| continue | |
| p, size, oid, sha = (line.split("\t") + ["", "", "", ""])[:4] | |
| rows.append({"path": p, "size": int(size), "git_oid": oid, "lfs_sha256": sha}) | |
| return header, rows | |
| def hash_file(args): | |
| root, row = args | |
| p = os.path.join(root, row["path"]) | |
| try: | |
| size = os.path.getsize(p) | |
| except OSError: | |
| return row["path"], "missing", None, None | |
| if size != row["size"]: | |
| return row["path"], "size_mismatch", size, None | |
| if row["lfs_sha256"]: | |
| h = hashlib.sha256() | |
| kind = "sha256" | |
| else: | |
| h = hashlib.sha1(f"blob {size}\0".encode()) | |
| kind = "git_sha1" | |
| with open(p, "rb") as f: | |
| for chunk in iter(lambda: f.read(CHUNK), b""): | |
| h.update(chunk) | |
| digest = h.hexdigest() | |
| expected = row["lfs_sha256"] or row["git_oid"] | |
| return row["path"], ("ok" if digest == expected else f"{kind}_mismatch"), size, digest | |
| def main(argv=None) -> int: | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--root", default=os.environ.get("PULSE_ROOT")) | |
| ap.add_argument("--manifest", required=True) | |
| ap.add_argument("--workers", type=int, default=8) | |
| ap.add_argument("--out", default=None, help="JSON report path") | |
| ap.add_argument("--expect-different", action="append", default=[], | |
| help="glob of Hub files allowed to differ (reported separately)") | |
| ap.add_argument("--ignore-extra", action="append", default=[], | |
| help="glob of local-only files to ignore (reported separately)") | |
| args = ap.parse_args(argv) | |
| if not args.root or not os.path.isdir(args.root): | |
| raise SystemExit("--root (or $PULSE_ROOT) must be the dataset root") | |
| header, rows = read_manifest(args.manifest) | |
| t0 = time.time() | |
| with ProcessPoolExecutor(max_workers=args.workers) as ex: | |
| results = list(ex.map(hash_file, [(args.root, r) for r in rows], chunksize=4)) | |
| status = {} | |
| for path, st, size, digest in results: | |
| status[path] = {"status": st, "size": size, "digest": digest} | |
| hub = {r["path"] for r in rows} | |
| extra, ignored_extra = [], [] | |
| for dp, dns, fns in os.walk(args.root): | |
| dns[:] = [d for d in dns if d != ".git" and d != ".cache"] | |
| for fn in fns: | |
| rel = os.path.relpath(os.path.join(dp, fn), args.root).replace(os.sep, "/") | |
| if rel in hub: | |
| continue | |
| if any(fnmatch.fnmatch(rel, g) for g in args.ignore_extra): | |
| ignored_extra.append(rel) | |
| else: | |
| extra.append(rel) | |
| def allowed(p): | |
| return any(fnmatch.fnmatch(p, g) for g in args.expect_different) | |
| bad = {p: s for p, s in status.items() if s["status"] != "ok" and not allowed(p)} | |
| expected_diff = {p: s for p, s in status.items() if s["status"] != "ok" and allowed(p)} | |
| counts = {} | |
| for s in status.values(): | |
| counts[s["status"]] = counts.get(s["status"], 0) + 1 | |
| n_bytes = sum(s["size"] or 0 for s in status.values()) | |
| report = { | |
| "hub": header, "root_name": os.path.basename(os.path.normpath(args.root)), | |
| "n_hub_files": len(rows), "status_counts": counts, | |
| "n_ok": counts.get("ok", 0), "problems": bad, "expected_differences": expected_diff, | |
| "extra_local_files": sorted(extra), "ignored_extra_local_files": sorted(ignored_extra), | |
| "bytes_checked": n_bytes, "seconds": round(time.time() - t0, 1), | |
| "verdict": "PASS" if not bad and not extra else "FAIL", | |
| } | |
| if args.out: | |
| with open(args.out, "w", encoding="utf-8", newline="\n") as f: | |
| json.dump(report, f, indent=1) | |
| f.write("\n") | |
| print(f"{header.get('repo')}@{header.get('revision')} ({header.get('commit', '')[:10]}): " | |
| f"{len(rows)} files, {n_bytes / 1e9:.2f} GB checked in {report['seconds']} s") | |
| print(f"status: {counts}") | |
| print(f"problems: {len(bad)}; expected differences: {len(expected_diff)} " | |
| f"{sorted(expected_diff)}; extra local files: {len(extra)}; ignored extra: " | |
| f"{len(ignored_extra)}") | |
| for p, s in sorted(bad.items())[:50]: | |
| print(f" {s['status']:16s} {p}") | |
| for p in sorted(extra)[:50]: | |
| print(f" extra {p}") | |
| print(f"verdict: {report['verdict']}") | |
| return 0 if report["verdict"] == "PASS" else 1 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |