File size: 5,806 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
#!/usr/bin/env python
"""Verify a local copy of the PULSE dataset against the Hugging Face file list.

For every file in the list (``scripts/fetch_hf_manifest.py`` output), LFS files
are compared by sha256 and plain git files by their git blob id
(sha1 of ``"blob <size>\\0" + content``).  Files present locally but not on the
Hub are reported as extra.  Exit status 1 if anything differs.

    python scripts/verify_data.py --root $PULSE_ROOT \
        --manifest manifests/hf_PULSE_v1.1_files.tsv --workers 16 --out verify_report.json

``--expect-different`` and ``--ignore-extra`` take glob patterns for files that
are known to differ (e.g. a de-anonymised README) or stray local files.
"""
from __future__ import annotations

import argparse
import fnmatch
import hashlib
import json
import os
import sys
import time
from concurrent.futures import ProcessPoolExecutor

CHUNK = 8 << 20


def read_manifest(path):
    header, rows = {}, []
    with open(path, encoding="utf-8") as f:
        for line in f:
            line = line.rstrip("\n")
            if line.startswith("#"):
                for kv in line[1:].strip().split("\t"):
                    if "=" in kv:
                        k, v = kv.split("=", 1)
                        header[k.strip()] = v.strip()
                continue
            if line.startswith("path\t") or not line:
                continue
            p, size, oid, sha = (line.split("\t") + ["", "", "", ""])[:4]
            rows.append({"path": p, "size": int(size), "git_oid": oid, "lfs_sha256": sha})
    return header, rows


def hash_file(args):
    root, row = args
    p = os.path.join(root, row["path"])
    try:
        size = os.path.getsize(p)
    except OSError:
        return row["path"], "missing", None, None
    if size != row["size"]:
        return row["path"], "size_mismatch", size, None
    if row["lfs_sha256"]:
        h = hashlib.sha256()
        kind = "sha256"
    else:
        h = hashlib.sha1(f"blob {size}\0".encode())
        kind = "git_sha1"
    with open(p, "rb") as f:
        for chunk in iter(lambda: f.read(CHUNK), b""):
            h.update(chunk)
    digest = h.hexdigest()
    expected = row["lfs_sha256"] or row["git_oid"]
    return row["path"], ("ok" if digest == expected else f"{kind}_mismatch"), size, digest


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--root", default=os.environ.get("PULSE_ROOT"))
    ap.add_argument("--manifest", required=True)
    ap.add_argument("--workers", type=int, default=8)
    ap.add_argument("--out", default=None, help="JSON report path")
    ap.add_argument("--expect-different", action="append", default=[],
                    help="glob of Hub files allowed to differ (reported separately)")
    ap.add_argument("--ignore-extra", action="append", default=[],
                    help="glob of local-only files to ignore (reported separately)")
    args = ap.parse_args(argv)
    if not args.root or not os.path.isdir(args.root):
        raise SystemExit("--root (or $PULSE_ROOT) must be the dataset root")
    header, rows = read_manifest(args.manifest)
    t0 = time.time()
    with ProcessPoolExecutor(max_workers=args.workers) as ex:
        results = list(ex.map(hash_file, [(args.root, r) for r in rows], chunksize=4))
    status = {}
    for path, st, size, digest in results:
        status[path] = {"status": st, "size": size, "digest": digest}

    hub = {r["path"] for r in rows}
    extra, ignored_extra = [], []
    for dp, dns, fns in os.walk(args.root):
        dns[:] = [d for d in dns if d != ".git" and d != ".cache"]
        for fn in fns:
            rel = os.path.relpath(os.path.join(dp, fn), args.root).replace(os.sep, "/")
            if rel in hub:
                continue
            if any(fnmatch.fnmatch(rel, g) for g in args.ignore_extra):
                ignored_extra.append(rel)
            else:
                extra.append(rel)

    def allowed(p):
        return any(fnmatch.fnmatch(p, g) for g in args.expect_different)

    bad = {p: s for p, s in status.items() if s["status"] != "ok" and not allowed(p)}
    expected_diff = {p: s for p, s in status.items() if s["status"] != "ok" and allowed(p)}
    counts = {}
    for s in status.values():
        counts[s["status"]] = counts.get(s["status"], 0) + 1
    n_bytes = sum(s["size"] or 0 for s in status.values())
    report = {
        "hub": header, "root_name": os.path.basename(os.path.normpath(args.root)),
        "n_hub_files": len(rows), "status_counts": counts,
        "n_ok": counts.get("ok", 0), "problems": bad, "expected_differences": expected_diff,
        "extra_local_files": sorted(extra), "ignored_extra_local_files": sorted(ignored_extra),
        "bytes_checked": n_bytes, "seconds": round(time.time() - t0, 1),
        "verdict": "PASS" if not bad and not extra else "FAIL",
    }
    if args.out:
        with open(args.out, "w", encoding="utf-8", newline="\n") as f:
            json.dump(report, f, indent=1)
            f.write("\n")
    print(f"{header.get('repo')}@{header.get('revision')} ({header.get('commit', '')[:10]}): "
          f"{len(rows)} files, {n_bytes / 1e9:.2f} GB checked in {report['seconds']} s")
    print(f"status: {counts}")
    print(f"problems: {len(bad)}; expected differences: {len(expected_diff)} "
          f"{sorted(expected_diff)}; extra local files: {len(extra)}; ignored extra: "
          f"{len(ignored_extra)}")
    for p, s in sorted(bad.items())[:50]:
        print(f"  {s['status']:16s} {p}")
    for p in sorted(extra)[:50]:
        print(f"  extra            {p}")
    print(f"verdict: {report['verdict']}")
    return 0 if report["verdict"] == "PASS" else 1


if __name__ == "__main__":
    sys.exit(main())