PULSE-code / scripts /fetch_hf_manifest.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
2.93 kB
#!/usr/bin/env python
"""Fetch the file list of a PULSE dataset revision from the Hugging Face Hub.
The Hub API is queried anonymously. For every file the script records the
path, the size, the git blob id (sha1 of ``"blob <size>\\0" + content``) and,
for files stored with LFS/Xet, the LFS sha256 of the content. The output is a
tab-separated table that ``scripts/verify_data.py`` compares against a local
copy of the dataset.
Usage::
python scripts/fetch_hf_manifest.py --revision v1.1 \
--out manifests/hf_PULSE_v1.1_files.tsv
Only the standard library is used, so this runs on any machine with network
access to huggingface.co (the compute cluster does not need it).
"""
from __future__ import annotations
import argparse
import json
import re
import sys
import urllib.parse
import urllib.request
HUB = "https://huggingface.co"
def _get(url: str):
req = urllib.request.Request(url, headers={"User-Agent": "pulse-fetch-manifest/1.0"})
with urllib.request.urlopen(req, timeout=120) as r:
return r.read(), r.headers
def resolve_commit(repo: str, revision: str) -> str:
body, _ = _get(f"{HUB}/api/datasets/{repo}/revision/{urllib.parse.quote(revision, safe='')}")
return json.loads(body)["sha"]
def list_tree(repo: str, commit: str):
url = f"{HUB}/api/datasets/{repo}/tree/{commit}?recursive=true&expand=false"
out = []
while url:
body, headers = _get(url)
out.extend(json.loads(body))
link = headers.get("Link") or ""
m = re.search(r'<([^>]+)>;\s*rel="next"', link)
url = m.group(1) if m else None
return out
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--repo", default="velvet-pine-22/PULSE", help="dataset repo id on the Hub")
ap.add_argument("--revision", default="v1.1", help="tag, branch or commit")
ap.add_argument("--out", required=True, help="output TSV path")
args = ap.parse_args(argv)
commit = resolve_commit(args.repo, args.revision)
entries = [e for e in list_tree(args.repo, commit) if e.get("type") == "file"]
entries.sort(key=lambda e: e["path"])
n_lfs = 0
with open(args.out, "w", encoding="utf-8", newline="\n") as f:
f.write(f"# repo={args.repo}\trevision={args.revision}\tcommit={commit}\tn_files={len(entries)}\n")
f.write("path\tsize\tgit_oid\tlfs_sha256\n")
for e in entries:
lfs = e.get("lfs") or {}
sha256 = lfs.get("oid", "")
n_lfs += bool(sha256)
f.write(f"{e['path']}\t{e['size']}\t{e.get('oid', '')}\t{sha256}\n")
total = sum(int(e["size"]) for e in entries)
print(f"{args.repo}@{args.revision} -> commit {commit}: {len(entries)} files "
f"({n_lfs} LFS), {total / 1e9:.2f} GB; wrote {args.out}")
return 0
if __name__ == "__main__":
sys.exit(main())