#!/usr/bin/env python """Fetch the file list of a PULSE dataset revision from the Hugging Face Hub. The Hub API is queried anonymously. For every file the script records the path, the size, the git blob id (sha1 of ``"blob \\0" + content``) and, for files stored with LFS/Xet, the LFS sha256 of the content. The output is a tab-separated table that ``scripts/verify_data.py`` compares against a local copy of the dataset. Usage:: python scripts/fetch_hf_manifest.py --revision v1.1 \ --out manifests/hf_PULSE_v1.1_files.tsv Only the standard library is used, so this runs on any machine with network access to huggingface.co (the compute cluster does not need it). """ from __future__ import annotations import argparse import json import re import sys import urllib.parse import urllib.request HUB = "https://huggingface.co" def _get(url: str): req = urllib.request.Request(url, headers={"User-Agent": "pulse-fetch-manifest/1.0"}) with urllib.request.urlopen(req, timeout=120) as r: return r.read(), r.headers def resolve_commit(repo: str, revision: str) -> str: body, _ = _get(f"{HUB}/api/datasets/{repo}/revision/{urllib.parse.quote(revision, safe='')}") return json.loads(body)["sha"] def list_tree(repo: str, commit: str): url = f"{HUB}/api/datasets/{repo}/tree/{commit}?recursive=true&expand=false" out = [] while url: body, headers = _get(url) out.extend(json.loads(body)) link = headers.get("Link") or "" m = re.search(r'<([^>]+)>;\s*rel="next"', link) url = m.group(1) if m else None return out def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--repo", default="velvet-pine-22/PULSE", help="dataset repo id on the Hub") ap.add_argument("--revision", default="v1.1", help="tag, branch or commit") ap.add_argument("--out", required=True, help="output TSV path") args = ap.parse_args(argv) commit = resolve_commit(args.repo, args.revision) entries = [e for e in list_tree(args.repo, commit) if e.get("type") == "file"] entries.sort(key=lambda e: e["path"]) n_lfs = 0 with open(args.out, "w", encoding="utf-8", newline="\n") as f: f.write(f"# repo={args.repo}\trevision={args.revision}\tcommit={commit}\tn_files={len(entries)}\n") f.write("path\tsize\tgit_oid\tlfs_sha256\n") for e in entries: lfs = e.get("lfs") or {} sha256 = lfs.get("oid", "") n_lfs += bool(sha256) f.write(f"{e['path']}\t{e['size']}\t{e.get('oid', '')}\t{sha256}\n") total = sum(int(e["size"]) for e in entries) print(f"{args.repo}@{args.revision} -> commit {commit}: {len(entries)} files " f"({n_lfs} LFS), {total / 1e9:.2f} GB; wrote {args.out}") return 0 if __name__ == "__main__": sys.exit(main())