Download scripts/fetch_hf_manifest.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 2.93 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/scripts/fetch_hf_manifest.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/scripts/fetch_hf_manifest.py
-
curl -L -o fetch_hf_manifest.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/scripts/fetch_hf_manifest.py
2.93 kB
| #!/usr/bin/env python | |
| """Fetch the file list of a PULSE dataset revision from the Hugging Face Hub. | |
| The Hub API is queried anonymously. For every file the script records the | |
| path, the size, the git blob id (sha1 of ``"blob <size>\\0" + content``) and, | |
| for files stored with LFS/Xet, the LFS sha256 of the content. The output is a | |
| tab-separated table that ``scripts/verify_data.py`` compares against a local | |
| copy of the dataset. | |
| Usage:: | |
| python scripts/fetch_hf_manifest.py --revision v1.1 \ | |
| --out manifests/hf_PULSE_v1.1_files.tsv | |
| Only the standard library is used, so this runs on any machine with network | |
| access to huggingface.co (the compute cluster does not need it). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import re | |
| import sys | |
| import urllib.parse | |
| import urllib.request | |
| HUB = "https://huggingface.co" | |
| def _get(url: str): | |
| req = urllib.request.Request(url, headers={"User-Agent": "pulse-fetch-manifest/1.0"}) | |
| with urllib.request.urlopen(req, timeout=120) as r: | |
| return r.read(), r.headers | |
| def resolve_commit(repo: str, revision: str) -> str: | |
| body, _ = _get(f"{HUB}/api/datasets/{repo}/revision/{urllib.parse.quote(revision, safe='')}") | |
| return json.loads(body)["sha"] | |
| def list_tree(repo: str, commit: str): | |
| url = f"{HUB}/api/datasets/{repo}/tree/{commit}?recursive=true&expand=false" | |
| out = [] | |
| while url: | |
| body, headers = _get(url) | |
| out.extend(json.loads(body)) | |
| link = headers.get("Link") or "" | |
| m = re.search(r'<([^>]+)>;\s*rel="next"', link) | |
| url = m.group(1) if m else None | |
| return out | |
| def main(argv=None) -> int: | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--repo", default="velvet-pine-22/PULSE", help="dataset repo id on the Hub") | |
| ap.add_argument("--revision", default="v1.1", help="tag, branch or commit") | |
| ap.add_argument("--out", required=True, help="output TSV path") | |
| args = ap.parse_args(argv) | |
| commit = resolve_commit(args.repo, args.revision) | |
| entries = [e for e in list_tree(args.repo, commit) if e.get("type") == "file"] | |
| entries.sort(key=lambda e: e["path"]) | |
| n_lfs = 0 | |
| with open(args.out, "w", encoding="utf-8", newline="\n") as f: | |
| f.write(f"# repo={args.repo}\trevision={args.revision}\tcommit={commit}\tn_files={len(entries)}\n") | |
| f.write("path\tsize\tgit_oid\tlfs_sha256\n") | |
| for e in entries: | |
| lfs = e.get("lfs") or {} | |
| sha256 = lfs.get("oid", "") | |
| n_lfs += bool(sha256) | |
| f.write(f"{e['path']}\t{e['size']}\t{e.get('oid', '')}\t{sha256}\n") | |
| total = sum(int(e["size"]) for e in entries) | |
| print(f"{args.repo}@{args.revision} -> commit {commit}: {len(entries)} files " | |
| f"({n_lfs} LFS), {total / 1e9:.2f} GB; wrote {args.out}") | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |