Download scripts/build_availability.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 2.14 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/scripts/build_availability.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/scripts/build_availability.py
-
curl -L -o build_availability.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/scripts/build_availability.py
2.14 kB
| #!/usr/bin/env python | |
| """Scan every recording of a dataset copy and write the frozen availability table. | |
| python scripts/build_availability.py --root $PULSE_ROOT --workers 16 \ | |
| --out manifests/availability_v1.1.json | |
| Loads each stream once with the strict loaders (see pulse/data/availability.py | |
| for the status codes). Exit status 1 if the scan finds data errors (a stream | |
| listed in metadata/recordings.csv without its file, or an ambiguous TSV). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from pulse.data.availability import build_availability, file_md5 # noqa: E402 | |
| from pulse.paths import DataRoot # noqa: E402 | |
| def main(argv=None) -> int: | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--root", default=None) | |
| ap.add_argument("--workers", type=int, default=1) | |
| ap.add_argument("--rids", default=None, help="comma-separated subset (testing)") | |
| ap.add_argument("--out", required=True) | |
| args = ap.parse_args(argv) | |
| dr = DataRoot.from_arg(args.root) | |
| t0 = time.time() | |
| rids = args.rids.split(",") if args.rids else None | |
| table = build_availability(dr, rids, workers=args.workers) | |
| table["generated_by"] = "scripts/build_availability.py" | |
| Path(args.out).parent.mkdir(parents=True, exist_ok=True) | |
| with open(args.out, "w", encoding="utf-8", newline="\n") as f: | |
| json.dump(table, f, indent=1, ensure_ascii=False) | |
| f.write("\n") | |
| s = table["summary"] | |
| print(f"scanned {table['n_recordings']} recordings in {time.time() - t0:.0f} s") | |
| for mod, c in s["modality_status_counts"].items(): | |
| print(f" {mod:16s} {c}") | |
| print(f" mocap TSV locations: {s['mocap_tsv_locations']}") | |
| print(f"errors ({len(s['errors'])}): {s['errors'][:20]}") | |
| print(f"warnings ({len(s['warnings'])}): {s['warnings'][:20]}") | |
| print(f"wrote {args.out} (md5 {file_md5(args.out)})") | |
| return 1 if s["errors"] else 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |