PULSE-code / scripts /build_availability.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
2.14 kB
#!/usr/bin/env python
"""Scan every recording of a dataset copy and write the frozen availability table.
python scripts/build_availability.py --root $PULSE_ROOT --workers 16 \
--out manifests/availability_v1.1.json
Loads each stream once with the strict loaders (see pulse/data/availability.py
for the status codes). Exit status 1 if the scan finds data errors (a stream
listed in metadata/recordings.csv without its file, or an ambiguous TSV).
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from pulse.data.availability import build_availability, file_md5 # noqa: E402
from pulse.paths import DataRoot # noqa: E402
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--root", default=None)
ap.add_argument("--workers", type=int, default=1)
ap.add_argument("--rids", default=None, help="comma-separated subset (testing)")
ap.add_argument("--out", required=True)
args = ap.parse_args(argv)
dr = DataRoot.from_arg(args.root)
t0 = time.time()
rids = args.rids.split(",") if args.rids else None
table = build_availability(dr, rids, workers=args.workers)
table["generated_by"] = "scripts/build_availability.py"
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
with open(args.out, "w", encoding="utf-8", newline="\n") as f:
json.dump(table, f, indent=1, ensure_ascii=False)
f.write("\n")
s = table["summary"]
print(f"scanned {table['n_recordings']} recordings in {time.time() - t0:.0f} s")
for mod, c in s["modality_status_counts"].items():
print(f" {mod:16s} {c}")
print(f" mocap TSV locations: {s['mocap_tsv_locations']}")
print(f"errors ({len(s['errors'])}): {s['errors'][:20]}")
print(f"warnings ({len(s['warnings'])}): {s['warnings'][:20]}")
print(f"wrote {args.out} (md5 {file_md5(args.out)})")
return 1 if s["errors"] else 0
if __name__ == "__main__":
sys.exit(main())