File size: 2,142 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
#!/usr/bin/env python
"""Scan every recording of a dataset copy and write the frozen availability table.

    python scripts/build_availability.py --root $PULSE_ROOT --workers 16 \
        --out manifests/availability_v1.1.json

Loads each stream once with the strict loaders (see pulse/data/availability.py
for the status codes).  Exit status 1 if the scan finds data errors (a stream
listed in metadata/recordings.csv without its file, or an ambiguous TSV).
"""
from __future__ import annotations

import argparse
import json
import sys
import time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from pulse.data.availability import build_availability, file_md5  # noqa: E402
from pulse.paths import DataRoot  # noqa: E402


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--root", default=None)
    ap.add_argument("--workers", type=int, default=1)
    ap.add_argument("--rids", default=None, help="comma-separated subset (testing)")
    ap.add_argument("--out", required=True)
    args = ap.parse_args(argv)
    dr = DataRoot.from_arg(args.root)
    t0 = time.time()
    rids = args.rids.split(",") if args.rids else None
    table = build_availability(dr, rids, workers=args.workers)
    table["generated_by"] = "scripts/build_availability.py"
    Path(args.out).parent.mkdir(parents=True, exist_ok=True)
    with open(args.out, "w", encoding="utf-8", newline="\n") as f:
        json.dump(table, f, indent=1, ensure_ascii=False)
        f.write("\n")
    s = table["summary"]
    print(f"scanned {table['n_recordings']} recordings in {time.time() - t0:.0f} s")
    for mod, c in s["modality_status_counts"].items():
        print(f"  {mod:16s} {c}")
    print(f"  mocap TSV locations: {s['mocap_tsv_locations']}")
    print(f"errors ({len(s['errors'])}): {s['errors'][:20]}")
    print(f"warnings ({len(s['warnings'])}): {s['warnings'][:20]}")
    print(f"wrote {args.out} (md5 {file_md5(args.out)})")
    return 1 if s["errors"] else 0


if __name__ == "__main__":
    sys.exit(main())