#!/usr/bin/env python """Freeze the expected recordings of the reference configurations. python scripts/build_manifests.py # writes manifests/expected_v1.1.json python scripts/build_manifests.py --check # fail if the committed file is stale Reads only the split files and the availability table (no data access). """ from __future__ import annotations import argparse import json import sys from pathlib import Path REPO = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO)) import yaml # noqa: E402 from pulse import DATA_TAG # noqa: E402 from pulse.data.availability import Availability, default_availability_path # noqa: E402 from pulse.data.manifest import expected_manifest # noqa: E402 from pulse.splits import load_split # noqa: E402 def build(config_path: Path, availability_path: Path) -> dict: av = Availability.load(availability_path) cfgs = yaml.safe_load(config_path.read_text(encoding="utf-8"))["configs"] out = {"data_tag": DATA_TAG, "availability_md5": av.md5, "configs": {}} for c in cfgs: sp = load_split(REPO / c["split"]) folds = [] for k in range(sp.n_folds): m = expected_manifest(av, sp.fold(k), c["modalities"], c.get("gate", []), require_annotation=bool(c.get("labelled", False))) folds.append({"fold": k, "manifest_md5": m.md5, "counts": {p: len(v) for p, v in m.partitions.items()}, "recordings": m.partitions, "excluded": m.excluded}) out["configs"][c["name"]] = { "split": c["split"], "split_md5": sp.md5, "modalities": c["modalities"], "gate": c.get("gate", []), "labelled": bool(c.get("labelled", False)), "totals": {p: sum(f["counts"][p] for f in folds) for p in ("train", "inner_val", "test")}, "folds": folds} return out def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--configs", default=str(REPO / "configs" / "manifests.yaml")) ap.add_argument("--availability", default=str(default_availability_path(REPO))) ap.add_argument("--out", default=str(REPO / "manifests" / f"expected_{DATA_TAG}.json")) ap.add_argument("--check", action="store_true") args = ap.parse_args(argv) d = build(Path(args.configs), Path(args.availability)) text = json.dumps(d, indent=1) + "\n" if args.check: same = Path(args.out).is_file() and Path(args.out).read_text(encoding="utf-8") == text print("up to date" if same else f"{args.out} is stale; rerun scripts/build_manifests.py") return 0 if same else 1 Path(args.out).write_text(text, encoding="utf-8", newline="\n") print(f"{'config':32s} {'test recordings per fold':28s} train/inner_val/test totals") for name, c in d["configs"].items(): per = "/".join(str(f["counts"]["test"]) for f in c["folds"]) t = c["totals"] print(f"{name:32s} {per:28s} {t['train']}/{t['inner_val']}/{t['test']}") print(f"wrote {args.out}") return 0 if __name__ == "__main__": sys.exit(main())