Download scripts/build_manifests.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 3.21 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/scripts/build_manifests.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/scripts/build_manifests.py
-
curl -L -o build_manifests.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/scripts/build_manifests.py
3.21 kB
| #!/usr/bin/env python | |
| """Freeze the expected recordings of the reference configurations. | |
| python scripts/build_manifests.py # writes manifests/expected_v1.1.json | |
| python scripts/build_manifests.py --check # fail if the committed file is stale | |
| Reads only the split files and the availability table (no data access). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| REPO = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(REPO)) | |
| import yaml # noqa: E402 | |
| from pulse import DATA_TAG # noqa: E402 | |
| from pulse.data.availability import Availability, default_availability_path # noqa: E402 | |
| from pulse.data.manifest import expected_manifest # noqa: E402 | |
| from pulse.splits import load_split # noqa: E402 | |
| def build(config_path: Path, availability_path: Path) -> dict: | |
| av = Availability.load(availability_path) | |
| cfgs = yaml.safe_load(config_path.read_text(encoding="utf-8"))["configs"] | |
| out = {"data_tag": DATA_TAG, "availability_md5": av.md5, "configs": {}} | |
| for c in cfgs: | |
| sp = load_split(REPO / c["split"]) | |
| folds = [] | |
| for k in range(sp.n_folds): | |
| m = expected_manifest(av, sp.fold(k), c["modalities"], c.get("gate", []), | |
| require_annotation=bool(c.get("labelled", False))) | |
| folds.append({"fold": k, "manifest_md5": m.md5, | |
| "counts": {p: len(v) for p, v in m.partitions.items()}, | |
| "recordings": m.partitions, "excluded": m.excluded}) | |
| out["configs"][c["name"]] = { | |
| "split": c["split"], "split_md5": sp.md5, "modalities": c["modalities"], | |
| "gate": c.get("gate", []), "labelled": bool(c.get("labelled", False)), | |
| "totals": {p: sum(f["counts"][p] for f in folds) for p in ("train", "inner_val", "test")}, | |
| "folds": folds} | |
| return out | |
| def main(argv=None) -> int: | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--configs", default=str(REPO / "configs" / "manifests.yaml")) | |
| ap.add_argument("--availability", default=str(default_availability_path(REPO))) | |
| ap.add_argument("--out", default=str(REPO / "manifests" / f"expected_{DATA_TAG}.json")) | |
| ap.add_argument("--check", action="store_true") | |
| args = ap.parse_args(argv) | |
| d = build(Path(args.configs), Path(args.availability)) | |
| text = json.dumps(d, indent=1) + "\n" | |
| if args.check: | |
| same = Path(args.out).is_file() and Path(args.out).read_text(encoding="utf-8") == text | |
| print("up to date" if same else f"{args.out} is stale; rerun scripts/build_manifests.py") | |
| return 0 if same else 1 | |
| Path(args.out).write_text(text, encoding="utf-8", newline="\n") | |
| print(f"{'config':32s} {'test recordings per fold':28s} train/inner_val/test totals") | |
| for name, c in d["configs"].items(): | |
| per = "/".join(str(f["counts"]["test"]) for f in c["folds"]) | |
| t = c["totals"] | |
| print(f"{name:32s} {per:28s} {t['train']}/{t['inner_val']}/{t['test']}") | |
| print(f"wrote {args.out}") | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |