"""Shared command line and run context for every ``pulse.tasks.*`` entry point. Contract (all tasks):: python -m pulse.tasks. --root --split splits/poolXX.json \ --fold k --seed s --config configs/....yaml [--out DIR] [--set key=value ...] Typical task body:: def main(argv=None): args = build_parser("t1", __doc__).parse_args(argv) run = TaskRun("t1", args) man = run.manifest(run.config["modalities"]) ... load exactly man.partitions[p] for p in train/inner_val/test ... ... select epochs on inner_val only; evaluate test once ... run.finish(manifest=man, loaded=loaded_ids, metric_defs=..., per_recording=..., ...) ``finish`` refuses to write results.json if the loaded recordings differ from the manifest or the reported test metrics cannot be recomputed from the per-recording statistics. """ from __future__ import annotations import argparse import math import os import random from pathlib import Path from typing import Any, Dict, Iterable, Optional, Sequence from .. import DATA_TAG, __version__ from ..config import load_config from ..data.availability import Availability, default_availability_path from ..data.manifest import Manifest, expected_manifest, verify_loaded from ..data.recordings import SCENARIOS_TASK from ..errors import ProvenanceError from ..metrics import compute_metrics from ..paths import DataRoot from ..provenance import (device_info, git_state, json_md5, now, relpath_or_name, write_results) from ..splits import load_split from ..taxonomy import Taxonomy, default_taxonomy_path REPO_ROOT = Path(__file__).resolve().parents[2] def build_parser(task: str, description: Optional[str] = None) -> argparse.ArgumentParser: ap = argparse.ArgumentParser(prog=f"python -m pulse.tasks.{task}", description=description, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--root", default=None, help="PULSE dataset root (contains data/, annotations/, metadata/); " "default $PULSE_ROOT") ap.add_argument("--split", required=True, help="split file, e.g. splits/pool33.json") ap.add_argument("--fold", type=int, required=True) ap.add_argument("--seed", type=int, required=True) ap.add_argument("--config", required=True, help="YAML config") ap.add_argument("--set", action="append", default=[], metavar="KEY=VALUE", help="override a config entry (YAML value), repeatable") ap.add_argument("--out", default=None, help="output directory (default runs///fold_seed)") ap.add_argument("--device", default="auto", help="auto | cpu | cuda | cuda:N") ap.add_argument("--availability", default=None, help=f"availability table (default manifests/availability_{DATA_TAG}.json)") ap.add_argument("--taxonomy", default=None, help=f"label taxonomy (default taxonomy/taxonomy_{DATA_TAG}.json)") ap.add_argument("--s9-annotations", default=None, help="directory with vX/s9.json labels (F5); default $PULSE_S9_ANNOTATIONS " "or /annotations") ap.add_argument("--num-workers", type=int, default=0) ap.add_argument("--print-manifest", action="store_true", help="print the expected manifest and exit without training") return ap def set_seed(seed: int, deterministic: bool = True) -> None: random.seed(seed) os.environ["PYTHONHASHSEED"] = str(seed) try: import numpy as np np.random.seed(seed) except ImportError: pass try: import torch torch.manual_seed(seed) if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed) torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = bool(deterministic) except ImportError: pass def resolve_device(spec: str = "auto") -> str: if spec != "auto": return spec try: import torch return "cuda" if torch.cuda.is_available() else "cpu" except ImportError: return "cpu" class TaskRun: def __init__(self, task: str, args: argparse.Namespace, needs_taxonomy: bool = False): self.task = task self.args = args self.started_at = now() # every task: results.json names the commit at the start; finish() refuses if the # checkout changed while the run executed (launch from a frozen clone) self.git_start = git_state() self.dr = DataRoot.from_arg(args.root, args.s9_annotations) self.split = load_split(args.split) self.fold = self.split.fold(args.fold) self.seed = int(args.seed) self.config_path = args.config self.config: Dict[str, Any] = load_config(args.config, args.set) self.config.setdefault("task", task) if self.config["task"] != task: raise ValueError(f"config {args.config} is for task {self.config['task']!r}, not {task!r}") set_seed(self.seed, bool(self.config.get("deterministic", True))) self.device = resolve_device(args.device) av_path = args.availability or default_availability_path(REPO_ROOT) self.availability = Availability.load(av_path) if self.availability.data_tag != DATA_TAG: raise ProvenanceError(f"availability table is for {self.availability.data_tag}, " f"code expects {DATA_TAG}") errs = self.availability.errors() if errs: raise ProvenanceError(f"availability table reports data errors: {errs[:5]}") self.taxonomy: Optional[Taxonomy] = None if needs_taxonomy or args.taxonomy: self.taxonomy = Taxonomy.load(args.taxonomy or default_taxonomy_path()) stem = Path(args.config).stem self.out_dir = Path(args.out) if args.out else ( REPO_ROOT / "runs" / task / stem / f"fold{self.fold.fold}_seed{self.seed}") self.manifests: Dict[str, Manifest] = {} # ------------------------------------------------------------------ def manifest(self, modalities: Sequence[str], gate_modalities: Sequence[str] = (), scenarios: Sequence[str] = SCENARIOS_TASK, require_annotation: bool = False, name: str = "main") -> Manifest: man = expected_manifest(self.availability, self.fold, modalities, gate_modalities, scenarios, require_annotation) self.manifests[name] = man print(f"[manifest:{name}] {man.summary()}", flush=True) if self.args.print_manifest: import json print(json.dumps(man.to_dict(), indent=1)) if not man.partitions["test"]: raise ProvenanceError(f"empty test partition: {man.summary()}") return man def finish(self, *, loaded: Dict[str, Iterable[str]], metric_defs: Dict[str, dict], per_recording: Dict[str, Dict[str, Dict[str, Any]]], metrics: Optional[Dict[str, Dict[str, float]]] = None, manifest: Optional[Manifest] = None, selection: Optional[dict] = None, stat_kinds: Optional[Dict[str, str]] = None, extra: Optional[dict] = None, run_name: Optional[str] = None, n_params: Optional[int] = None, tolerance: float = 1e-9) -> Path: man = manifest or self.manifests.get("main") if man is None: raise ProvenanceError("finish() needs the manifest the run was checked against") loaded = {k: [str(r) for r in v] for k, v in loaded.items()} verify_loaded(man, loaded) payload: Dict[str, Any] = { "task": self.task, "run_name": run_name or Path(self.config_path).stem, "metric_defs": metric_defs, "stat_kinds": stat_kinds or {}, "per_recording": per_recording, } recomputed = compute_metrics(payload | {"per_recording": per_recording}) metrics = dict(metrics or {}) if "test" in metrics: for k, v in metrics["test"].items(): if k in recomputed and not _close(v, recomputed[k], tolerance): raise ProvenanceError(f"reported test {k}={v} but per-recording statistics " f"give {recomputed[k]}") metrics["test"] = {**metrics.get("test", {}), **recomputed} payload["metrics"] = metrics if selection is not None: payload["selection"] = selection if n_params is not None: payload["n_params"] = int(n_params) tax = self.taxonomy git_end = git_state() if git_end != self.git_start: raise ProvenanceError( f"the repository changed while this run was executing ({self.git_start} -> " f"{git_end}); results.json would name code that did not run. Launch runs from a " "clone nobody commits to.") payload["provenance"] = { **git_end, "code_version": __version__, "data_tag": DATA_TAG, "split_file": relpath_or_name(self.split.path), "split_md5": self.split.md5, "pool": self.fold.pool, "fold": self.fold.fold, "seed": self.seed, "taxonomy_file": relpath_or_name(tax.path) if tax else None, "taxonomy_md5": tax.md5 if tax else None, "availability_md5": self.availability.md5, "manifest_md5": man.md5, "manifest_counts": {p: len(v) for p, v in man.partitions.items()}, "manifests": {n: m.md5 for n, m in self.manifests.items()}, "device": device_info(self.device), "train_volunteers": list(self.fold.train), "inner_val_volunteers": list(self.fold.inner_val), "test_volunteers": list(self.fold.test), "loaded_recordings": loaded, "config_file": relpath_or_name(self.config_path), "config": self.config, "config_md5": json_md5(self.config), "overrides": list(self.args.set), "started_at": self.started_at, "finished_at": now(), } if extra: payload["extra"] = extra path = write_results(self.out_dir, payload) print(f"[results] wrote {path}", flush=True) return path def _close(a: float, b: float, tol: float) -> bool: if a is None or b is None: return a is b if isinstance(a, float) and math.isnan(a) and isinstance(b, float) and math.isnan(b): return True return abs(float(a) - float(b)) <= tol * max(1.0, abs(float(b)))