PULSE-code / pulse /tasks /common.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
10.8 kB
"""Shared command line and run context for every ``pulse.tasks.*`` entry point.
Contract (all tasks)::
python -m pulse.tasks.<task> --root <dataset root> --split splits/poolXX.json \
--fold k --seed s --config configs/....yaml [--out DIR] [--set key=value ...]
Typical task body::
def main(argv=None):
args = build_parser("t1", __doc__).parse_args(argv)
run = TaskRun("t1", args)
man = run.manifest(run.config["modalities"])
... load exactly man.partitions[p] for p in train/inner_val/test ...
... select epochs on inner_val only; evaluate test once ...
run.finish(manifest=man, loaded=loaded_ids, metric_defs=..., per_recording=..., ...)
``finish`` refuses to write results.json if the loaded recordings differ from
the manifest or the reported test metrics cannot be recomputed from the
per-recording statistics.
"""
from __future__ import annotations
import argparse
import math
import os
import random
from pathlib import Path
from typing import Any, Dict, Iterable, Optional, Sequence
from .. import DATA_TAG, __version__
from ..config import load_config
from ..data.availability import Availability, default_availability_path
from ..data.manifest import Manifest, expected_manifest, verify_loaded
from ..data.recordings import SCENARIOS_TASK
from ..errors import ProvenanceError
from ..metrics import compute_metrics
from ..paths import DataRoot
from ..provenance import (device_info, git_state, json_md5, now, relpath_or_name,
write_results)
from ..splits import load_split
from ..taxonomy import Taxonomy, default_taxonomy_path
REPO_ROOT = Path(__file__).resolve().parents[2]
def build_parser(task: str, description: Optional[str] = None) -> argparse.ArgumentParser:
ap = argparse.ArgumentParser(prog=f"python -m pulse.tasks.{task}", description=description,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--root", default=None,
help="PULSE dataset root (contains data/, annotations/, metadata/); "
"default $PULSE_ROOT")
ap.add_argument("--split", required=True, help="split file, e.g. splits/pool33.json")
ap.add_argument("--fold", type=int, required=True)
ap.add_argument("--seed", type=int, required=True)
ap.add_argument("--config", required=True, help="YAML config")
ap.add_argument("--set", action="append", default=[], metavar="KEY=VALUE",
help="override a config entry (YAML value), repeatable")
ap.add_argument("--out", default=None,
help="output directory (default runs/<task>/<config>/fold<k>_seed<s>)")
ap.add_argument("--device", default="auto", help="auto | cpu | cuda | cuda:N")
ap.add_argument("--availability", default=None,
help=f"availability table (default manifests/availability_{DATA_TAG}.json)")
ap.add_argument("--taxonomy", default=None,
help=f"label taxonomy (default taxonomy/taxonomy_{DATA_TAG}.json)")
ap.add_argument("--s9-annotations", default=None,
help="directory with vX/s9.json labels (F5); default $PULSE_S9_ANNOTATIONS "
"or <root>/annotations")
ap.add_argument("--num-workers", type=int, default=0)
ap.add_argument("--print-manifest", action="store_true",
help="print the expected manifest and exit without training")
return ap
def set_seed(seed: int, deterministic: bool = True) -> None:
random.seed(seed)
os.environ["PYTHONHASHSEED"] = str(seed)
try:
import numpy as np
np.random.seed(seed)
except ImportError:
pass
try:
import torch
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = bool(deterministic)
except ImportError:
pass
def resolve_device(spec: str = "auto") -> str:
if spec != "auto":
return spec
try:
import torch
return "cuda" if torch.cuda.is_available() else "cpu"
except ImportError:
return "cpu"
class TaskRun:
def __init__(self, task: str, args: argparse.Namespace, needs_taxonomy: bool = False):
self.task = task
self.args = args
self.started_at = now()
# every task: results.json names the commit at the start; finish() refuses if the
# checkout changed while the run executed (launch from a frozen clone)
self.git_start = git_state()
self.dr = DataRoot.from_arg(args.root, args.s9_annotations)
self.split = load_split(args.split)
self.fold = self.split.fold(args.fold)
self.seed = int(args.seed)
self.config_path = args.config
self.config: Dict[str, Any] = load_config(args.config, args.set)
self.config.setdefault("task", task)
if self.config["task"] != task:
raise ValueError(f"config {args.config} is for task {self.config['task']!r}, not {task!r}")
set_seed(self.seed, bool(self.config.get("deterministic", True)))
self.device = resolve_device(args.device)
av_path = args.availability or default_availability_path(REPO_ROOT)
self.availability = Availability.load(av_path)
if self.availability.data_tag != DATA_TAG:
raise ProvenanceError(f"availability table is for {self.availability.data_tag}, "
f"code expects {DATA_TAG}")
errs = self.availability.errors()
if errs:
raise ProvenanceError(f"availability table reports data errors: {errs[:5]}")
self.taxonomy: Optional[Taxonomy] = None
if needs_taxonomy or args.taxonomy:
self.taxonomy = Taxonomy.load(args.taxonomy or default_taxonomy_path())
stem = Path(args.config).stem
self.out_dir = Path(args.out) if args.out else (
REPO_ROOT / "runs" / task / stem / f"fold{self.fold.fold}_seed{self.seed}")
self.manifests: Dict[str, Manifest] = {}
# ------------------------------------------------------------------
def manifest(self, modalities: Sequence[str], gate_modalities: Sequence[str] = (),
scenarios: Sequence[str] = SCENARIOS_TASK, require_annotation: bool = False,
name: str = "main") -> Manifest:
man = expected_manifest(self.availability, self.fold, modalities, gate_modalities,
scenarios, require_annotation)
self.manifests[name] = man
print(f"[manifest:{name}] {man.summary()}", flush=True)
if self.args.print_manifest:
import json
print(json.dumps(man.to_dict(), indent=1))
if not man.partitions["test"]:
raise ProvenanceError(f"empty test partition: {man.summary()}")
return man
def finish(self, *, loaded: Dict[str, Iterable[str]], metric_defs: Dict[str, dict],
per_recording: Dict[str, Dict[str, Dict[str, Any]]],
metrics: Optional[Dict[str, Dict[str, float]]] = None,
manifest: Optional[Manifest] = None, selection: Optional[dict] = None,
stat_kinds: Optional[Dict[str, str]] = None, extra: Optional[dict] = None,
run_name: Optional[str] = None, n_params: Optional[int] = None,
tolerance: float = 1e-9) -> Path:
man = manifest or self.manifests.get("main")
if man is None:
raise ProvenanceError("finish() needs the manifest the run was checked against")
loaded = {k: [str(r) for r in v] for k, v in loaded.items()}
verify_loaded(man, loaded)
payload: Dict[str, Any] = {
"task": self.task,
"run_name": run_name or Path(self.config_path).stem,
"metric_defs": metric_defs,
"stat_kinds": stat_kinds or {},
"per_recording": per_recording,
}
recomputed = compute_metrics(payload | {"per_recording": per_recording})
metrics = dict(metrics or {})
if "test" in metrics:
for k, v in metrics["test"].items():
if k in recomputed and not _close(v, recomputed[k], tolerance):
raise ProvenanceError(f"reported test {k}={v} but per-recording statistics "
f"give {recomputed[k]}")
metrics["test"] = {**metrics.get("test", {}), **recomputed}
payload["metrics"] = metrics
if selection is not None:
payload["selection"] = selection
if n_params is not None:
payload["n_params"] = int(n_params)
tax = self.taxonomy
git_end = git_state()
if git_end != self.git_start:
raise ProvenanceError(
f"the repository changed while this run was executing ({self.git_start} -> "
f"{git_end}); results.json would name code that did not run. Launch runs from a "
"clone nobody commits to.")
payload["provenance"] = {
**git_end,
"code_version": __version__,
"data_tag": DATA_TAG,
"split_file": relpath_or_name(self.split.path),
"split_md5": self.split.md5,
"pool": self.fold.pool,
"fold": self.fold.fold,
"seed": self.seed,
"taxonomy_file": relpath_or_name(tax.path) if tax else None,
"taxonomy_md5": tax.md5 if tax else None,
"availability_md5": self.availability.md5,
"manifest_md5": man.md5,
"manifest_counts": {p: len(v) for p, v in man.partitions.items()},
"manifests": {n: m.md5 for n, m in self.manifests.items()},
"device": device_info(self.device),
"train_volunteers": list(self.fold.train),
"inner_val_volunteers": list(self.fold.inner_val),
"test_volunteers": list(self.fold.test),
"loaded_recordings": loaded,
"config_file": relpath_or_name(self.config_path),
"config": self.config,
"config_md5": json_md5(self.config),
"overrides": list(self.args.set),
"started_at": self.started_at,
"finished_at": now(),
}
if extra:
payload["extra"] = extra
path = write_results(self.out_dir, payload)
print(f"[results] wrote {path}", flush=True)
return path
def _close(a: float, b: float, tol: float) -> bool:
if a is None or b is None:
return a is b
if isinstance(a, float) and math.isnan(a) and isinstance(b, float) and math.isnan(b):
return True
return abs(float(a) - float(b)) <= tol * max(1.0, abs(float(b)))