Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / src /wisp /recipes.py
Zipeng365's picture
wisp-har 0.3.0: scikit-learn interface, recipe catalog, Optuna search, cost-accuracy figures
4731b11 verified
Raw History Blame Contribute Delete
12.7 kB
"""Recipe descriptions and the catalog of recipes that have saved results.
A :class:`Recipe` is an immutable description of one WISP configuration. Its dictionary
form is the ``canonical_recipe`` format of ``results/search_selections.csv``::
{"name": "state_rocket_interval_hmm",
"params": {"classifier": "ridge", "n_intervals": 24, "n_kernels": 384, "use_hmm": true},
"route": "cpu"}
The catalog (``list``/``get``/``best``) only contains recipes with rows in the saved
result files: the seven fixed families, the six CIS block subsets (plus ``"cis:RIS"``,
which is ``"cis"``) and one search best recipe per data stream. ``search_runs()`` exposes
the 360 per-search final recipes for checking where ``best`` comes from; they are not
catalog entries.
"""
from __future__ import annotations
import csv
import json
from functools import lru_cache
from importlib import resources
from typing import Any
__all__ = ["Recipe", "list", "get", "best", "search_runs", "resolve", "dataset_id", "FAMILIES"]
_list = list # the module defines ``list()``; keep the builtin reachable
#: The seven kernel family names (``name`` in the canonical recipe format).
FAMILIES = ("har_sculpt_forest_hmm", "rocket_sketch_ridge_hmm", "symbolic_interval_forest",
"hybrid_cpu_ensemble", "invariant_interval_shape_forest", "state_rocket_interval_hmm",
"posterior_cpu_ensemble")
#: Families whose recipe has one feature vector and one readout.
SINGLE_READOUT = ("har_sculpt_forest_hmm", "rocket_sketch_ridge_hmm", "symbolic_interval_forest",
"invariant_interval_shape_forest", "state_rocket_interval_hmm")
_CODE_CLASS_TO_FAMILY = {
"HarSculptForestHMM": "har_sculpt_forest_hmm", "RocketSketchRidgeHMM": "rocket_sketch_ridge_hmm",
"SymbolicIntervalForest": "symbolic_interval_forest", "HybridCPUEnsemble": "hybrid_cpu_ensemble",
"InvariantIntervalShapeForest": "invariant_interval_shape_forest",
"StateRocketIntervalHMM": "state_rocket_interval_hmm", "PosteriorCPUEnsemble": "posterior_cpu_ensemble",
}
_FIXED_SHORT = {"wisp_ss": "ss", "wisp_rc": "rc", "wisp_so": "so", "wisp_cse": "cse", "wisp_gis": "gis",
"wisp_cis": "cis", "wisp_ese": "ese"}
_RUNTIME_KEYS = ("seed", "n_jobs")
def _freeze(value: Any) -> Any:
"""Top-level dict -> sorted item tuple; nested lists -> tuples (kernel convention)."""
def seq(v):
if isinstance(v, (_list, tuple)):
return tuple(seq(x) for x in v)
if hasattr(v, "item") and not isinstance(v, (str, bytes)):
return v.item()
return v
return tuple(sorted((k, seq(v)) for k, v in value.items()))
def _jsonable(value: Any) -> Any:
if isinstance(value, tuple):
return [_jsonable(v) for v in value]
if isinstance(value, dict):
return {k: _jsonable(v) for k, v in value.items()}
return value
class Recipe:
"""Immutable recipe: kernel family plus parameters, with catalog metadata.
``Recipe("state_rocket_interval_hmm", {"n_kernels": 128, "use_hmm": True})`` makes a
custom recipe. ``params`` never contains the runtime keys ``seed`` or ``n_jobs``; the
estimator's ``random_state`` and ``n_jobs`` supply them.
"""
__slots__ = ("family", "_params", "id", "paper_name", "source", "_provenance")
def __init__(self, family: str, params: dict | None = None, *, id: str | None = None,
paper_name: str | None = None, source: str = "custom", provenance: dict | None = None):
if family not in FAMILIES:
raise ValueError(f"Unknown WISP family {family!r}; expected one of {', '.join(FAMILIES)}")
clean = {str(k): v for k, v in dict(params or {}).items() if k not in _RUNTIME_KEYS}
for name, value in (("family", str(family)), ("_params", _freeze(clean)), ("id", id),
("paper_name", paper_name), ("source", str(source)),
("_provenance", json.dumps(provenance or {}, sort_keys=True, default=str))):
object.__setattr__(self, name, value)
def __setattr__(self, name, value):
raise AttributeError("Recipe is immutable; use with_params() to derive a custom recipe")
def __reduce__(self):
return (_rebuild, (self.family, self.params, self.id, self.paper_name, self.source, self.provenance))
def _key(self):
return (self.family, self._params, self.id, self.source)
def __eq__(self, other):
return isinstance(other, Recipe) and self._key() == other._key()
def __hash__(self):
return hash(self._key())
def __repr__(self):
label = f"id={self.id!r}, " if self.id else ""
return f"Recipe({label}family={self.family!r}, params={self.to_dict()['params']!r}, source={self.source!r})"
# ---- views ----
@property
def params(self) -> dict[str, Any]:
"""Parameters passed to the kernel family (sequences as tuples); a fresh copy."""
return {k: v for k, v in self._params}
@property
def provenance(self) -> dict[str, Any]:
return json.loads(self._provenance)
@property
def use_hmm(self) -> bool:
return bool(self.params.get("use_hmm", True))
@property
def single_readout(self) -> bool:
return self.family in SINGLE_READOUT
@property
def in_catalog(self) -> bool:
return self.source in ("fixed", "cis_subset", "search_best")
# ---- canonical dictionary / JSON ----
def to_dict(self) -> dict[str, Any]:
"""Canonical ``{"name", "params", "route"}`` form used by ``search_selections.csv``."""
return {"name": self.family, "params": _jsonable(dict(sorted(self.params.items()))), "route": "cpu"}
@classmethod
def from_dict(cls, d: dict[str, Any]) -> "Recipe":
if not isinstance(d, dict) or "name" not in d:
raise ValueError("A recipe dictionary needs at least 'name' (and usually 'params')")
if d.get("route", "cpu") != "cpu":
raise ValueError("Only CPU recipes exist in this package")
return cls(str(d["name"]), d.get("params", {}))
def to_json(self) -> str:
return json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":"))
@classmethod
def from_json(cls, text: str) -> "Recipe":
return cls.from_dict(json.loads(text))
def with_params(self, **changes) -> "Recipe":
"""A custom (non-catalog) copy with some parameters changed."""
return Recipe(self.family, {**self.params, **changes})
def kernel_spec(self, *, seed: int, n_jobs: int) -> dict[str, Any]:
"""Spec for ``build_cpu_estimator`` with the runtime keys added."""
params = self.params
params.update(seed=int(seed), n_jobs=int(n_jobs))
return {"name": self.family, "params": params, "route": "cpu"}
def describe(self) -> dict[str, Any]:
return {"id": self.id, "paper_name": self.paper_name, "source": self.source, **self.to_dict(),
"provenance": self.provenance}
def _rebuild(family, params, id, paper_name, source, provenance):
return Recipe(family, params, id=id, paper_name=paper_name, source=source, provenance=provenance)
# --------------------------------------------------------------------------- catalog
def _data_text(name: str) -> str:
return resources.files("wisp").joinpath("_data", name).read_text(encoding="utf-8")
@lru_cache(maxsize=1)
def _aliases() -> dict[str, str]:
return json.loads(_data_text("dataset_aliases.json"))
def dataset_id(name: str) -> str:
"""Map a short stream name (``"capture24"``) or full ID to the full dataset ID."""
ids = set(_aliases().values())
key = str(name).strip().lower().replace("-", "_")
if key in ids:
return key
if key in _aliases():
return _aliases()[key]
raise KeyError(f"Unknown data stream {name!r}; expected one of {sorted(_aliases())} or {sorted(ids)}")
@lru_cache(maxsize=1)
def _catalog() -> dict[str, Recipe]:
from wisp.ablations import CIS_ABLATIONS
from wisp.registry import list_specs
out: dict[str, Recipe] = {}
for spec in list_specs():
rid = _FIXED_SHORT[spec.method_id]
out[rid] = Recipe(_CODE_CLASS_TO_FAMILY[spec.code_class], dict(spec.params), id=rid,
paper_name=spec.paper_name, source="fixed",
provenance={"method_id": spec.method_id, "results": "results/run_results.csv",
"paper_ids": {"hmm": spec.method_id,
**({"none": "cis_ablation_RIS_direct"}
if spec.method_id == "wisp_cis" else {})}})
for spec in CIS_ABLATIONS:
if not spec.use_hmm or spec.component_tag == "RIS":
continue
cand = spec.to_candidate(seed=0, n_jobs=1)
rid = f"cis:{spec.component_tag}"
out[rid] = Recipe(cand.name, cand.params, id=rid, paper_name=f"WISP-CIS block subset {spec.component_tag}",
source="cis_subset",
provenance={"component_tag": spec.component_tag,
"results": "results/cis_and_budget_runs.csv",
"paper_ids": {"hmm": f"cis_ablation_{spec.component_tag}_viterbi",
"none": f"cis_ablation_{spec.component_tag}_direct"}})
for row in json.loads(_data_text("search_best.json")):
out[row["id"]] = Recipe(row["family"], row["params"], id=row["id"], paper_name=row["paper_name"],
source="search_best", provenance=row["provenance"])
return out
def get(name: str) -> Recipe:
"""Catalog recipe by name: ``"cis"``, ``"cis:RS"``, ``"best:capture24"`` (``"cis:RIS"`` is ``"cis"``)."""
key = str(name).strip()
low = key.lower()
if low.startswith("best:"):
return best(key.split(":", 1)[1])
if low.startswith("cis:"):
tags = key.split(":", 1)[1].strip().upper()
if tags and not set(tags) - set("RIS") and len(set(tags)) == len(tags):
tags = "".join(c for c in "RIS" if c in tags)
low = "cis" if tags == "RIS" else f"cis:{tags}"
try:
return _catalog()[low]
except KeyError:
raise KeyError(f"Unknown recipe {name!r}. Catalog: {', '.join(sorted(_catalog()))}") from None
def best(stream: str) -> Recipe:
"""The search's best recipe on one data stream (see docs/RECIPES.md for the rule)."""
full = dataset_id(stream)
for recipe in _catalog().values():
if recipe.source == "search_best" and recipe.provenance.get("dataset_id") == full:
return recipe
raise KeyError(f"No search best recipe for {stream!r}")
def list(source: str | None = None):
"""Table of all catalog recipes (a pandas DataFrame when pandas is installed, else a list of dicts)."""
rows = []
for rid, r in _catalog().items():
if source is not None and r.source != source:
continue
prov = r.provenance
rows.append({"id": rid, "paper_name": r.paper_name, "source": r.source, "family": r.family,
"params": json.dumps(r.to_dict()["params"], sort_keys=True), "use_hmm": r.use_hmm,
"single_readout": r.single_readout,
"results": prov.get("results", prov.get("source_file")),
"selected": prov.get("selected"), "folds_covered": prov.get("folds_covered"),
"same_as": prov.get("same_as")})
try:
import pandas as pd
except ImportError:
return rows
return pd.DataFrame(rows)
def search_runs():
"""The 360 final recipes of the budget-329 searches (read-only; for checking ``best``)."""
rows = []
reader = csv.DictReader(_data_text("search_selections.csv").splitlines())
for row in reader:
if int(float(row["budget"])) == 329:
rows.append({k: row[k] for k in ("method_id", "dataset_id", "fold", "model_seed", "candidate_id",
"family", "use_hmm", "canonical_recipe")})
try:
import pandas as pd
except ImportError:
return rows
return pd.DataFrame(rows)
def resolve(recipe: Any) -> Recipe:
"""Accept a catalog name, a :class:`Recipe` or a canonical-format dictionary."""
if isinstance(recipe, Recipe):
return recipe
if isinstance(recipe, str):
return get(recipe)
if isinstance(recipe, dict):
return Recipe.from_dict(recipe)
raise TypeError(f"recipe must be a catalog name, a Recipe or a dict; got {type(recipe).__name__}")