"""Generate src/wisp/_data/search_best.json from results/search_selections.csv. Rule (validation only; no test metric is read): for each data stream take the 30 final recipes of the budget-329 searches (5 folds x 3 seeds x WISP-Random/WISP-Evolution). Each final recipe is the candidate with the highest validation utility in its search. The stream's best recipe is the recipe selected most often; ties are broken by the number of folds it covers, then by the canonical JSON text (deterministic, never needed for the released files). Only ``canonical_recipe``, ``dataset_id``, ``fold``, ``model_seed``, ``method_id`` and ``budget`` are read. Usage: python scripts/build_search_best.py [--check] """ from __future__ import annotations import argparse import csv import json from collections import defaultdict from pathlib import Path ROOT = Path(__file__).resolve().parents[1] SOURCE = ROOT / "results" / "search_selections.csv" TARGET = ROOT / "src" / "wisp" / "_data" / "search_best.json" ALIASES = ROOT / "src" / "wisp" / "_data" / "dataset_aliases.json" FINAL_BUDGET = 329 # Canonical forms of catalog recipes, for the "same as" column. FIXED_CANONICAL = { "ss": {"name": "har_sculpt_forest_hmm", "params": {"classifier": "extratrees", "segment_sizes": [4, 8, 16], "symbolic_bins": 6, "use_hmm": True}}, "rc": {"name": "rocket_sketch_ridge_hmm", "params": {"classifier": "ridge", "n_kernels": 384, "use_hmm": True}}, "so": {"name": "symbolic_interval_forest", "params": {"use_hmm": True}}, "cse": {"name": "hybrid_cpu_ensemble", "params": {"use_hmm": True}}, "gis": {"name": "invariant_interval_shape_forest", "params": {"classifier": "extratrees", "n_shapelets": 32, "segment_sizes": [4, 8, 16], "symbolic_bins": 6, "use_hmm": True}}, "cis": {"name": "state_rocket_interval_hmm", "params": {"classifier": "ridge", "n_intervals": 24, "n_kernels": 384, "use_hmm": True}}, "ese": {"name": "posterior_cpu_ensemble", "params": {"use_hmm": True, "weights": [0.34, 0.33, 0.33]}}, } DISPLAY = {"adl": "ADL", "capture24": "CAPTURE-24", "domino": "DOMINO", "gotov": "GOTOV", "handy": "HANDY", "harmes_left": "HARMES left wrist", "harmes_right": "HARMES right wrist", "iuwds": "IUWDS", "paal_adl": "PAAL-ADL", "wisdm": "WISDM", "mhealth": "mHealth", "pamap2": "PAMAP2"} def _same_as(recipe): for rid, canon in FIXED_CANONICAL.items(): if recipe["name"] == canon["name"] and recipe["params"] == canon["params"]: return rid flipped = dict(canon["params"], use_hmm=False) if recipe["name"] == canon["name"] and recipe["params"] == flipped: return f"{rid} with decoder='none'" return None def build(): aliases = json.loads(ALIASES.read_text()) short = {} for name, full in aliases.items(): short.setdefault(full, name) # first alias listed is the primary short name runs = defaultdict(list) with SOURCE.open(newline="") as f: for row in csv.DictReader(f): if int(float(row["budget"])) != FINAL_BUDGET: continue runs[row["dataset_id"]].append(row) out = [] for dataset_id in sorted(runs, key=lambda d: list(short).index(d)): rows = runs[dataset_id] if len(rows) != 30: raise ValueError(f"{dataset_id}: expected 30 final searches, found {len(rows)}") groups = defaultdict(list) for row in rows: groups[row["canonical_recipe"]].append(row) ranked = sorted(groups.items(), key=lambda kv: (-len(kv[1]), -len({r["fold"] for r in kv[1]}), kv[0])) text, chosen = ranked[0] runner_up = ranked[1][1] if len(ranked) > 1 else [] recipe = json.loads(text) name = short[dataset_id] out.append({ "id": f"best:{name}", "family": recipe["name"], "params": recipe["params"], "paper_name": f"WISP search best recipe ({DISPLAY[name]})", "source": "search_best", "provenance": { "dataset_id": dataset_id, "selected": len(chosen), "searches": len(rows), "folds_covered": len({r["fold"] for r in chosen}), "folds_total": len({r["fold"] for r in rows}), "cells": sorted([f"{r['method_id']}/fold{r['fold']}/seed{r['model_seed']}" for r in chosen]), "runner_up_selected": len(runner_up), "same_as": _same_as(recipe), "selection_rule": ("most frequently selected final recipe among the 30 budget-329 searches " "(validation utility only); ties by folds covered"), "source_file": "results/search_selections.csv", }, }) return out def main(): ap = argparse.ArgumentParser() ap.add_argument("--check", action="store_true", help="fail if the committed file differs") a = ap.parse_args() text = json.dumps(build(), indent=1, sort_keys=False) + "\n" if a.check: if TARGET.read_text() != text: raise SystemExit("search_best.json is out of date; rerun scripts/build_search_best.py") print("search_best.json is up to date") return TARGET.write_text(text) print(f"wrote {TARGET.relative_to(ROOT)}") if __name__ == "__main__": main()